- merge with master

563 files changed, 37646 insertions, 60858 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index 120d80a3f..4a1d909ed 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -6,18 +6,13 @@
 if HAVE_TESTING
  TESTING = testing
  TESTBED = testbed-logger testbed
- CONSENSUS = consensus
- SECRETSHARING = secretsharing
  ATS_TESTS = ats-tests
 endif
 
 if HAVE_EXPERIMENTAL
  EXP_DIR = \
   dv \
-  rps \
-  $(CONSENSUS) \
-  $(SECRETSHARING)
-
+  rps
 endif
 
 if HAVE_JSON
@@ -45,6 +40,10 @@ CONVERSATION_DIR = conversation
 endif
 endif
 
+if HAVE_SQLITE
+ SQLITE_DIR = sq
+endif
+
 if HAVE_MYSQL
  MYSQL_DIR = mysql my
 endif
@@ -83,6 +82,7 @@ SUBDIRS = \
   arm \
   $(TESTING) \
   peerinfo \
+  $(SQLITE_DIR) \
   $(MYSQL_DIR) \
   $(POSTGRES_DIR) \
   datacache \
@@ -111,6 +111,7 @@ SUBDIRS = \
   peerstore \
   cadet \
   set \
+  consensus \
   scalarproduct \
   revocation \
   vpn \
@@ -121,6 +122,7 @@ SUBDIRS = \
   fs \
   exit \
   pt \
+  secretsharing \
   integration-tests \
   multicast \
   psycutil \
diff --git a/src/arm/Makefile.am b/src/arm/Makefile.am
index d5e59619e..373847fde 100644
--- a/src/arm/Makefile.am
+++ b/src/arm/Makefile.am
@@ -68,7 +68,7 @@ check_SCRIPTS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)  $(check_SCRIPTS)
 endif
 
diff --git a/src/arm/arm_api.c b/src/arm/arm_api.c
index a61343833..56af544b1 100644
--- a/src/arm/arm_api.c
+++ b/src/arm/arm_api.c
@@ -753,7 +753,7 @@ change_service (struct GNUNET_ARM_Handle *h,
 
   slen = strlen (service_name) + 1;
   if (slen + sizeof (struct GNUNET_ARM_Message) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/arm/gnunet-arm.c b/src/arm/gnunet-arm.c
index b6f4d99a8..4c30985b1 100644
--- a/src/arm/gnunet-arm.c
+++ b/src/arm/gnunet-arm.c
@@ -121,12 +121,12 @@ static struct GNUNET_SCHEDULER_Task *timeout_task;
 /**
  * Do we want to give our stdout to gnunet-service-arm?
  */
-static unsigned int no_stdout;
+static int no_stdout;
 
 /**
  * Do we want to give our stderr to gnunet-service-arm?
  */
-static unsigned int no_stderr;
+static int no_stderr;
 
 /**
  * Handle for the task running the #action_loop().
@@ -220,14 +220,8 @@ req_string (enum GNUNET_ARM_RequestStatus rs)
   {
   case GNUNET_ARM_REQUEST_SENT_OK:
     return _("Message was sent successfully");
-  case GNUNET_ARM_REQUEST_CONFIGURATION_ERROR:
-    return _("Misconfiguration (can not connect to the ARM service)");
   case GNUNET_ARM_REQUEST_DISCONNECTED:
     return _("We disconnected from ARM before we could send a request");
-  case GNUNET_ARM_REQUEST_BUSY:
-    return _("ARM API is busy");
-  case GNUNET_ARM_REQUEST_TIMEOUT:
-    return _("Request timed out");
   }
   return _("Unknown request status");
 }
@@ -245,27 +239,27 @@ ret_string (enum GNUNET_ARM_Result result)
   switch (result)
   {
   case GNUNET_ARM_RESULT_STOPPED:
-    return _("%s is stopped");
+    return _("is stopped");
   case GNUNET_ARM_RESULT_STARTING:
-    return _("%s is starting");
+    return _("is starting");
   case GNUNET_ARM_RESULT_STOPPING:
-    return _("%s is stopping");
+    return _("is stopping");
   case GNUNET_ARM_RESULT_IS_STARTING_ALREADY:
-    return _("%s is starting already");
+    return _("is starting already");
   case GNUNET_ARM_RESULT_IS_STOPPING_ALREADY:
-    return _("%s is stopping already");
+    return _("is stopping already");
   case GNUNET_ARM_RESULT_IS_STARTED_ALREADY:
-    return _("%s is started already");
+    return _("is started already");
   case GNUNET_ARM_RESULT_IS_STOPPED_ALREADY:
-    return _("%s is stopped already");
+    return _("is stopped already");
   case GNUNET_ARM_RESULT_IS_NOT_KNOWN:
-    return _("%s service is not known to ARM");
+    return _("service is not known to ARM");
   case GNUNET_ARM_RESULT_START_FAILED:
-    return _("%s service failed to start");
+    return _("service failed to start");
   case GNUNET_ARM_RESULT_IN_SHUTDOWN:
-    return _("%s service cannot be started because ARM is shutting down");
+    return _("service cannot be manipulated because ARM is shutting down");
   }
-  return _("%.s Unknown result code.");
+  return _("Unknown result code.");
 }
 
 
@@ -321,23 +315,21 @@ start_callback (void *cls,
                 enum GNUNET_ARM_RequestStatus rs,
                 enum GNUNET_ARM_Result result)
 {
-  char *msg;
-
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
-    GNUNET_asprintf (&msg, "%s", _("Failed to start the ARM service: %s\n"));
-    FPRINTF (stdout, msg, req_string (rs));
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to start the ARM service: %s\n"),
+             req_string (rs));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
   if ( (GNUNET_ARM_RESULT_STARTING != result) &&
        (GNUNET_ARM_RESULT_IS_STARTED_ALREADY != result) )
   {
-    GNUNET_asprintf (&msg, "%s", _("Failed to start the ARM service: %s\n"));
-    FPRINTF (stdout, msg, ret_string (result));
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to start the ARM service: %s\n"),
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -380,10 +372,9 @@ stop_callback (void *cls,
       (GNUNET_ARM_RESULT_STOPPED != result) &&
       (GNUNET_ARM_RESULT_IS_STOPPED_ALREADY != result))
   {
-    GNUNET_asprintf (&msg, "%s",
-                     _("Failed to stop the ARM service: %s\n"));
-    FPRINTF (stdout, msg, ret_string (result));
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to stop the ARM service: %s\n"),
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -417,28 +408,23 @@ init_callback (void *cls,
                enum GNUNET_ARM_RequestStatus rs,
                enum GNUNET_ARM_Result result)
 {
-  char *msg;
-
   op = NULL;
   if (GNUNET_ARM_REQUEST_SENT_OK != rs)
   {
-    GNUNET_asprintf (&msg,
-                     _("Failed to send a request to start the `%s' service: %%s\n"),
-                     init);
-    FPRINTF (stdout, msg, req_string (rs));
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to send a request to start the `%s' service: %s\n"),
+             init,
+             req_string (rs));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
   if ((GNUNET_ARM_RESULT_STARTING != result) &&
       (GNUNET_ARM_RESULT_IS_STARTED_ALREADY != result))
   {
-    GNUNET_asprintf (&msg,
-                     _("Failed to start the `%s' service: %s\n"),
-                     init,
-                     ret_string (result));
-    FPRINTF (stdout, "%s", msg);
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to start the `%s' service: %s\n"),
+             init,
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -483,11 +469,10 @@ term_callback (void *cls,
   if ((GNUNET_ARM_RESULT_STOPPED != result) &&
       (GNUNET_ARM_RESULT_IS_STOPPED_ALREADY != result))
   {
-    GNUNET_asprintf (&msg,
-                     _("Failed to kill the `%s' service: %s\n"),
-                     term, ret_string (result));
-    FPRINTF (stdout, "%s", msg);
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to kill the `%s' service: %s\n"),
+             term,
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -777,35 +762,70 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "end", NULL, gettext_noop ("stop all GNUnet services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &end},
-    {'i', "init", "SERVICE", gettext_noop ("start a particular service"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &init},
-    {'k', "kill", "SERVICE", gettext_noop ("stop a particular service"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &term},
-    {'s', "start", NULL, gettext_noop ("start all GNUnet default services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &start},
-    {'r', "restart", NULL,
-     gettext_noop ("stop and start all GNUnet default services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &restart},
-    {'d', "delete", NULL,
-     gettext_noop ("delete config file and directory on exit"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &delete},
-    {'m', "monitor", NULL,
-     gettext_noop ("monitor ARM activities"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &monitor},
-    {'q', "quiet", NULL, gettext_noop ("don't print status messages"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &quiet},
-    {'T', "timeout", "DELAY",
-     gettext_noop ("exit with error status if operation does not finish after DELAY"),
-     GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &timeout},
-    {'I', "info", NULL, gettext_noop ("list currently running services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &list},
-    {'O', "no-stdout", NULL, gettext_noop ("don't let gnunet-service-arm inherit standard output"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &no_stdout},
-    {'E', "no-stderr", NULL, gettext_noop ("don't let gnunet-service-arm inherit standard error"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &no_stderr},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_flag ('e',
+                                  "end",
+                                  gettext_noop ("stop all GNUnet services"),
+                                  &end),
+
+    GNUNET_GETOPT_option_string ('i',
+                                 "init",
+                                 "SERVICE",
+                                 gettext_noop ("start a particular service"),
+                                 &init),
+
+    GNUNET_GETOPT_option_string ('k',
+                                 "kill",
+                                 "SERVICE",
+                                 gettext_noop ("stop a particular service"),
+                                 &term),
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "start",
+                                  gettext_noop ("start all GNUnet default services"),
+                                  &start),
+
+    GNUNET_GETOPT_option_flag ('r',
+                                  "restart",
+                                  gettext_noop ("stop and start all GNUnet default services"),
+                                  &restart),
+    GNUNET_GETOPT_option_flag ('d',
+                                  "delete",
+                                  gettext_noop ("delete config file and directory on exit"),
+                                  &delete),
+
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor ARM activities"),
+                                  &monitor),
+
+    GNUNET_GETOPT_option_flag ('q',
+                                  "quiet",
+                                  gettext_noop ("don't print status messages"),
+                                  &quiet),
+
+    GNUNET_GETOPT_option_relative_time ('T',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("exit with error status if operation does not finish after DELAY"),
+                                            &timeout),
+
+    GNUNET_GETOPT_option_flag ('I',
+                                  "info",
+                                  gettext_noop ("list currently running services"),
+                                  &list), 
+
+    GNUNET_GETOPT_option_flag ('O',
+                                  "no-stdout",
+                                  gettext_noop ("don't let gnunet-service-arm inherit standard output"),
+                                  &no_stdout),
+
+    GNUNET_GETOPT_option_flag ('E',
+                                  "no-stderr",
+                                  gettext_noop ("don't let gnunet-service-arm inherit standard error"),
+                                  &no_stderr),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/arm/gnunet-service-arm.c b/src/arm/gnunet-service-arm.c
index 7af3659a4..19088c5cb 100644
--- a/src/arm/gnunet-service-arm.c
+++ b/src/arm/gnunet-service-arm.c
@@ -290,7 +290,7 @@ add_unixpath (struct sockaddr **saddrs,
   if (GNUNET_YES == abstract)
     un->sun_path[0] = '\0';
 #endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
+#if HAVE_SOCKADDR_UN_SUN_LEN
   un->sun_len = (u_char) sizeof (struct sockaddr_un);
 #endif
   *saddrs = (struct sockaddr *) un;
@@ -812,6 +812,7 @@ start_process (struct ServiceList *sl,
                        "%s %s",
                        fin_options,
                        optpos);
+      GNUNET_free (fin_options);
       GNUNET_free (optpos);
     }
     else
@@ -2224,7 +2225,7 @@ main (int argc,
   shc_chld =
     GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD,
                                    &sighandler_child_death);
-  ret = GNUNET_SERVICE_ruN_ (argc,
+  ret = GNUNET_SERVICE_run_ (argc,
                              argv,
                              "arm",
                              GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN,
diff --git a/src/ats-tests/Makefile.am b/src/ats-tests/Makefile.am
index 21366484b..4811bc8d7 100644
--- a/src/ats-tests/Makefile.am
+++ b/src/ats-tests/Makefile.am
@@ -13,6 +13,7 @@ if USE_COVERAGE
   AM_CFLAGS = -fprofile-arcs -ftest-coverage
 endif
 
+if HAVE_EXPERIMENTAL
 if HAVE_LIBGLPK
  PERF_MLP = \
  perf_ats_mlp_transport_none \
@@ -22,6 +23,7 @@ if HAVE_LIBGLPK
  perf_ats_mlp_core_bandwidth \
  perf_ats_mlp_core_latency
 endif
+endif
 
 if HAVE_TESTING
 TESTING_TESTS = \
@@ -44,7 +46,7 @@ check_PROGRAMS = \
  $(TESTING_TESTS)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
@@ -140,7 +142,7 @@ perf_ats_proportional_transport_bandwidth_LDADD = \
 perf_ats_proportional_transport_bandwidth_DEPENDENCIES = \
  libgnunetatstesting.la \
  $(top_builddir)/src/util/libgnunetutil.la
- 
+
 perf_ats_proportional_core_latency_SOURCES = \
   perf_ats.c
 perf_ats_proportional_core_latency_LDADD = \
diff --git a/src/ats-tests/gnunet-ats-sim.c b/src/ats-tests/gnunet-ats-sim.c
index 56f8f2223..0f32df511 100644
--- a/src/ats-tests/gnunet-ats-sim.c
+++ b/src/ats-tests/gnunet-ats-sim.c
@@ -81,15 +81,19 @@ evaluate (struct GNUNET_TIME_Relative duration_total)
 
 
   duration = (duration_total.rel_value_us / (1000 * 1000));
+  if (0 == duration)
+    duration = 1;
   for (c_m = 0; c_m < e->num_masters; c_m++)
   {
     mp = &masters_p[c_m];
     fprintf (stderr,
-        _("Master [%u]: sent: %u KiB in %u sec. = %u KiB/s, received: %u KiB in %u sec. = %u KiB/s\n"),
-        mp->no, mp->total_bytes_sent / 1024, duration,
-        (mp->total_bytes_sent / 1024) / duration,
-        mp->total_bytes_received / 1024, duration,
-        (mp->total_bytes_received / 1024) / duration);
+             _("Master [%u]: sent: %u KiB in %u sec. = %u KiB/s, received: %u KiB in %u sec. = %u KiB/s\n"),
+             mp->no, mp->total_bytes_sent / 1024,
+             duration,
+             (mp->total_bytes_sent / 1024) / duration,
+             mp->total_bytes_received / 1024,
+             duration,
+             (mp->total_bytes_received / 1024) / duration);
 
     for (c_s = 0; c_s < e->num_slaves; c_s++)
     {
diff --git a/src/ats-tests/gnunet-solver-eval.c b/src/ats-tests/gnunet-solver-eval.c
index 0e49a3a32..04508d1df 100644
--- a/src/ats-tests/gnunet-solver-eval.c
+++ b/src/ats-tests/gnunet-solver-eval.c
@@ -331,12 +331,14 @@ load_episode (struct Experiment *e,
             o->pref_type = GNUNET_ATS_PREFERENCE_LATENCY;
           else
           {
-              fprintf (stderr, "Invalid preference in operation %u `%s' in episode %u\n",
-                  op_counter, op, cur->id);
+              fprintf (stderr,
+                       "Invalid preference in operation %u `%s' in episode %u\n",
+                       op_counter,
+                       op,
+                       cur->id);
               GNUNET_free (type);
               GNUNET_free (op_name);
               GNUNET_free (op);
-              GNUNET_free (pref);
               GNUNET_free (sec_name);
               GNUNET_free_non_null (pref);
               GNUNET_free (o);
@@ -929,22 +931,35 @@ main (int argc, char *argv[])
   opt_log = GNUNET_NO;
   opt_plot = GNUNET_NO;
 
-  static struct GNUNET_GETOPT_CommandLineOption options[] =
-  {
-    { 's', "solver", NULL,
-        gettext_noop ("solver to use"),
-        1, &GNUNET_GETOPT_set_string, &opt_solver},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_string, &opt_exp_file},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_one, &opt_verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] =
+  {
+    GNUNET_GETOPT_option_string ('s',
+                                 "solver",
+                                 NULL,
+                                 gettext_noop ("solver to use"),
+                                 &opt_solver),
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "experiment",
+                                 NULL,
+                                 gettext_noop ("experiment to use"),
+                                 &opt_exp_file),
+ 
+    GNUNET_GETOPT_option_flag ('e',
+                                  "experiment",
+                                  gettext_noop ("experiment to use"),
+                                  &opt_verbose),
     GNUNET_GETOPT_OPTION_END
   };
 
-  GNUNET_PROGRAM_run (argc, argv, argv[0], NULL, options, &run, argv[0]);
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run (argc,
+                          argv, argv[0],
+                          NULL,
+                          options,
+                          &run, argv[0]))
+    return 1;
 
   return 0;
 }
-/* end of file ats-testing-experiment.c*/
+/* end of file gnunet-solver-eval.c*/
diff --git a/src/ats-tests/template_perf_ats.conf b/src/ats-tests/template_perf_ats.conf
index 05d11449d..9d320c3dd 100644
--- a/src/ats-tests/template_perf_ats.conf
+++ b/src/ats-tests/template_perf_ats.conf
@@ -14,6 +14,8 @@ USE_INCLUDED_HELLOS = NO
 #PREFIX = valgrind --leak-check=yes
 
 [ats]
+# PREFIX = valgrind
+
 # Network specific inbound/outbound quotas
 UNSPECIFIED_QUOTA_IN = 128 KiB
 UNSPECIFIED_QUOTA_OUT = 128 KiB
@@ -47,4 +49,4 @@ CDTU8QQ8UPLGHR3B91V0CLTDOHONLB8QGHGUEM2JM1GANTEV0O6T20SD2N2HDN2QSHDG6IDTBR48KRDC
 
 [transport-blacklist-548J7M14O4I0F8I84U0UFARVJ97DB6QOT3MCA8O8SNAIT5JJ8TR95LUVAP3N5L7DN33IB49SNMF3Q3C0VPLTGP9ASCULA9S2OIMHHH8]
 HIJN5O404QNUR37OSJUTNJ6H2KJS198DHI2J3I8SE3DMKVRG1RNQPODN1IJBF14KEMPPPRM0B9F9ILFKHOFCA655CH6M5OCNCMR0FE0 =
-CDTU8QQ8UPLGHR3B91V0CLTDOHONLB8QGHGUEM2JM1GANTEV0O6T20SD2N2HDN2QSHDG6IDTBR48KRDCS601FI6VHG59E7DQA98JD2O =
\ No newline at end of file
+CDTU8QQ8UPLGHR3B91V0CLTDOHONLB8QGHGUEM2JM1GANTEV0O6T20SD2N2HDN2QSHDG6IDTBR48KRDCS601FI6VHG59E7DQA98JD2O =
diff --git a/src/ats-tool/gnunet-ats.c b/src/ats-tool/gnunet-ats.c
index 5fc1d6e92..5ec7693b1 100644
--- a/src/ats-tool/gnunet-ats.c
+++ b/src/ats-tool/gnunet-ats.c
@@ -944,34 +944,62 @@ main (int argc,
   stat_receive_done = GNUNET_NO;
   opt_type_str = NULL;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'u', "used", NULL,
-      gettext_noop ("get list of active addresses currently used"), 0,
-      &GNUNET_GETOPT_set_one, &opt_list_used },
-    { 'a', "all", NULL, gettext_noop ("get list of all active addresses"), 0,
-      &GNUNET_GETOPT_set_one, &opt_list_all },
-    { 'C', "connect", "PEER",
-      gettext_noop ("connect to PEER"), 1,
-      &GNUNET_GETOPT_set_string, &cpid_str },
-    { 'n', "numeric", NULL,
-      gettext_noop ("do not resolve IP addresses to hostnames"), 0,
-      &GNUNET_GETOPT_set_one, &opt_resolve_addresses_numeric },
-    { 'm', "monitor", NULL, gettext_noop ("monitor mode"), 0,
-      &GNUNET_GETOPT_set_one, &opt_monitor },
-    { 'p', "preference", NULL, gettext_noop ("set preference for the given peer"),
-      0, &GNUNET_GETOPT_set_one, &opt_set_pref },
-    { 'q', "quotas", NULL, gettext_noop ("print all configured quotas"), 0,
-      &GNUNET_GETOPT_set_one, &opt_print_quotas },
-    { 'i', "id", "TYPE", gettext_noop ("peer id"), 1, &GNUNET_GETOPT_set_string,
-      &opt_pid_str },
-    { 't', "type", "TYPE",
-      gettext_noop ("preference type to set: latency | bandwidth"), 1,
-      &GNUNET_GETOPT_set_string, &opt_type_str },
-    { 'k', "value", "VALUE", gettext_noop ("preference value"), 1,
-      &GNUNET_GETOPT_set_uint, &opt_pref_value },
-    { 'V', "verbose", NULL,
-      gettext_noop ("verbose output (include ATS address properties)"), 0,
-      &GNUNET_GETOPT_set_one, &opt_verbose },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('u',
+                                  "used",
+                                  gettext_noop ("get list of active addresses currently used"),
+                                  &opt_list_used),
+    GNUNET_GETOPT_option_flag ('a',
+                                  "all",
+                                  gettext_noop ("get list of all active addresses"),
+                                  &opt_list_all),
+
+    GNUNET_GETOPT_option_string ('C',
+                                 "connect",
+                                 NULL,
+                                 gettext_noop ("connect to PEER"),
+                                 &cpid_str),
+    GNUNET_GETOPT_option_flag ('n',
+                                  "numeric",
+                                  gettext_noop ("do not resolve IP addresses to hostnames"),
+                                  &opt_resolve_addresses_numeric),
+
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor mode"),
+                                  &opt_monitor),
+
+    GNUNET_GETOPT_option_flag ('p',
+                                  "preference",
+                                  gettext_noop ("set preference for the given peer"),
+                                  &opt_set_pref),
+
+    GNUNET_GETOPT_option_flag ('q', 
+                                  "quotas",
+                                  gettext_noop ("print all configured quotas"),
+                                  &opt_print_quotas),
+    GNUNET_GETOPT_option_string ('i',
+                                 "id",
+                                 "TYPE",
+                                 gettext_noop ("peer id"),
+                                 &opt_pid_str),
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("preference type to set: latency | bandwidth"),
+                                 &opt_type_str),
+
+    GNUNET_GETOPT_option_uint ('k',
+                                   "value",
+                                   "VALUE",
+                                   gettext_noop ("preference value"),
+                                   &opt_pref_value),
+
+    GNUNET_GETOPT_option_flag ('V',
+                                  "verbose",
+                                  gettext_noop ("verbose output (include ATS address properties)"),
+                                  &opt_verbose),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/ats/.gitignore b/src/ats/.gitignore
index 983dc2843..97f1088b9 100644
--- a/src/ats/.gitignore
+++ b/src/ats/.gitignore
@@ -1,3 +1,5 @@
 gnunet-service-ats
 test_ats_api_proportional
 test_ats_reservation_api_proportional
+test_ats_api_mlp
+test_ats_api_ril
diff --git a/src/ats/Makefile.am b/src/ats/Makefile.am
index aa3612c2d..759dac0be 100644
--- a/src/ats/Makefile.am
+++ b/src/ats/Makefile.am
@@ -21,14 +21,16 @@ endif
 lib_LTLIBRARIES = libgnunetats.la
 
 plugin_LTLIBRARIES = \
-  libgnunet_plugin_ats_proportional.la \
-  $(GN_MLP_LIB) \
-  libgnunet_plugin_ats_ril.la
+  libgnunet_plugin_ats_proportional.la
 
+if HAVE_EXPERIMENTAL
+plugin_LTLIBRARIES += \
+  libgnunet_plugin_ats_ril.la
 if HAVE_LIBGLPK
 plugin_LTLIBRARIES += \
   libgnunet_plugin_ats_mlp.la
 endif
+endif
 
 libgnunetats_la_SOURCES = \
   ats_api_connectivity.c \
@@ -54,7 +56,6 @@ libgnunet_plugin_ats_proportional_la_LDFLAGS = \
   $(GN_PLUGIN_LDFLAGS)
 
 
-if HAVE_LIBGLPK
 libgnunet_plugin_ats_mlp_la_SOURCES = \
   plugin_ats_mlp.c
 libgnunet_plugin_ats_mlp_la_LIBADD = \
@@ -64,7 +65,6 @@ libgnunet_plugin_ats_mlp_la_LIBADD = \
 libgnunet_plugin_ats_mlp_la_LDFLAGS = \
   $(GN_PLUGIN_LDFLAGS) \
   -lglpk
-endif
 
 libgnunet_plugin_ats_ril_la_SOURCES = \
   plugin_ats_ril.c
@@ -99,7 +99,7 @@ if HAVE_TESTING
 TESTING_TESTS = \
  test_ats_api_proportional \
  test_ats_reservation_api_proportional
-if HAVE_WACHS
+if HAVE_EXPERIMENTAL
 TESTING_TESTS += \
  test_ats_api_ril
 if HAVE_LIBGLPK
@@ -113,7 +113,7 @@ check_PROGRAMS = \
  $(TESTING_TESTS)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/ats/ats_api_performance.c b/src/ats/ats_api_performance.c
index cd67583d1..c5b126e08 100644
--- a/src/ats/ats_api_performance.c
+++ b/src/ats/ats_api_performance.c
@@ -816,7 +816,7 @@ GNUNET_ATS_performance_list_addresses_cancel (struct GNUNET_ATS_AddressListHandl
  * @return a string or NULL if invalid
  */
 const char *
-GNUNET_ATS_print_preference_type (uint32_t type)
+GNUNET_ATS_print_preference_type (enum GNUNET_ATS_PreferenceKind type)
 {
   const char *prefs[] = GNUNET_ATS_PreferenceTypeString;
 
diff --git a/src/ats/ats_api_scheduling.c b/src/ats/ats_api_scheduling.c
index faeeb6081..81ae01b6a 100644
--- a/src/ats/ats_api_scheduling.c
+++ b/src/ats/ats_api_scheduling.c
@@ -657,9 +657,9 @@ GNUNET_ATS_address_add (struct GNUNET_ATS_SchedulingHandle *sh,
   GNUNET_break (GNUNET_ATS_NET_UNSPECIFIED != prop->scope);
   namelen = strlen (address->transport_name) + 1;
   msize = address->address_length + namelen;
-  if ((msize + sizeof (struct AddressUpdateMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (address->address_length >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (namelen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) )
+  if ((msize + sizeof (struct AddressUpdateMessage) >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (address->address_length >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (namelen >= GNUNET_MAX_MESSAGE_SIZE) )
   {
     /* address too large for us, this should not happen */
     GNUNET_break (0);
diff --git a/src/ats/gnunet-ats-solver-eval.c b/src/ats/gnunet-ats-solver-eval.c
index e2e68562c..5af2fbce3 100644
--- a/src/ats/gnunet-ats-solver-eval.c
+++ b/src/ats/gnunet-ats-solver-eval.c
@@ -3289,24 +3289,33 @@ main (int argc, char *argv[])
 
   static struct GNUNET_GETOPT_CommandLineOption options[] =
   {
-    { 's', "solver", NULL,
-        gettext_noop ("solver to use"),
-        1, &GNUNET_GETOPT_set_string, &opt_solver},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_string, &opt_exp_file},
-    {  'V', "verbose", NULL,
-      gettext_noop ("be verbose"),
-      0, &GNUNET_GETOPT_set_one, &opt_verbose},
-    {  'p', "print", NULL,
-      gettext_noop ("print logging"),
-      0, &GNUNET_GETOPT_set_one, &opt_print},
-    {  'f', "file", NULL,
-        gettext_noop ("save logging to disk"),
-        0, &GNUNET_GETOPT_set_one, &opt_save},
-    {  'd', "dn", NULL,
-        gettext_noop ("disable normalization"),
-        0, &GNUNET_GETOPT_set_one, &opt_disable_normalization},
+    GNUNET_GETOPT_option_string ('s',
+                                 "solver",
+                                 gettext_noop ("solver to use"),
+                                 &opt_solver),
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "experiment"
+                                 gettext_noop ("experiment to use"),
+                                 &opt_exp_file),
+
+    GNUNET_GETOPT_option_verbose (&opt_verbose),
+
+    GNUNET_GETOPT_option_flag ('p', 
+                                  "print",
+                                  gettext_noop ("print logging"),
+                                  &opt_print),
+
+    GNUNET_GETOPT_option_flag ('f',
+                                  "file",
+                                  gettext_noop ("save logging to disk"),
+                                  &opt_save),
+
+    GNUNET_GETOPT_option_flag ('d', 
+                                  "dn",
+                                  gettext_noop ("disable normalization"),
+                                  &opt_disable_normalization),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/ats/gnunet-service-ats_addresses.c b/src/ats/gnunet-service-ats_addresses.c
index 1a4a33206..ba34cbacb 100644
--- a/src/ats/gnunet-service-ats_addresses.c
+++ b/src/ats/gnunet-service-ats_addresses.c
@@ -571,7 +571,7 @@ transmit_req_addr (struct AddressIteration *ai,
   msize = plugin_addr_len + plugin_name_length;
 
   GNUNET_assert (sizeof (struct PeerInformationMessage) + msize
-                 < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 < GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (msg,
                              msize,
                              GNUNET_MESSAGE_TYPE_ATS_ADDRESSLIST_RESPONSE);
diff --git a/src/ats/gnunet-service-ats_performance.c b/src/ats/gnunet-service-ats_performance.c
index 5252a71bc..07ddf0531 100644
--- a/src/ats/gnunet-service-ats_performance.c
+++ b/src/ats/gnunet-service-ats_performance.c
@@ -85,7 +85,7 @@ notify_client (struct GNUNET_SERVICE_Client *client,
 
   if (NULL != prop)
     GNUNET_break (GNUNET_ATS_NET_UNSPECIFIED != prop->scope);
-  GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (msize < GNUNET_MAX_MESSAGE_SIZE);
   msg = (struct PeerInformationMessage *) buf;
   msg->header.size = htons (msize);
   msg->header.type = htons (GNUNET_MESSAGE_TYPE_ATS_PEER_INFORMATION);
diff --git a/src/ats/perf_ats_solver.c b/src/ats/perf_ats_solver.c
index f05668e9b..7fae3624f 100644
--- a/src/ats/perf_ats_solver.c
+++ b/src/ats/perf_ats_solver.c
@@ -1404,27 +1404,42 @@ main (int argc, char *argv[])
   ph.total_iterations = 1;
 
   static struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'a', "addresses", NULL,
-          gettext_noop ("addresses to use"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_address },
-      { 's', "start", NULL,
-          gettext_noop ("start with peer"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_peers_start },
-      { 'e', "end", NULL,
-          gettext_noop ("end with peer"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_peers_end },
-      { 'i', "iterations", NULL,
-          gettext_noop ("number of iterations used for averaging (default: 1)"),
-          1, &GNUNET_GETOPT_set_uint, &ph.total_iterations },
-      { 'p', "percentage", NULL,
-          gettext_noop ("update a fix percentage of addresses"),
-          1, &GNUNET_GETOPT_set_uint, &ph.opt_update_percent },
-      { 'd', "data", NULL,
-          gettext_noop ("create data file"),
-          0, &GNUNET_GETOPT_set_one, &ph.create_datafile},
-      { 'u', "update", NULL,
-          gettext_noop ("measure updates"),
-          0, &GNUNET_GETOPT_set_one, &ph.measure_updates},
+
+      GNUNET_GETOPT_option_uint ('a',
+                                     "addresses",
+                                     gettext_noop ("addresses to use"),
+                                     &ph.N_address),
+
+      GNUNET_GETOPT_option_uint ('s',
+                                     "start",
+                                     gettext_noop ("start with peer"),
+                                     &ph.N_peers_start),
+
+      GNUNET_GETOPT_option_uint ('e',
+                                     "end",
+                                     gettext_noop ("end with peer"),
+                                     &ph.N_peers_end),
+
+      GNUNET_GETOPT_option_uint ('i',
+                                     "iterations",
+                                     gettext_noop ("number of iterations used for averaging (default: 1)"),
+                                     &ph.total_iterations),
+
+      GNUNET_GETOPT_option_uint ('p',
+                                     "percentage",
+                                     gettext_noop ("update a fix percentage of addresses"),
+                                     &ph.opt_update_percent),
+
+      GNUNET_GETOPT_option_flag ('d',
+                                    "data",
+                                    gettext_noop ("create data file"),
+                                    &ph.create_datafile),
+
+      GNUNET_GETOPT_option_flag ('u',
+                                    "update",
+                                    gettext_noop ("measure updates"),
+                                    &ph.measure_updates),
+
       GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/auction/Makefile.am b/src/auction/Makefile.am
index 87f917283..bdede0ce0 100644
--- a/src/auction/Makefile.am
+++ b/src/auction/Makefile.am
@@ -1,74 +1,74 @@
 # This Makefile.am is in the public domain
 AM_CPPFLAGS = -I$(top_srcdir)/src/include
 
-pkgcfgdir= $(pkgdatadir)/config.d/
+pkgcfgdir = $(pkgdatadir)/config.d/
 
-libexecdir= $(pkglibdir)/libexec/
+libexecdir = $(pkglibdir)/libexec/
 
 
 pkgcfg_DATA = \
-  auction.conf
+        auction.conf
 
 if MINGW
- WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols
+        WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols
 endif
 
 if USE_COVERAGE
-  AM_CFLAGS = -fprofile-arcs -ftest-coverage
+        AM_CFLAGS = -fprofile-arcs -ftest-coverage
 endif
 
 
 libexec_PROGRAMS = \
- gnunet-service-auction
+        gnunet-service-auction
 
 gnunet_service_auction_SOURCES = \
- gnunet-service-auction.c
+        gnunet-service-auction.c
 gnunet_service_auction_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -ljansson \
-  $(GN_LIBINTL)
+        $(top_builddir)/src/util/libgnunetutil.la \
+        -ljansson \
+        $(GN_LIBINTL)
 
 
 bin_PROGRAMS = \
- gnunet-auction-create \
- gnunet-auction-info \
- gnunet-auction-join
+        gnunet-auction-create \
+        gnunet-auction-info \
+        gnunet-auction-join
 
 gnunet_auction_create_SOURCES = \
- gnunet-auction-create.c
+        gnunet-auction-create.c
 gnunet_auction_create_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -ljansson \
-  $(GN_LIBINTL)
+        $(top_builddir)/src/util/libgnunetutil.la \
+        -ljansson \
+        $(GN_LIBINTL)
 
 gnunet_auction_info_SOURCES = \
- gnunet-auction-info.c
+        gnunet-auction-info.c
 gnunet_auction_info_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -ljansson \
-  $(GN_LIBINTL)
+        $(top_builddir)/src/util/libgnunetutil.la \
+        -ljansson \
+        $(GN_LIBINTL)
 
 gnunet_auction_join_SOURCES = \
- gnunet-auction-join.c
+        gnunet-auction-join.c
 gnunet_auction_join_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -ljansson \
-  $(GN_LIBINTL)
+        $(top_builddir)/src/util/libgnunetutil.la \
+        -ljansson \
+        $(GN_LIBINTL)
 
 
 check_PROGRAMS = \
- test_auction_api
+        test_auction_api
 
 test_auction_api_SOURCES = \
- test_auction_api.c
+        test_auction_api.c
 test_auction_api_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la
+        $(top_builddir)/src/util/libgnunetutil.la
 
 
 check_SCRIPTS = \
- test_auction_create.sh
+        test_auction_create.sh
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
-TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
+        AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
+        TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
 endif
diff --git a/src/auction/gnunet-auction-create.c b/src/auction/gnunet-auction-create.c
index a4c029572..9e6c23b88 100644
--- a/src/auction/gnunet-auction-create.c
+++ b/src/auction/gnunet-auction-create.c
@@ -155,30 +155,49 @@ fail:
 int
 main (int argc, char *const *argv)
 {
-        static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-                {'d', "description", "FILE",
-                        gettext_noop ("description of the item to be sold"),
-                        1, &GNUNET_GETOPT_set_filename, &fndesc},
-                {'p', "pricemap", "FILE",
-                        gettext_noop ("mapping of possible prices"),
-                        1, &GNUNET_GETOPT_set_filename, &fnprices},
-                {'r', "roundtime", "DURATION",
-                        gettext_noop ("max duration per round"),
-                        1, &GNUNET_GETOPT_set_relative_time, &dround},
-                {'s', "regtime", "DURATION",
-                        gettext_noop ("duration until auction starts"),
-                        1, &GNUNET_GETOPT_set_relative_time, &dstart},
-                {'m', "m", "NUMBER",
-                        gettext_noop ("number of items to sell\n"
-                                      "0 for first price auction\n"
-                                      ">0 for vickrey/M+1st price auction"),
-                        1, &GNUNET_GETOPT_set_uint, &m},
-                {'u', "public", NULL,
-                        gettext_noop ("public auction outcome"),
-                        0, &GNUNET_GETOPT_set_one, &outcome},
-                {'i', "interactive", NULL,
-                        gettext_noop ("keep running in foreground until auction completes"),
-                        0, &GNUNET_GETOPT_set_one, &interactive},
+        struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+                GNUNET_GETOPT_option_filename ('d',
+                                               "description",
+                                               "FILE",
+                                               gettext_noop ("description of the item to be sold"),
+                                               &fndesc),
+
+                GNUNET_GETOPT_option_filename ('p',
+                                               "pricemap",
+                                               "FILE",
+                                               gettext_noop ("mapping of possible prices"),
+                                               &fnprices),
+
+                GNUNET_GETOPT_option_relative_time ('r',
+                                                        "roundtime",
+                                                        "DURATION",
+                                                        gettext_noop ("max duration per round"),
+                                                        &dround),
+
+                GNUNET_GETOPT_option_relative_time ('s',
+                                                        "regtime",
+                                                        "DURATION",
+                                                        gettext_noop ("duration until auction starts"),
+                                                        &dstart),
+                GNUNET_GETOPT_option_uint ('m',
+                                               "m",
+                                               "NUMBER",
+                                               gettext_noop ("number of items to sell\n"
+                                                             "0 for first price auction\n"
+                                                             ">0 for vickrey/M+1st price auction"),
+                                               &m), 
+
+                GNUNET_GETOPT_option_flag ('u',
+                                              "public",
+                                              gettext_noop ("public auction outcome"),
+                                              &outcome),
+
+                GNUNET_GETOPT_option_flag ('i',
+                                              "interactive",
+                                              gettext_noop ("keep running in foreground until auction completes"),
+                                              &interactive),
+
                 GNUNET_GETOPT_OPTION_END
         };
         if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/block/Makefile.am b/src/block/Makefile.am
index c54a4c246..05df7541d 100644
--- a/src/block/Makefile.am
+++ b/src/block/Makefile.am
@@ -11,7 +11,9 @@ if USE_COVERAGE
   AM_CFLAGS = --coverage
 endif
 
-lib_LTLIBRARIES = libgnunetblock.la
+lib_LTLIBRARIES = \
+  libgnunetblock.la \
+  libgnunetblockgroup.la
 
 plugin_LTLIBRARIES = \
   libgnunet_plugin_block_test.la
@@ -24,7 +26,7 @@ noinst_LTLIBRARIES = \
 libgnunet_plugin_block_template_la_SOURCES = \
   plugin_block_template.c
 libgnunet_plugin_block_template_la_LIBADD = \
-  libgnunetblock.la \
+  libgnunetblockgroup.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(LTLIBINTL)
 libgnunet_plugin_block_template_la_LDFLAGS = \
@@ -33,8 +35,8 @@ libgnunet_plugin_block_template_la_LDFLAGS = \
 libgnunet_plugin_block_test_la_SOURCES = \
   plugin_block_test.c
 libgnunet_plugin_block_test_la_LIBADD = \
-  libgnunetblock.la \
-  $(top_builddir)/src/util/libgnunetutil.la  \
+  libgnunetblockgroup.la \
+$(top_builddir)/src/util/libgnunetutil.la  \
   $(LTLIBINTL)
 libgnunet_plugin_block_test_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
@@ -49,3 +51,16 @@ libgnunetblock_la_DEPENDENCIES = \
 libgnunetblock_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS) \
   -version-info 0:0:0
+
+
+libgnunetblockgroup_la_SOURCES = \
+  bg_bf.c
+libgnunetblockgroup_la_LIBADD = \
+ libgnunetblock.la \
+ $(top_builddir)/src/util/libgnunetutil.la
+libgnunetblockgroup_la_DEPENDENCIES = \
+ libgnunetblock.la \
+ $(top_builddir)/src/util/libgnunetutil.la
+libgnunetblockgroup_la_LDFLAGS = \
+  $(GN_LIB_LDFLAGS) \
+  -version-info 0:0:0
diff --git a/src/block/bg_bf.c b/src/block/bg_bf.c
new file mode 100644
index 000000000..3e7d38892
--- /dev/null
+++ b/src/block/bg_bf.c
@@ -0,0 +1,268 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file block/bg_bf.c
+ * @brief implementation of a block group using a Bloom filter
+ *        to drop duplicate blocks
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_block_group_lib.h"
+#include "gnunet_block_plugin.h"
+
+
+/**
+ * Internal data structure for a block group.
+ */
+struct BfGroupInternals
+{
+  /**
+   * A Bloom filter to weed out duplicate replies probabilistically.
+   */
+  struct GNUNET_CONTAINER_BloomFilter *bf;
+
+  /**
+   * Set from the nonce to mingle the hashes before going into the @e bf.
+   */
+  uint32_t bf_mutator;
+
+  /**
+   * Size of @a bf.
+   */
+  uint32_t bf_size;
+
+};
+
+
+/**
+ * Serialize state of a block group.
+ *
+ * @param bg group to serialize
+ * @param[out] nonce set to the nonce of the @a bg
+ * @param[out] raw_data set to the serialized state
+ * @param[out] raw_data_size set to the number of bytes in @a raw_data
+ * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
+ *         supported, #GNUNET_SYSERR on error
+ */
+static int
+bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg,
+                       uint32_t *nonce,
+                       void **raw_data,
+                       size_t *raw_data_size)
+{
+  struct BfGroupInternals *gi = bg->internal_cls;
+  char *raw;
+
+  raw = GNUNET_malloc (gi->bf_size);
+  if (GNUNET_OK !=
+      GNUNET_CONTAINER_bloomfilter_get_raw_data (gi->bf,
+                                                 raw,
+                                                 gi->bf_size))
+  {
+    GNUNET_free (raw);
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *nonce = gi->bf_mutator;
+  *raw_data = raw;
+  *raw_data_size = gi->bf_size;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Mark elements as "seen" using a hash of the element. Not supported
+ * by all block plugins.
+ *
+ * @param bg group to update
+ * @param seen_results results already seen
+ * @param seen_results_count number of entries in @a seen_results
+ */
+static void
+bf_group_mark_seen_cb (struct GNUNET_BLOCK_Group *bg,
+                       const struct GNUNET_HashCode *seen_results,
+                       unsigned int seen_results_count)
+{
+  struct BfGroupInternals *gi = bg->internal_cls;
+
+  for (unsigned int i=0;i<seen_results_count;i++)
+  {
+    struct GNUNET_HashCode mhash;
+
+    GNUNET_BLOCK_mingle_hash (&seen_results[i],
+                              gi->bf_mutator,
+                              &mhash);
+    GNUNET_CONTAINER_bloomfilter_add (gi->bf,
+                                      &mhash);
+  }
+}
+
+
+/**
+ * Merge two groups, if possible. Not supported by all block plugins,
+ * can also fail if the nonces were different.
+ *
+ * @param bg1 group to update
+ * @param bg2 group to merge into @a bg1
+ * @return #GNUNET_OK on success, #GNUNET_NO if the nonces were different and thus
+ *         we failed.
+ */
+static int
+bf_group_merge_cb (struct GNUNET_BLOCK_Group *bg1,
+                   const struct GNUNET_BLOCK_Group *bg2)
+{
+  struct BfGroupInternals *gi1 = bg1->internal_cls;
+  struct BfGroupInternals *gi2 = bg2->internal_cls;
+
+  if (gi1->bf_mutator != gi2->bf_mutator)
+    return GNUNET_NO;
+  if (gi1->bf_size != gi2->bf_size)
+    return GNUNET_NO;
+  GNUNET_CONTAINER_bloomfilter_or2 (gi1->bf,
+                                    gi2->bf);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Destroy resources used by a block group.
+ *
+ * @param bg group to destroy, NULL is allowed
+ */
+static void
+bf_group_destroy_cb (struct GNUNET_BLOCK_Group *bg)
+{
+  struct BfGroupInternals *gi = bg->internal_cls;
+
+  GNUNET_CONTAINER_bloomfilter_free (gi->bf);
+  GNUNET_free (gi);
+  GNUNET_free (bg);
+}
+
+
+/**
+ * Create a new block group that filters duplicates using a Bloom filter.
+ *
+ * @param ctx block context in which the block group is created
+ * @param bf_size size of the Bloom filter
+ * @param bf_k K-value for the Bloom filter
+ * @param type block type
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+struct GNUNET_BLOCK_Group *
+GNUNET_BLOCK_GROUP_bf_create (void *cls,
+                              size_t bf_size,
+                              unsigned int bf_k,
+                              enum GNUNET_BLOCK_Type type,
+                              uint32_t nonce,
+                              const void *raw_data,
+                              size_t raw_data_size)
+{
+  struct BfGroupInternals *gi;
+  struct GNUNET_BLOCK_Group *bg;
+
+  gi = GNUNET_new (struct BfGroupInternals);
+  gi->bf = GNUNET_CONTAINER_bloomfilter_init ((bf_size != raw_data_size) ? NULL : raw_data,
+                                              bf_size,
+                                              bf_k);
+  gi->bf_mutator = nonce;
+  gi->bf_size = bf_size;
+  bg = GNUNET_new (struct GNUNET_BLOCK_Group);
+  bg->type = type;
+  bg->serialize_cb = &bf_group_serialize_cb;
+  bg->mark_seen_cb = &bf_group_mark_seen_cb;
+  bg->merge_cb = &bf_group_merge_cb;
+  bg->destroy_cb = &bf_group_destroy_cb;
+  bg->internal_cls = gi;
+  return bg;
+}
+
+
+/**
+ * Test if @a hc is contained in the Bloom filter of @a bg.  If so,
+ * return #GNUNET_YES.  If not, add @a hc to the Bloom filter and
+ * return #GNUNET_NO.
+ *
+ * @param bg block group to use for testing
+ * @param hc hash of element to evaluate
+ * @return #GNUNET_YES if @a hc is (likely) a duplicate
+ *         #GNUNET_NO if @a hc was definitively not in @bg (but now is)
+ */
+int
+GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg,
+                                    const struct GNUNET_HashCode *hc)
+{
+  struct BfGroupInternals *gi;
+  struct GNUNET_HashCode mhash;
+
+  if (NULL == bg)
+    return GNUNET_NO;
+  gi = bg->internal_cls;
+  GNUNET_BLOCK_mingle_hash (hc,
+                            gi->bf_mutator,
+                            &mhash);
+  if (GNUNET_YES ==
+      GNUNET_CONTAINER_bloomfilter_test (gi->bf,
+                                         &mhash))
+    return GNUNET_YES;
+  GNUNET_CONTAINER_bloomfilter_add (gi->bf,
+                                    &mhash);
+  return GNUNET_NO;
+}
+
+
+/**
+ * How many bytes should a bloomfilter be if we have already seen
+ * entry_count responses?  Sized so that do not have to
+ * re-size the filter too often (to keep it cheap).
+ *
+ * Since other peers will also add entries but not resize the filter,
+ * we should generally pick a slightly larger size than what the
+ * strict math would suggest.
+ *
+ * @param entry_count expected number of entries in the Bloom filter
+ * @param k number of bits set per entry
+ * @return must be a power of two and smaller or equal to 2^15.
+ */
+size_t
+GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count,
+                                             unsigned int k)
+{
+  size_t size;
+  unsigned int ideal = (entry_count * k) / 4;
+  uint16_t max = 1 << 15;
+
+  if (entry_count > max)
+    return max;
+  size = 8;
+  while ((size < max) && (size < ideal))
+    size *= 2;
+  if (size > max)
+    return max;
+  return size;
+}
+
+
+/* end of bg_bf.c */
diff --git a/src/block/block.c b/src/block/block.c
index c104f4bd1..4b6f3826d 100644
--- a/src/block/block.c
+++ b/src/block/block.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2010 GNUnet e.V.
+     Copyright (C) 2010, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -84,8 +84,12 @@ GNUNET_BLOCK_mingle_hash (const struct GNUNET_HashCode *in,
 {
   struct GNUNET_HashCode m;
 
-  GNUNET_CRYPTO_hash (&mingle_number, sizeof (uint32_t), &m);
-  GNUNET_CRYPTO_hash_xor (&m, in, hc);
+  GNUNET_CRYPTO_hash (&mingle_number,
+                      sizeof (uint32_t),
+                      &m);
+  GNUNET_CRYPTO_hash_xor (&m,
+                          in,
+                          hc);
 }
 
 
@@ -111,7 +115,9 @@ add_plugin (void *cls,
   plugin = GNUNET_new (struct Plugin);
   plugin->api = api;
   plugin->library_name = GNUNET_strdup (library_name);
-  GNUNET_array_append (ctx->plugins, ctx->num_plugins, plugin);
+  GNUNET_array_append (ctx->plugins,
+                       ctx->num_plugins,
+                       plugin);
 }
 
 
@@ -129,7 +135,10 @@ GNUNET_BLOCK_context_create (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
   ctx = GNUNET_new (struct GNUNET_BLOCK_Context);
   ctx->cfg = cfg;
-  GNUNET_PLUGIN_load_all ("libgnunet_plugin_block_", NULL, &add_plugin, ctx);
+  GNUNET_PLUGIN_load_all ("libgnunet_plugin_block_",
+                          (void *) cfg,
+                          &add_plugin,
+                          ctx);
   return ctx;
 }
 
@@ -149,7 +158,8 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx)
   {
     plugin = ctx->plugins[i];
     GNUNET_break (NULL ==
-                  GNUNET_PLUGIN_unload (plugin->library_name, plugin->api));
+                  GNUNET_PLUGIN_unload (plugin->library_name,
+                                        plugin->api));
     GNUNET_free (plugin->library_name);
     GNUNET_free (plugin);
   }
@@ -159,6 +169,85 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx)
 
 
 /**
+ * Serialize state of a block group.
+ *
+ * @param bg group to serialize
+ * @param[out] nonce set to the nonce of the @a bg
+ * @param[out] raw_data set to the serialized state
+ * @param[out] raw_data_size set to the number of bytes in @a raw_data
+ * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
+ *         supported, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg,
+                              uint32_t *nonce,
+                              void **raw_data,
+                              size_t *raw_data_size)
+{
+  *nonce = 0;
+  *raw_data = NULL;
+  *raw_data_size = 0;
+  if (NULL == bg)
+    return GNUNET_NO;
+  if (NULL == bg->serialize_cb)
+    return GNUNET_NO;
+  return bg->serialize_cb (bg,
+                           nonce,
+                           raw_data,
+                           raw_data_size);
+}
+
+
+/**
+ * Destroy resources used by a block group.
+ *
+ * @param bg group to destroy, NULL is allowed
+ */
+void
+GNUNET_BLOCK_group_destroy (struct GNUNET_BLOCK_Group *bg)
+{
+  if (NULL == bg)
+    return;
+  bg->destroy_cb (bg);
+}
+
+
+/**
+ * Try merging two block groups.  Afterwards, @a bg1 should remain
+ * valid and contain the rules from both @a bg1 and @bg2, and
+ * @a bg2 should be destroyed (as part of this call).  The latter
+ * should happen even if merging is not supported.
+ *
+ * @param[in,out] bg1 first group to merge, is updated
+ * @param bg2 second group to merge, is destroyed
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_NO if merge failed due to different nonce
+ *         #GNUNET_SYSERR if merging is not supported
+ */
+int
+GNUNET_BLOCK_group_merge (struct GNUNET_BLOCK_Group *bg1,
+                          struct GNUNET_BLOCK_Group *bg2)
+{
+  int ret;
+
+  if (NULL == bg2)
+    return GNUNET_OK;
+  if (NULL == bg1)
+  {
+    bg2->destroy_cb (bg2);
+    return GNUNET_OK;
+  }
+  if (NULL == bg1->merge_cb)
+    return GNUNET_SYSERR;
+  GNUNET_assert (bg1->merge_cb == bg1->merge_cb);
+  ret = bg1->merge_cb (bg1,
+                       bg2);
+  bg2->destroy_cb (bg2);
+  return ret;
+}
+
+
+/**
  * Find a plugin for the given type.
  *
  * @param ctx context to search
@@ -170,10 +259,9 @@ find_plugin (struct GNUNET_BLOCK_Context *ctx,
              enum GNUNET_BLOCK_Type type)
 {
   struct Plugin *plugin;
-  unsigned int i;
   unsigned int j;
 
-  for (i = 0; i < ctx->num_plugins; i++)
+  for (unsigned i = 0; i < ctx->num_plugins; i++)
   {
     plugin = ctx->plugins[i];
     j = 0;
@@ -189,6 +277,46 @@ find_plugin (struct GNUNET_BLOCK_Context *ctx,
 
 
 /**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+struct GNUNET_BLOCK_Group *
+GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
+                           enum GNUNET_BLOCK_Type type,
+                           uint32_t nonce,
+                           const void *raw_data,
+                           size_t raw_data_size,
+                           ...)
+{
+  struct GNUNET_BLOCK_PluginFunctions *plugin;
+  struct GNUNET_BLOCK_Group *bg;
+  va_list ap;
+
+  plugin = find_plugin (ctx,
+                        type);
+  if (NULL == plugin->create_group)
+    return NULL;
+  va_start (ap,
+            raw_data_size);
+  bg = plugin->create_group (plugin->cls,
+                             type,
+                             nonce,
+                             raw_data,
+                             raw_data_size,
+                             ap);
+  va_end (ap);
+  return bg;
+}
+
+
+/**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the reply_block.
  * Note that it is assumed that the reply has already been
@@ -197,10 +325,9 @@ find_plugin (struct GNUNET_BLOCK_Context *ctx,
  *
  * @param ctx block contxt
  * @param type block type
+ * @param block block group to use
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -210,25 +337,25 @@ find_plugin (struct GNUNET_BLOCK_Context *ctx,
 enum GNUNET_BLOCK_EvaluationResult
 GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx,
                        enum GNUNET_BLOCK_Type type,
+                       struct GNUNET_BLOCK_Group *group,
                        enum GNUNET_BLOCK_EvaluationOptions eo,
                        const struct GNUNET_HashCode *query,
-                       struct GNUNET_CONTAINER_BloomFilter **bf,
-                       int32_t bf_mutator,
                        const void *xquery,
                        size_t xquery_size,
                        const void *reply_block,
                        size_t reply_block_size)
 {
-  struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx, type);
+  struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx,
+                                                             type);
 
-  if (plugin == NULL)
+  if (NULL == plugin)
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
   return plugin->evaluate (plugin->cls,
+                           ctx,
                            type,
+                           group,
                            eo,
                            query,
-                           bf,
-                           bf_mutator,
                            xquery,
                            xquery_size,
                            reply_block,
@@ -254,74 +381,43 @@ GNUNET_BLOCK_get_key (struct GNUNET_BLOCK_Context *ctx,
                       size_t block_size,
                       struct GNUNET_HashCode *key)
 {
-  struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx, type);
+  struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx,
+                                                             type);
 
   if (plugin == NULL)
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
-  return plugin->get_key (plugin->cls, type, block, block_size, key);
+  return plugin->get_key (plugin->cls,
+                          type,
+                          block,
+                          block_size,
+                          key);
 }
 
 
 /**
- * How many bytes should a bloomfilter be if we have already seen
- * entry_count responses?  Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
- * gives us the number of bits set per entry.  Furthermore, we should
- * not re-size the filter too often (to keep it cheap).
- *
- * Since other peers will also add entries but not resize the filter,
- * we should generally pick a slightly larger size than what the
- * strict math would suggest.
- *
- * @param entry_count expected number of entries in the Bloom filter
- * @return must be a power of two and smaller or equal to 2^15.
- */
-static size_t
-compute_bloomfilter_size (unsigned int entry_count)
-{
-  size_t size;
-  unsigned int ideal = (entry_count * GNUNET_CONSTANTS_BLOOMFILTER_K) / 4;
-  uint16_t max = 1 << 15;
-
-  if (entry_count > max)
-    return max;
-  size = 8;
-  while ((size < max) && (size < ideal))
-    size *= 2;
-  if (size > max)
-    return max;
-  return size;
-}
-
-
-/**
- * Construct a bloom filter that would filter out the given
- * results.
+ * Update block group to filter out the given results.  Note that the
+ * use of a hash for seen results implies that the caller magically
+ * knows how the specific block engine hashes for filtering
+ * duplicates, so this API may not always apply.
  *
  * @param bf_mutator mutation value to use
  * @param seen_results results already seen
  * @param seen_results_count number of entries in @a seen_results
- * @return NULL if seen_results_count is 0, otherwise a BF
- *         that would match the given results.
+ * @return #GNUNET_SYSERR if not supported, #GNUNET_OK on success
  */
-struct GNUNET_CONTAINER_BloomFilter *
-GNUNET_BLOCK_construct_bloomfilter (int32_t bf_mutator,
-                                    const struct GNUNET_HashCode *seen_results,
-                                    unsigned int seen_results_count)
+int
+GNUNET_BLOCK_group_set_seen (struct GNUNET_BLOCK_Group *bg,
+                             const struct GNUNET_HashCode *seen_results,
+                             unsigned int seen_results_count)
 {
-  struct GNUNET_CONTAINER_BloomFilter *bf;
-  struct GNUNET_HashCode mhash;
-  unsigned int i;
-  size_t nsize;
-
-  nsize = compute_bloomfilter_size (seen_results_count);
-  bf = GNUNET_CONTAINER_bloomfilter_init (NULL, nsize,
-                                          GNUNET_CONSTANTS_BLOOMFILTER_K);
-  for (i = 0; i < seen_results_count; i++)
-  {
-    GNUNET_BLOCK_mingle_hash (&seen_results[i], bf_mutator, &mhash);
-    GNUNET_CONTAINER_bloomfilter_add (bf, &mhash);
-  }
-  return bf;
+  if (NULL == bg)
+    return GNUNET_OK;
+  if (NULL == bg->mark_seen_cb)
+    return GNUNET_SYSERR;
+  bg->mark_seen_cb (bg,
+                    seen_results,
+                    seen_results_count);
+  return GNUNET_OK;
 }
 
 
diff --git a/src/block/plugin_block_template.c b/src/block/plugin_block_template.c
index 6cb69ef5f..0105fac38 100644
--- a/src/block/plugin_block_template.c
+++ b/src/block/plugin_block_template.c
@@ -26,20 +26,80 @@
 
 #include "platform.h"
 #include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
 
 #define DEBUG_TEMPLATE GNUNET_EXTRA_LOGGING
 
+/**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * How big is the BF we use for DHT blocks?
+ */
+#define TEMPLATE_BF_SIZE 8
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_template_create_group (void *cls,
+                                    enum GNUNET_BLOCK_Type type,
+                                    uint32_t nonce,
+                                    const void *raw_data,
+                                    size_t raw_data_size,
+                                    va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = TEMPLATE_BF_SIZE;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
 
 /**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the reply_block.
  *
  * @param cls closure
+ * @param ctx context
  * @param type block type
+ * @param group block group to use
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in xquery
  * @param reply_block response to validate
@@ -48,37 +108,27 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_template_evaluate (void *cls,
+                                struct GNUNET_BLOCK_Context *ctx,
                                 enum GNUNET_BLOCK_Type type,
+                                struct GNUNET_BLOCK_Group *group,
                                 enum GNUNET_BLOCK_EvaluationOptions eo,
                                 const struct GNUNET_HashCode *query,
-                                struct GNUNET_CONTAINER_BloomFilter **bf,
-                                int32_t bf_mutator,
                                 const void *xquery,
                                 size_t xquery_size,
                                 const void *reply_block,
                                 size_t reply_block_size)
 {
   struct GNUNET_HashCode chash;
-  struct GNUNET_HashCode mhash;
-  /* FIXME: check validity first... */
 
-  /* mandatory duplicate-detection code... */
-  if (NULL != bf)
-  {
-    GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash);
-    GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash);
-    if (NULL != *bf)
-    {
-      if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-        return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-    }
-    else
-    {
-      *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, 64 /* BLOOMFILTER_K */);
-    }
-    GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-  }
-  /* FIXME: other stuff here... */
+  if (NULL == reply_block)
+    return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
+  GNUNET_CRYPTO_hash (reply_block,
+                      reply_block_size,
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (group,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
 }
 
@@ -91,13 +141,15 @@ block_plugin_template_evaluate (void *cls,
  * @param block block to get the key for
  * @param block_size number of bytes in block
  * @param key set to the key (query) for the given block
- * @return GNUNET_OK on success, GNUNET_SYSERR if type not supported
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
  *         (or if extracting a key from a block of this type does not work)
  */
 static int
-block_plugin_template_get_key (void *cls, enum GNUNET_BLOCK_Type type,
-                               const void *block, size_t block_size,
-                               struct GNUNET_HashCode * key)
+block_plugin_template_get_key (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               const void *block,
+                               size_t block_size,
+                               struct GNUNET_HashCode *key)
 {
   return GNUNET_SYSERR;
 }
@@ -105,6 +157,8 @@ block_plugin_template_get_key (void *cls, enum GNUNET_BLOCK_Type type,
 
 /**
  * Entry point for the plugin.
+ *
+ * @param cls a `const struct GNUNET_CONFIGURATION_Handle`
  */
 void *
 libgnunet_plugin_block_template_init (void *cls)
@@ -119,6 +173,7 @@ libgnunet_plugin_block_template_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_template_evaluate;
   api->get_key = &block_plugin_template_get_key;
+  api->create_group = &block_plugin_template_create_group;
   api->types = types;
   return api;
 }
@@ -130,7 +185,7 @@ libgnunet_plugin_block_template_init (void *cls)
 void *
 libgnunet_plugin_block_template_done (void *cls)
 {
-  struct GNUNET_TRANSPORT_PluginFunctions *api = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
 
   GNUNET_free (api);
   return NULL;
diff --git a/src/block/plugin_block_test.c b/src/block/plugin_block_test.c
index b692d6230..e359acd7f 100644
--- a/src/block/plugin_block_test.c
+++ b/src/block/plugin_block_test.c
@@ -27,7 +27,7 @@
 
 #include "platform.h"
 #include "gnunet_block_plugin.h"
-
+#include "gnunet_block_group_lib.h"
 
 /**
  * Number of bits we set per entry in the bloomfilter.
@@ -36,15 +36,68 @@
 #define BLOOMFILTER_K 16
 
 /**
+ * How big is the BF we use for DHT blocks?
+ */
+#define TEST_BF_SIZE 8
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_test_create_group (void *cls,
+                                enum GNUNET_BLOCK_Type type,
+                                uint32_t nonce,
+                                const void *raw_data,
+                                size_t raw_data_size,
+                                va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = TEST_BF_SIZE;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
+
+/**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the reply_block.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param group group to check against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -53,21 +106,23 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_test_evaluate (void *cls,
+                            struct GNUNET_BLOCK_Context *ctx,
                             enum GNUNET_BLOCK_Type type,
+                            struct GNUNET_BLOCK_Group *group,
                             enum GNUNET_BLOCK_EvaluationOptions eo,
                             const struct GNUNET_HashCode *query,
-                            struct GNUNET_CONTAINER_BloomFilter **bf,
-                            int32_t bf_mutator,
                             const void *xquery,
                             size_t xquery_size,
                             const void *reply_block,
                             size_t reply_block_size)
 {
   struct GNUNET_HashCode chash;
-  struct GNUNET_HashCode mhash;
 
   if ( GNUNET_BLOCK_TYPE_TEST != type)
+  {
+    GNUNET_break (0);
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
+  }
   if (0 != xquery_size)
   {
     GNUNET_break_op (0);
@@ -75,22 +130,13 @@ block_plugin_test_evaluate (void *cls,
   }
   if (NULL == reply_block)
     return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
-
-  if (NULL != bf)
-  {
-    GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash);
-    GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash);
-    if (NULL != *bf)
-    {
-      if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-        return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-    }
-    else
-    {
-      *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K);
-    }
-    GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-  }
+  GNUNET_CRYPTO_hash (reply_block,
+                      reply_block_size,
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (group,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_OK_MORE;
 }
 
@@ -107,9 +153,11 @@ block_plugin_test_evaluate (void *cls,
  *         (or if extracting a key from a block of this type does not work)
  */
 static int
-block_plugin_test_get_key (void *cls, enum GNUNET_BLOCK_Type type,
-                           const void *block, size_t block_size,
-                           struct GNUNET_HashCode * key)
+block_plugin_test_get_key (void *cls,
+                           enum GNUNET_BLOCK_Type type,
+                           const void *block,
+                           size_t block_size,
+                           struct GNUNET_HashCode *key)
 {
   /* always fails since there is no fixed relationship between
    * keys and values for test values */
@@ -136,6 +184,7 @@ libgnunet_plugin_block_test_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_test_evaluate;
   api->get_key = &block_plugin_test_get_key;
+  api->create_group = &block_plugin_test_create_group;
   api->types = types;
   return api;
 }
diff --git a/src/cadet/.gitignore b/src/cadet/.gitignore
index 096ee06eb..44382fde9 100644
--- a/src/cadet/.gitignore
+++ b/src/cadet/.gitignore
@@ -19,3 +19,6 @@ test_cadet_5_speed_reliable
 test_cadet_5_speed_reliable_backwards
 test_cadet_local
 test_cadet_single
+gnunet-service-cadet-new
+test_cadet_local_mq
+test_cadet_*_new
\ No newline at end of file
diff --git a/src/cadet/Makefile.am b/src/cadet/Makefile.am
index 53d17dd9c..ce30ebe46 100644
--- a/src/cadet/Makefile.am
+++ b/src/cadet/Makefile.am
@@ -23,24 +23,24 @@ AM_CLFAGS = -g
 
 libexec_PROGRAMS = \
  gnunet-service-cadet \
- gnunet-service-cadet-new \
  $(EXP_LIBEXEC)
 
 bin_PROGRAMS = \
  gnunet-cadet
 
 lib_LTLIBRARIES = \
-  libgnunetcadet.la $(EXP_LIB)
+  libgnunetcadet.la \
+  $(EXP_LIB)
 
 libgnunetcadet_la_SOURCES = \
-  cadet_api.c cadet_common.c
+  cadet_api.c
 libgnunetcadet_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(XLIB) \
   $(LTLIBINTL)
 libgnunetcadet_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS) $(WINFLAGS) \
-  -version-info 5:0:0
+  -version-info 7:0:0
 
 gnunet_cadet_SOURCES = \
   gnunet-cadet.c
@@ -48,46 +48,23 @@ gnunet_cadet_LDADD = \
   libgnunetcadet.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-gnunet_service_cadet_new_SOURCES = \
- gnunet-service-cadet-new.c gnunet-service-cadet-new.h \
- gnunet-service-cadet-new_channel.c gnunet-service-cadet-new_channel.h \
- gnunet-service-cadet-new_connection.c gnunet-service-cadet-new_connection.h \
- gnunet-service-cadet-new_core.c gnunet-service-cadet-new_core.h \
- gnunet-service-cadet-new_dht.c gnunet-service-cadet-new_dht.h \
- gnunet-service-cadet-new_hello.c gnunet-service-cadet-new_hello.h \
- gnunet-service-cadet-new_tunnels.c gnunet-service-cadet-new_tunnels.h \
- gnunet-service-cadet-new_paths.c gnunet-service-cadet-new_paths.h \
- gnunet-service-cadet-new_peer.c gnunet-service-cadet-new_peer.h
-gnunet_service_cadet_new_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/ats/libgnunetats.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
-  $(top_builddir)/src/dht/libgnunetdht.la \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
-  $(top_builddir)/src/hello/libgnunethello.la \
-  $(top_builddir)/src/block/libgnunetblock.la
-
 gnunet_service_cadet_SOURCES = \
- gnunet-service-cadet_tunnel.c gnunet-service-cadet_tunnel.h \
- gnunet-service-cadet_connection.c gnunet-service-cadet_connection.h \
+ gnunet-service-cadet.c gnunet-service-cadet.h \
  gnunet-service-cadet_channel.c gnunet-service-cadet_channel.h \
- gnunet-service-cadet_local.c gnunet-service-cadet_local.h \
- gnunet-service-cadet_peer.c gnunet-service-cadet_peer.h \
+ gnunet-service-cadet_connection.c gnunet-service-cadet_connection.h \
+ gnunet-service-cadet_core.c gnunet-service-cadet_core.h \
  gnunet-service-cadet_dht.c gnunet-service-cadet_dht.h \
  gnunet-service-cadet_hello.c gnunet-service-cadet_hello.h \
- cadet_path.c cadet_path.h \
- cadet_common.c \
- gnunet-service-cadet.c
-gnunet_service_cadet_CFLAGS = $(AM_CFLAGS)
+ gnunet-service-cadet_tunnels.c gnunet-service-cadet_tunnels.h \
+ gnunet-service-cadet_paths.c gnunet-service-cadet_paths.h \
+ gnunet-service-cadet_peer.c gnunet-service-cadet_peer.h
 gnunet_service_cadet_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
   $(top_builddir)/src/ats/libgnunetats.la \
+  $(top_builddir)/src/core/libgnunetcore.la \
   $(top_builddir)/src/dht/libgnunetdht.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/transport/libgnunettransport.la \
   $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
   $(top_builddir)/src/hello/libgnunethello.la \
   $(top_builddir)/src/block/libgnunetblock.la
@@ -97,26 +74,14 @@ endif
 
 
 if HAVE_TESTING
- noinst_LIBRARIES = libgnunetcadettest.a $(noinst_LIB_EXP)
- noinst_PROGRAMS = gnunet-cadet-profiler
+ noinst_LTLIBRARIES = libgnunetcadettest.la $(noinst_LIB_EXP)
+# noinst_PROGRAMS = gnunet-cadet-profiler
 endif
 
-libgnunetcadettest_a_SOURCES = \
-  cadet_test_lib.c cadet_test_lib.h
-libgnunetcadettest_a_LIBADD = \
- $(top_builddir)/src/util/libgnunetutil.la \
- $(top_builddir)/src/testbed/libgnunettestbed.la \
- libgnunetcadet.la
-
 if HAVE_TESTING
 check_PROGRAMS = \
-  test_cadet_2_speed_reliable_backwards \
-  test_cadet_5_speed \
-  test_cadet_5_speed_ack \
-  test_cadet_5_speed_reliable \
-  test_cadet_5_speed_reliable_backwards \
-  test_cadet_single \
-  test_cadet_local \
+  test_cadet_local_mq \
+  test_cadet_2_forward \
   test_cadet_2_forward \
   test_cadet_2_signal \
   test_cadet_2_keepalive \
@@ -124,40 +89,50 @@ check_PROGRAMS = \
   test_cadet_2_speed_ack \
   test_cadet_2_speed_backwards \
   test_cadet_2_speed_reliable \
+  test_cadet_2_speed_reliable_backwards \
   test_cadet_5_forward \
   test_cadet_5_signal \
   test_cadet_5_keepalive \
+  test_cadet_5_speed \
+  test_cadet_5_speed_ack \
+  test_cadet_5_speed_reliable \
+  test_cadet_5_speed_reliable_backwards \
   test_cadet_5_speed_backwards
 endif
 
+
+#gnunet_cadet_profiler_SOURCES = \
+#  gnunet-cadet-profiler.c
+#gnunet_cadet_profiler_LDADD = $(ld_cadet_test_lib)
+
+
+test_cadet_local_mq_SOURCES = \
+  test_cadet_local_mq.c
+test_cadet_local_mq_LDADD = \
+  libgnunetcadet.la \
+  $(top_builddir)/src/testing/libgnunettesting.la \
+  $(top_builddir)/src/util/libgnunetutil.la
+
+
+libgnunetcadettest_la_SOURCES = \
+  cadet_test_lib.c cadet_test_lib.h
+libgnunetcadettest_la_LIBADD = \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ $(top_builddir)/src/testbed/libgnunettestbed.la \
+ libgnunetcadet.la
+
 ld_cadet_test_lib = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/testing/libgnunettesting.la \
-  libgnunetcadettest.a \
   libgnunetcadet.la \
+  libgnunetcadettest.la \
   $(top_builddir)/src/testbed/libgnunettestbed.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la
-
 dep_cadet_test_lib = \
   libgnunetcadet.la \
-  libgnunetcadettest.a \
+  libgnunetcadettest.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la
 
-
-gnunet_cadet_profiler_SOURCES = \
-  gnunet-cadet-profiler.c
-gnunet_cadet_profiler_LDADD = $(ld_cadet_test_lib)
-
-
-test_cadet_single_SOURCES = \
-  test_cadet_single.c
-test_cadet_single_LDADD = $(ld_cadet_test_lib)
-
-test_cadet_local_SOURCES = \
-  test_cadet_local.c
-test_cadet_local_LDADD = $(ld_cadet_test_lib)
-
-
 test_cadet_2_forward_SOURCES = \
   test_cadet.c
 test_cadet_2_forward_LDADD = $(ld_cadet_test_lib)
@@ -190,7 +165,6 @@ test_cadet_2_speed_reliable_backwards_SOURCES = \
   test_cadet.c
 test_cadet_2_speed_reliable_backwards_LDADD = $(ld_cadet_test_lib)
 
-
 test_cadet_5_forward_SOURCES = \
   test_cadet.c
 test_cadet_5_forward_LDADD = $(ld_cadet_test_lib)
@@ -225,7 +199,7 @@ test_cadet_5_speed_reliable_backwards_LDADD = $(ld_cadet_test_lib)
 
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
  $(check_PROGRAMS)
 endif
diff --git a/src/cadet/TODO b/src/cadet/TODO
new file mode 100644
index 000000000..06567b0ad
--- /dev/null
+++ b/src/cadet/TODO
@@ -0,0 +1,36 @@
+- URGENT:
+  + if 'client-not-ready', we do not ACK at all, and sender keeps
+    retransmitting again and again; would be good to do flow-control notification instead
+    of not ACKing that we got the data but are simply not ready for more!
+  + Congestion/flow control (CHANNEL):
+    estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+    (and figure out how/where to use this!)
+
+- HIGH: revisit handling of 'unbuffered' traffic! (CHANNEL/TUNNEL)
+        (need to push down through tunnel into connection selection);
+        At Tunnel-level, try to create connections that match channel
+        preferences (buffered/unbuffered) and select connections for
+        channel traffic that match channel preferences.
+        BUT: not sure this is ideal, discloses traffic type to
+        routers. We don't want that! (Maybe revise decision to do this?)
+
+- HIGH: revisit handling of 'buffered' traffic: 4 is a rather small buffer; (CHANNEL)
+        maybe reserve more bits in 'options' to allow for buffer size control?
+        Or: maybe even better, calculated required buffer size based on latency
+        and throughput (and available memory)
+
+- HIGH: if we receive BROKEN messages, cut down corresponding PATH (up to the
+        point of breakage) as well as connection/route (CORE)
+
+- OPTIMIZATION: proper connection evaluation during connection management:
+  + TUNNELS:
+    * consider quality of current connection set when deciding
+      how often to do maintenance
+    * interact with PEER to drive DHT GET/PUT operations based
+      on how much we like our connections
+
+
+- OPTIMIZATION: optimize stopping/restarting DHT search to situations
+  where we actually need it (i.e. not if we have a direct connection,
+  or if we already have plenty of good short ones, or maybe even
+  to take a break if we have some connections and have searched a lot (?)) (PEER)
diff --git a/src/cadet/cadet.conf.in b/src/cadet/cadet.conf.in
index 48fd03329..d50e168f0 100644
--- a/src/cadet/cadet.conf.in
+++ b/src/cadet/cadet.conf.in
@@ -11,13 +11,43 @@ UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-cadet.sock
 UNIX_MATCH_UID = YES
 UNIX_MATCH_GID = YES
 
+
+# How often do we send KEEPALIVE messages on connections to keep them
+# from timing out?
 REFRESH_CONNECTION_TIME = 5 min
+
+# Percentage of packets CADET is artificially dropping. Used for testing only!
+# DROP_PERCENT =
+
+# How frequently do we usually anounce our presence in the DHT?
 ID_ANNOUNCE_TIME = 1 h
+
+# FIXME: document
 CONNECT_TIMEOUT = 30 s
+
+# What is the replication level we give to the DHT when announcing our
+# existence?  Usually there is no need to change this.
 DHT_REPLICATION_LEVEL = 3
-MAX_TUNNELS = 1000
+
+# FIXME: not implemented
+# MAX_TUNNELS = 1000
+
+# FIXME: not implemented, replaced by MAX_ROUTES in NEW CADET!
 MAX_CONNECTIONS = 1000
+
+# How many routes do we participate in at most?  Should be smaller
+# than MAX_MSGS_QUEUE
+MAX_ROUTES = 5000
+
+# FIXME: not implemented
 MAX_MSGS_QUEUE = 10000
+
+# FIXME: not implemented
 MAX_PEERS = 1000
+
+# How often do we advance the ratchet even if there is not
+# any traffic?
 RATCHET_TIME = 1 h
+
+# How often do we advance the ratched if there is traffic?
 RATCHET_MESSAGES = 64
diff --git a/src/cadet/cadet.h b/src/cadet/cadet.h
index 451d1f354..99f9f2653 100644
--- a/src/cadet/cadet.h
+++ b/src/cadet/cadet.h
@@ -59,7 +59,7 @@ extern "C"
 #include "gnunet_core_service.h"
 #include "gnunet_cadet_service.h"
 #include "gnunet_protocols.h"
-#include <gnunet_cadet_service.h>
+#include "gnunet_cadet_service.h"
 
 /******************************************************************************/
 /**************************       CONSTANTS      ******************************/
diff --git a/src/cadet/cadet_api.c b/src/cadet/cadet_api.c
index 72b7b692d..decf473a9 100644
--- a/src/cadet/cadet_api.c
+++ b/src/cadet/cadet_api.c
@@ -21,8 +21,8 @@
  * @file cadet/cadet_api.c
  * @brief cadet api: client implementation of cadet service
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
-
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_constants.h"
@@ -32,55 +32,9 @@
 
 #define LOG(kind,...) GNUNET_log_from (kind, "cadet-api",__VA_ARGS__)
 
-/******************************************************************************/
-/************************      DATA STRUCTURES     ****************************/
-/******************************************************************************/
-
 /**
- * Transmission queue to the service
+ * Ugly legacy hack.
  */
-struct GNUNET_CADET_TransmitHandle
-{
-  /**
-   * Double Linked list
-   */
-  struct GNUNET_CADET_TransmitHandle *next;
-
-  /**
-   * Double Linked list
-   */
-  struct GNUNET_CADET_TransmitHandle *prev;
-
-  /**
-   * Channel this message is sent on / for (may be NULL for control messages).
-   */
-  struct GNUNET_CADET_Channel *channel;
-
-  /**
-   * Request data task.
-   */
-  struct GNUNET_SCHEDULER_Task *request_data_task;
-
-  /**
-   * Callback to obtain the message to transmit, or NULL if we
-   * got the message in 'data'.  Notice that messages built
-   * by 'notify' need to be encapsulated with information about
-   * the 'target'.
-   */
-  GNUNET_CONNECTION_TransmitReadyNotify notify;
-
-  /**
-   * Closure for 'notify'
-   */
-  void *notify_cls;
-
-  /**
-   * Size of the payload.
-   */
-  size_t size;
-};
-
-
 union CadetInfoCB
 {
 
@@ -117,54 +71,19 @@ union CadetInfoCB
 struct GNUNET_CADET_Handle
 {
   /**
-   * Message queue (if available).
+   * Message queue.
    */
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * Set of handlers used for processing incoming messages in the channels
-   */
-  const struct GNUNET_CADET_MessageHandler *message_handlers;
-
-  /**
-   * Number of handlers in the handlers array.
-   */
-  unsigned int n_handlers;
-
-  /**
    * Ports open.
    */
   struct GNUNET_CONTAINER_MultiHashMap *ports;
 
   /**
-   * Double linked list of the channels this client is connected to, head.
-   */
-  struct GNUNET_CADET_Channel *channels_head;
-
-  /**
-   * Double linked list of the channels this client is connected to, tail.
+   * Channels open.
    */
-  struct GNUNET_CADET_Channel *channels_tail;
-
-  /**
-   * Callback for inbound channel disconnection
-   */
-  GNUNET_CADET_ChannelEndHandler *cleaner;
-
-  /**
-   * Closure for all the handlers given by the client
-   */
-  void *cls;
-
-  /**
-   * Messages to send to the service, head.
-   */
-  struct GNUNET_CADET_TransmitHandle *th_head;
-
-  /**
-   * Messages to send to the service, tail.
-   */
-  struct GNUNET_CADET_TransmitHandle *th_tail;
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
 
   /**
    * child of the next channel to create (to avoid reusing IDs often)
@@ -177,14 +96,9 @@ struct GNUNET_CADET_Handle
   const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
-   * Time to the next reconnect in case one reconnect fails
-   */
-  struct GNUNET_TIME_Relative reconnect_time;
-
-  /**
    * Task for trying to reconnect.
    */
-  struct GNUNET_SCHEDULER_Task * reconnect_task;
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
 
   /**
    * Callback for an info task (only one active at a time).
@@ -195,65 +109,73 @@ struct GNUNET_CADET_Handle
    * Info callback closure for @c info_cb.
    */
   void *info_cls;
+
+  /**
+   * Time to the next reconnect in case one reconnect fails
+   */
+  struct GNUNET_TIME_Relative reconnect_time;
+
 };
 
 
 /**
- * Description of a peer
+ * Opaque handle to a channel.
  */
-struct GNUNET_CADET_Peer
+struct GNUNET_CADET_Channel
 {
+
   /**
-   * ID of the peer in short form
+   * Other end of the channel.
    */
-  GNUNET_PEER_Id id;
+  struct GNUNET_PeerIdentity peer;
 
   /**
-   * Channel this peer belongs to
+   * Handle to the cadet this channel belongs to
    */
-  struct GNUNET_CADET_Channel *t;
-};
+  struct GNUNET_CADET_Handle *cadet;
 
+  /**
+   * Channel's port, if incoming.
+   */
+  struct GNUNET_CADET_Port *incoming_port;
 
-/**
- * Opaque handle to a channel.
- */
-struct GNUNET_CADET_Channel
-{
   /**
-   * DLL next
+   * Any data the caller wants to put in here, used for the
+   * various callbacks (@e disconnects, @e window_changes, handlers).
    */
-  struct GNUNET_CADET_Channel *next;
+  void *ctx;
 
   /**
-   * DLL prev
+   * Message Queue for the channel (which we are implementing).
    */
-  struct GNUNET_CADET_Channel *prev;
+  struct GNUNET_MQ_Handle *mq;
 
   /**
-   * Handle to the cadet this channel belongs to
+   * Task to allow mq to send more traffic.
    */
-  struct GNUNET_CADET_Handle *cadet;
+  struct GNUNET_SCHEDULER_Task *mq_cont;
 
   /**
-   * Local ID of the channel
+   * Pending envelope with a message to be transmitted to the
+   * service as soon as we are allowed to.  Should only be
+   * non-NULL if @e allow_send is 0.
    */
-  struct GNUNET_CADET_ClientChannelNumber ccn;
+  struct GNUNET_MQ_Envelope *pending_env;
 
   /**
-   * Channel's port, if any.
+   * Window change handler.
    */
-  struct GNUNET_CADET_Port *port;
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
 
   /**
-   * Other end of the channel.
+   * Disconnect handler.
    */
-  GNUNET_PEER_Id peer;
+  GNUNET_CADET_DisconnectEventHandler disconnects;
 
   /**
-   * Any data the caller wants to put in here
+   * Local ID of the channel, #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI bit is set if outbound.
    */
-  void *ctx;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
   /**
    * Channel options: reliability, etc.
@@ -261,7 +183,7 @@ struct GNUNET_CADET_Channel
   enum GNUNET_CADET_ChannelOption options;
 
   /**
-   * Are we allowed to send to the service?
+   * How many messages are we allowed to send to the service right now?
    */
   unsigned int allow_send;
 
@@ -273,82 +195,62 @@ struct GNUNET_CADET_Channel
  */
 struct GNUNET_CADET_Port
 {
+
+  /**
+   * Port "number"
+   */
+  struct GNUNET_HashCode id;
+
   /**
    * Handle to the CADET session this port belongs to.
    */
   struct GNUNET_CADET_Handle *cadet;
 
   /**
-   * Port ID.
+   * Closure for @a handler.
    */
-  struct GNUNET_HashCode *hash;
+  void *cls;
 
   /**
-   * Callback handler for incoming channels on this port.
+   * Handler for incoming channels on this port
    */
-  GNUNET_CADET_InboundChannelNotificationHandler *handler;
+  GNUNET_CADET_ConnectEventHandler connects;
 
   /**
-   * Closure for @a handler.
+   * Closure for @ref connects
    */
-  void *cls;
-};
+  void *connects_cls;
 
+  /**
+   * Window size change handler.
+   */
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
 
-/**
- * Implementation state for cadet's message queue.
- */
-struct CadetMQState
-{
   /**
-   * The current transmit handle, or NULL
-   * if no transmit is active.
+   * Handler called when an incoming channel is destroyed.
    */
-  struct GNUNET_CADET_TransmitHandle *th;
+  GNUNET_CADET_DisconnectEventHandler disconnects;
 
   /**
-   * Channel to send the data over.
+   * Payload handlers for incoming channels.
    */
-  struct GNUNET_CADET_Channel *channel;
+  struct GNUNET_MQ_MessageHandler *handlers;
 };
 
 
-/******************************************************************************/
-/***********************     AUXILIARY FUNCTIONS      *************************/
-/******************************************************************************/
-
-/**
- * Check if transmission is a payload packet.
- *
- * @param th Transmission handle.
- *
- * @return #GNUNET_YES if it is a payload packet,
- *         #GNUNET_NO if it is a cadet management packet.
- */
-static int
-th_is_payload (struct GNUNET_CADET_TransmitHandle *th)
-{
-  return (th->notify != NULL) ? GNUNET_YES : GNUNET_NO;
-}
-
-
 /**
  * Find the Port struct for a hash.
  *
  * @param h CADET handle.
  * @param hash HashCode for the port number.
- *
  * @return The port handle if known, NULL otherwise.
  */
 static struct GNUNET_CADET_Port *
 find_port (const struct GNUNET_CADET_Handle *h,
            const struct GNUNET_HashCode *hash)
 {
-  struct GNUNET_CADET_Port *p;
-
-  p = GNUNET_CONTAINER_multihashmap_get (h->ports, hash);
-
-  return p;
+  return GNUNET_CONTAINER_multihashmap_get (h->ports,
+                                            hash);
 }
 
 
@@ -360,15 +262,11 @@ find_port (const struct GNUNET_CADET_Handle *h,
  * @return handle to the required channel or NULL if not found
  */
 static struct GNUNET_CADET_Channel *
-retrieve_channel (struct GNUNET_CADET_Handle *h,
-                  struct GNUNET_CADET_ClientChannelNumber ccn)
+find_channel (struct GNUNET_CADET_Handle *h,
+              struct GNUNET_CADET_ClientChannelNumber ccn)
 {
-  struct GNUNET_CADET_Channel *ch;
-
-  for (ch = h->channels_head; NULL != ch; ch = ch->next)
-    if (ch->ccn.channel_of_client == ccn.channel_of_client)
-      return ch;
-  return NULL;
+  return GNUNET_CONTAINER_multihashmap32_get (h->channels,
+                                              ntohl (ccn.channel_of_client));
 }
 
 
@@ -376,38 +274,37 @@ retrieve_channel (struct GNUNET_CADET_Handle *h,
  * Create a new channel and insert it in the channel list of the cadet handle
  *
  * @param h Cadet handle
- * @param ccn Desired ccn of the channel, 0 to assign one automatically.
- *
+ * @param ccnp pointer to desired ccn of the channel, NULL to assign one automatically.
  * @return Handle to the created channel.
  */
 static struct GNUNET_CADET_Channel *
 create_channel (struct GNUNET_CADET_Handle *h,
-                struct GNUNET_CADET_ClientChannelNumber ccn)
+                const struct GNUNET_CADET_ClientChannelNumber *ccnp)
 {
   struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
   ch = GNUNET_new (struct GNUNET_CADET_Channel);
-  GNUNET_CONTAINER_DLL_insert (h->channels_head,
-                               h->channels_tail,
-                               ch);
   ch->cadet = h;
-  if (0 == ccn.channel_of_client)
+  if (NULL == ccnp)
   {
-    ch->ccn = h->next_ccn;
-    while (NULL != retrieve_channel (h,
-                                     h->next_ccn))
-    {
+    while (NULL !=
+           find_channel (h,
+                         h->next_ccn))
       h->next_ccn.channel_of_client
-        = htonl (1 + ntohl (h->next_ccn.channel_of_client));
-      if (0 == ntohl (h->next_ccn.channel_of_client))
-        h->next_ccn.channel_of_client
-          = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-    }
+        = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI | (1 + ntohl (h->next_ccn.channel_of_client)));
+    ccn = h->next_ccn;
   }
   else
   {
-    ch->ccn = ccn;
+    ccn = *ccnp;
   }
+  ch->ccn = ccn;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (h->channels,
+                                                      ntohl (ch->ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   return ch;
 }
 
@@ -421,145 +318,273 @@ create_channel (struct GNUNET_CADET_Handle *h,
  *
  * @param ch Pointer to the channel.
  * @param call_cleaner Whether to call the cleaner handler.
+ */
+static void
+destroy_channel (struct GNUNET_CADET_Channel *ch)
+{
+  struct GNUNET_CADET_Handle *h = ch->cadet;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying channel %X of %p\n",
+       ch->ccn,
+       h);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (h->channels,
+                                                         ntohl (ch->ccn.channel_of_client),
+                                                         ch));
+  if (NULL != ch->mq_cont)
+  {
+    GNUNET_SCHEDULER_cancel (ch->mq_cont);
+    ch->mq_cont = NULL;
+  }
+  /* signal channel destruction */
+  if (NULL != ch->disconnects)
+    ch->disconnects (ch->ctx,
+                     ch);
+  if (NULL != ch->pending_env)
+    GNUNET_MQ_discard (ch->pending_env);
+  GNUNET_MQ_destroy (ch->mq);
+  GNUNET_free (ch);
+}
+
+
+/**
+ * Reconnect to the service, retransmit all infomation to try to restore the
+ * original state.
  *
- * @return Handle to the required channel or NULL if not found.
+ * @param h handle to the cadet
  */
-// FIXME: simplify: call_cleaner is always #GNUNET_YES!!!
 static void
-destroy_channel (struct GNUNET_CADET_Channel *ch,
-                 int call_cleaner)
+reconnect (struct GNUNET_CADET_Handle *h);
+
+
+/**
+ * Reconnect callback: tries to reconnect again after a failer previous
+ * reconnecttion
+ *
+ * @param cls closure (cadet handle)
+ */
+static void
+reconnect_cbk (void *cls)
 {
-  struct GNUNET_CADET_Handle *h;
-  struct GNUNET_CADET_TransmitHandle *th;
-  struct GNUNET_CADET_TransmitHandle *next;
+  struct GNUNET_CADET_Handle *h = cls;
 
-  if (NULL == ch)
+  h->reconnect_task = NULL;
+  reconnect (h);
+}
+
+
+/**
+ * Function called during #reconnect() to destroy
+ * all channels that are still open.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param cid chanenl ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_channel_on_reconnect_cb (void *cls,
+                                 uint32_t cid,
+                                 void *value)
+{
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Channel *ch = value;
+
+  destroy_channel (ch);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Reconnect to the service, retransmit all infomation to try to restore the
+ * original state.
+ *
+ * @param h handle to the cadet
+ *
+ * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
+ */
+static void
+schedule_reconnect (struct GNUNET_CADET_Handle *h)
+{
+  if (NULL != h->reconnect_task)
+    return;
+  GNUNET_CONTAINER_multihashmap32_iterate (h->channels,
+                                           &destroy_channel_on_reconnect_cb,
+                                           h);
+  h->reconnect_task
+    = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
+                                    &reconnect_cbk,
+                                    h);
+  h->reconnect_time
+    = GNUNET_TIME_STD_BACKOFF (h->reconnect_time);
+}
+
+
+/**
+ * Notify the application about a change in the window size (if needed).
+ *
+ * @param ch Channel to notify about.
+ */
+static void
+notify_window_size (struct GNUNET_CADET_Channel *ch)
+{
+  if (NULL != ch->window_changes)
+    ch->window_changes (ch->ctx,
+                        ch, /* FIXME: remove 'ch'? */
+                        ch->allow_send);
+}
+
+
+/**
+ * Transmit the next message from our queue.
+ *
+ * @param cls Closure (channel whose mq to activate).
+ */
+static void
+cadet_mq_send_now (void *cls)
+{
+  struct GNUNET_CADET_Channel *ch = cls;
+  struct GNUNET_MQ_Envelope *env = ch->pending_env;
+
+  ch->mq_cont = NULL;
+  if (0 == ch->allow_send)
   {
+    /* how did we get here? */
     GNUNET_break (0);
     return;
   }
-  h = ch->cadet;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " destroy_channel %X of %p\n",
-       ch->ccn,
-       h);
+  if (NULL == env)
+  {
+    /* how did we get here? */
+    GNUNET_break (0);
+    return;
+  }
+  ch->allow_send--;
+  ch->pending_env = NULL;
+  GNUNET_MQ_send (ch->cadet->mq,
+                  env);
+  GNUNET_MQ_impl_send_continue (ch->mq);
+}
 
-  GNUNET_CONTAINER_DLL_remove (h->channels_head,
-                               h->channels_tail,
-                               ch);
 
-  /* signal channel destruction */
-  if ( (NULL != h->cleaner) &&
-       (0 != ch->peer) &&
-       (GNUNET_YES == call_cleaner) )
+/**
+ * Implement sending functionality of a message queue for
+ * us sending messages to a peer.
+ *
+ * Encapsulates the payload message in a #GNUNET_CADET_LocalData message
+ * in order to label the message with the channel ID and send the
+ * encapsulated message to the service.
+ *
+ * @param mq the message queue
+ * @param msg the message to send
+ * @param impl_state state of the implementation
+ */
+static void
+cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
+                    const struct GNUNET_MessageHeader *msg,
+                    void *impl_state)
+{
+  struct GNUNET_CADET_Channel *ch = impl_state;
+  struct GNUNET_CADET_Handle *h = ch->cadet;
+  uint16_t msize;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalData *cadet_msg;
+
+  if (NULL == h->mq)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         " calling cleaner\n");
-    h->cleaner (h->cls, ch, ch->ctx);
+    /* We're currently reconnecting, pretend this worked */
+    GNUNET_MQ_impl_send_continue (mq);
+    return;
   }
 
-  /* check that clients did not leave messages behind in the queue */
-  for (th = h->th_head; NULL != th; th = next)
+  /* check message size for sanity */
+  msize = ntohs (msg->size);
+  if (msize > GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE)
   {
-    next = th->next;
-    if (th->channel != ch)
-      continue;
-    /* Clients should have aborted their requests already.
-     * Management traffic should be ok, as clients can't cancel that.
-     * If the service crashed and we are reconnecting, it's ok.
-     */
-    GNUNET_break (GNUNET_NO == th_is_payload (th));
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
+    GNUNET_break (0);
+    GNUNET_MQ_impl_send_continue (mq);
+    return;
   }
-
-  if (0 != ch->peer)
-    GNUNET_PEER_change_rc (ch->peer, -1);
-  GNUNET_free (ch);
+  env = GNUNET_MQ_msg_nested_mh (cadet_msg,
+                                 GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                                 msg);
+  cadet_msg->ccn = ch->ccn;
+  GNUNET_assert (NULL == ch->pending_env);
+  ch->pending_env = env;
+  if (0 < ch->allow_send)
+    ch->mq_cont
+      = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
+                                  ch);
 }
 
 
 /**
- * Add a transmit handle to the transmission queue and set the
- * timeout if needed.
+ * Handle destruction of a message queue.  Implementations must not
+ * free @a mq, but should take care of @a impl_state.
  *
- * @param h cadet handle with the queue head and tail
- * @param th handle to the packet to be transmitted
+ * @param mq the message queue to destroy
+ * @param impl_state state of the implementation
  */
 static void
-add_to_queue (struct GNUNET_CADET_Handle *h,
-              struct GNUNET_CADET_TransmitHandle *th)
+cadet_mq_destroy_impl (struct GNUNET_MQ_Handle *mq,
+                       void *impl_state)
 {
-  GNUNET_CONTAINER_DLL_insert_tail (h->th_head,
-                                    h->th_tail,
-                                    th);
+  struct GNUNET_CADET_Channel *ch = impl_state;
+
+  GNUNET_assert (mq == ch->mq);
+  ch->mq = NULL;
 }
 
 
 /**
- * Remove a transmit handle from the transmission queue, if present.
- *
- * Safe to call even if not queued.
+ * We had an error processing a message we forwarded from a peer to
+ * the CADET service.  We should just complain about it but otherwise
+ * continue processing.
  *
- * @param th handle to the packet to be unqueued.
+ * @param cls closure with our `struct GNUNET_CADET_Channel`
+ * @param error error code
  */
 static void
-remove_from_queue (struct GNUNET_CADET_TransmitHandle *th)
+cadet_mq_error_handler (void *cls,
+                        enum GNUNET_MQ_Error error)
 {
-  struct GNUNET_CADET_Handle *h = th->channel->cadet;
+  struct GNUNET_CADET_Channel *ch = cls;
 
-  /* It might or might not have been queued (rarely not), but check anyway. */
-  if (NULL != th->next || h->th_tail == th)
+  GNUNET_break (0);
+  if (GNUNET_MQ_ERROR_NO_MATCH == error)
+  {
+    /* Got a message we did not understand, still try to continue! */
+    GNUNET_CADET_receive_done (ch);
+  }
+  else
   {
-    GNUNET_CONTAINER_DLL_remove (h->th_head, h->th_tail, th);
+    schedule_reconnect (ch->cadet);
   }
 }
 
 
-
-/******************************************************************************/
-/***********************      RECEIVE HANDLERS     ****************************/
-/******************************************************************************/
-
-
 /**
- * Call the @a notify callback given to #GNUNET_CADET_notify_transmit_ready to
- * request the data to send over MQ. Since MQ manages the queue, this function
- * is scheduled immediatly after a transmit ready notification.
+ * Implementation function that cancels the currently sent message.
+ * Should basically undo whatever #mq_send_impl() did.
  *
- * @param cls Closure (transmit handle).
+ * @param mq message queue
+ * @param impl_state state specific to the implementation
  */
 static void
-request_data (void *cls)
+cadet_mq_cancel_impl (struct GNUNET_MQ_Handle *mq,
+                     void *impl_state)
 {
-  struct GNUNET_CADET_TransmitHandle *th = cls;
-  struct GNUNET_CADET_LocalData *msg;
-  struct GNUNET_MQ_Envelope *env;
-  size_t osize;
+  struct GNUNET_CADET_Channel *ch = impl_state;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Requesting Data: %u bytes (allow send is %u)\n",
-       th->size,
-       th->channel->allow_send);
-
-  GNUNET_assert (0 < th->channel->allow_send);
-  th->channel->allow_send--;
-  /* NOTE: we may be allowed to send another packet immediately,
-     albeit the current logic waits for the ACK. */
-  th->request_data_task = NULL;
-  remove_from_queue (th);
-
-  env = GNUNET_MQ_msg_extra (msg,
-                             th->size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  msg->ccn = th->channel->ccn;
-  osize = th->notify (th->notify_cls,
-                      th->size,
-                      &msg[1]);
-  GNUNET_assert (osize == th->size);
-
-  GNUNET_MQ_send (th->channel->cadet->mq,
-                  env);
-  GNUNET_free (th);
+  GNUNET_assert (NULL != ch->pending_env);
+  GNUNET_MQ_discard (ch->pending_env);
+  ch->pending_env = NULL;
+  if (NULL != ch->mq_cont)
+  {
+    GNUNET_SCHEDULER_cancel (ch->mq_cont);
+    ch->mq_cont = NULL;
+  }
 }
 
 
@@ -586,18 +611,20 @@ handle_channel_created (void *cls,
     GNUNET_break (0);
     return;
   }
-  port = find_port (h, port_number);
+  port = find_port (h,
+                    port_number);
   if (NULL == port)
   {
+    /* We could have closed the port but the service didn't know about it yet
+     * This is not an error.
+     */
     struct GNUNET_CADET_LocalChannelDestroyMessage *d_msg;
     struct GNUNET_MQ_Envelope *env;
 
-    GNUNET_break (0);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "No handler for incoming channel %X [%s]\n",
+         "No handler for incoming channel %X (on port %s, recently closed?)\n",
          ntohl (ccn.channel_of_client),
          GNUNET_h2s (port_number));
-    /* FIXME: should disconnect instead, this is a serious error! */
     env = GNUNET_MQ_msg (d_msg,
                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
     d_msg->ccn = msg->ccn;
@@ -607,23 +634,32 @@ handle_channel_created (void *cls,
   }
 
   ch = create_channel (h,
-                       ccn);
-  ch->peer = GNUNET_PEER_intern (&msg->peer);
+                       &ccn);
+  ch->peer = msg->peer;
   ch->cadet = h;
-  ch->ccn = ccn;
-  ch->port = port;
+  ch->incoming_port = port;
   ch->options = ntohl (msg->opt);
-
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Creating incoming channel %X [%s] %p\n",
        ntohl (ccn.channel_of_client),
        GNUNET_h2s (port_number),
        ch);
-  ch->ctx = port->handler (port->cls,
-                           ch,
-                           &msg->peer,
-                           port->hash,
-                           ch->options);
+
+  GNUNET_assert (NULL != port->connects);
+  ch->window_changes = port->window_changes;
+  ch->disconnects = port->disconnects;
+  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
+                                          &cadet_mq_destroy_impl,
+                                          &cadet_mq_cancel_impl,
+                                          ch,
+                                          port->handlers,
+                                          &cadet_mq_error_handler,
+                                          ch);
+  ch->ctx = port->connects (port->cls,
+                            ch,
+                            &msg->peer);
+  GNUNET_MQ_set_handlers_closure (ch->mq,
+                                  ch->ctx);
 }
 
 
@@ -639,24 +675,20 @@ handle_channel_destroy (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Channel %X Destroy from service\n",
-       ntohl (ccn.channel_of_client));
-  ch = retrieve_channel (h,
-                         ccn);
 
+  ch = find_channel (h,
+                     msg->ccn);
   if (NULL == ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "channel %X unknown\n",
-         ntohl (ccn.channel_of_client));
+         "Received channel destroy for unknown channel %X from CADET service (recently close?)\n",
+         ntohl (msg->ccn.channel_of_client));
     return;
   }
-  destroy_channel (ch,
-                   GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received channel destroy for channel %X from CADET service\n",
+       ntohl (msg->ccn.channel_of_client));
+  destroy_channel (ch);
 }
 
 
@@ -672,25 +704,14 @@ static int
 check_local_data (void *cls,
                   const struct GNUNET_CADET_LocalData *message)
 {
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
   uint16_t size;
 
   size = ntohs (message->header.size);
   if (sizeof (*message) + sizeof (struct GNUNET_MessageHeader) > size)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-
-  ch = retrieve_channel (h,
-                         message->ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break_op (0);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
 }
 
@@ -707,46 +728,31 @@ handle_local_data (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   const struct GNUNET_MessageHeader *payload;
-  const struct GNUNET_CADET_MessageHandler *handler;
   struct GNUNET_CADET_Channel *ch;
   uint16_t type;
+  int fwd;
 
-  ch = retrieve_channel (h,
-                         message->ccn);
-  GNUNET_assert (NULL != ch);
+  ch = find_channel (h,
+                     message->ccn);
+  if (NULL == ch)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Unknown channel %X for incoming data (recently closed?)\n",
+         ntohl (message->ccn.channel_of_client));
+    return;
+  }
 
-  payload = (struct GNUNET_MessageHeader *) &message[1];
+  payload = (const struct GNUNET_MessageHeader *) &message[1];
   type = ntohs (payload->type);
+  fwd = ntohl (ch->ccn.channel_of_client) <= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got a %s data on channel %s [%X] of type %s (%u)\n",
-       GC_f2s (ntohl (ch->ccn.channel_of_client) >=
-               GNUNET_CADET_LOCAL_CHANNEL_ID_CLI),
-       GNUNET_i2s (GNUNET_PEER_resolve2 (ch->peer)),
+       "Got a %s data on channel %s [%X] of type %u\n",
+       fwd ? "FWD" : "BWD",
+       GNUNET_i2s (&ch->peer),
        ntohl (message->ccn.channel_of_client),
-       GC_m2s (type),
        type);
-  for (unsigned i=0;i<h->n_handlers;i++)
-  {
-    handler = &h->message_handlers[i];
-    if (handler->type == type)
-    {
-      if (GNUNET_OK !=
-          handler->callback (h->cls,
-                             ch,
-                             &ch->ctx,
-                             payload))
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "callback caused disconnection\n");
-        GNUNET_CADET_channel_destroy (ch);
-        return;
-      }
-      return;
-    }
-  }
-  /* Other peer sent message we do not comprehend. */
-  GNUNET_break_op (0);
-  GNUNET_CADET_receive_done (ch);
+  GNUNET_MQ_inject_message (ch->mq,
+                            payload);
 }
 
 
@@ -763,57 +769,36 @@ handle_local_ack (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct GNUNET_CADET_TransmitHandle *th;
 
-  ccn = message->ccn;
-  ch = retrieve_channel (h, ccn);
+  ch = find_channel (h,
+                     message->ccn);
   if (NULL == ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "ACK on unknown channel %X\n",
-         ntohl (ccn.channel_of_client));
+         ntohl (message->ccn.channel_of_client));
     return;
   }
   ch->allow_send++;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got an ACK on channel %X, allow send now %u!\n",
-       ntohl (ch->ccn.channel_of_client),
-       ch->allow_send);
-  for (th = h->th_head; NULL != th; th = th->next)
+  if (NULL == ch->pending_env)
   {
-    if ( (th->channel == ch) &&
-         (NULL == th->request_data_task) )
-    {
-      th->request_data_task
-        = GNUNET_SCHEDULER_add_now (&request_data,
-                                    th);
-      break;
-    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got an ACK on mq channel %X, allow send now %u!\n",
+         ntohl (ch->ccn.channel_of_client),
+         ch->allow_send);
+    notify_window_size (ch);
+    return;
   }
+  if (NULL != ch->mq_cont)
+    return; /* already working on it! */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got an ACK on mq channel %X, sending pending message!\n",
+       ntohl (ch->ccn.channel_of_client));
+  ch->mq_cont
+    = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
+                                ch);
 }
 
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
- *
- * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
- */
-static void
-reconnect (struct GNUNET_CADET_Handle *h);
-
-
-/**
- * Reconnect callback: tries to reconnect again after a failer previous
- * reconnection.
- *
- * @param cls closure (cadet handle)
- */
-static void
-reconnect_cbk (void *cls);
-
 
 /**
  * Generic error handler, called with the appropriate error code and
@@ -829,134 +814,15 @@ handle_mq_error (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MQ ERROR: %u\n", error);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "MQ ERROR: %u\n",
+              error);
   GNUNET_MQ_destroy (h->mq);
   h->mq = NULL;
   reconnect (h);
 }
 
 
-/*
- * Process a local reply about info on all channels, pass info to the user.
- *
- * @param h Cadet handle.
- * @param message Message itself.
- */
-// static void
-// process_get_channels (struct GNUNET_CADET_Handle *h,
-//                      const struct GNUNET_MessageHeader *message)
-// {
-//   struct GNUNET_CADET_LocalInfo *msg;
-//
-//   GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Get Channels messasge received\n");
-//
-//   if (NULL == h->channels_cb)
-//   {
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  ignored\n");
-//     return;
-//   }
-//
-//   msg = (struct GNUNET_CADET_LocalInfo *) message;
-//   if (ntohs (message->size) !=
-//       (sizeof (struct GNUNET_CADET_LocalInfo) +
-//        sizeof (struct GNUNET_PeerIdentity)))
-//   {
-//     GNUNET_break_op (0);
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-//                 "Get channels message: size %hu - expected %u\n",
-//                 ntohs (message->size),
-//                 sizeof (struct GNUNET_CADET_LocalInfo));
-//     return;
-//   }
-//   h->channels_cb (h->channels_cls,
-//                   ntohl (msg->channel_id),
-//                   &msg->owner,
-//                   &msg->destination);
-// }
-
-
-
-/*
- * Process a local monitor_channel reply, pass info to the user.
- *
- * @param h Cadet handle.
- * @param message Message itself.
- */
-// static void
-// process_show_channel (struct GNUNET_CADET_Handle *h,
-//                      const struct GNUNET_MessageHeader *message)
-// {
-//   struct GNUNET_CADET_LocalInfo *msg;
-//   size_t esize;
-//
-//   GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Show Channel messasge received\n");
-//
-//   if (NULL == h->channel_cb)
-//   {
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  ignored\n");
-//     return;
-//   }
-//
-//   /* Verify message sanity */
-//   msg = (struct GNUNET_CADET_LocalInfo *) message;
-//   esize = sizeof (struct GNUNET_CADET_LocalInfo);
-//   if (ntohs (message->size) != esize)
-//   {
-//     GNUNET_break_op (0);
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-//                 "Show channel message: size %hu - expected %u\n",
-//                 ntohs (message->size),
-//                 esize);
-//
-//     h->channel_cb (h->channel_cls, NULL, NULL);
-//     h->channel_cb = NULL;
-//     h->channel_cls = NULL;
-//
-//     return;
-//   }
-//
-//   h->channel_cb (h->channel_cls,
-//                  &msg->destination,
-//                  &msg->owner);
-// }
-
-
-
-/**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param message the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_peers (void *cls,
-                 const struct GNUNET_CADET_LocalInfoPeer *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  uint16_t size;
-
-  if (NULL == h->info_cb.peers_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for peesr monitor message!\n");
-    return GNUNET_SYSERR;
-  }
-
-  size = ntohs (message->header.size);
-  if (sizeof (struct GNUNET_CADET_LocalInfoPeer) > size)
-  {
-    h->info_cb.peers_cb (h->info_cls, NULL, -1, 0, 0);
-    h->info_cb.peers_cb = NULL;
-    h->info_cls = NULL;
-    return GNUNET_SYSERR;
-  }
-
-  return GNUNET_OK;
-}
-
-
 /**
  * Process a local reply about info on all tunnels, pass info to the user.
  *
@@ -968,9 +834,13 @@ handle_get_peers (void *cls,
                   const struct GNUNET_CADET_LocalInfoPeer *msg)
 {
   struct GNUNET_CADET_Handle *h = cls;
-  h->info_cb.peers_cb (h->info_cls, &msg->destination,
+
+  if (NULL == h->info_cb.peers_cb)
+    return;
+  h->info_cb.peers_cb (h->info_cls,
+                       &msg->destination,
                        (int) ntohs (msg->tunnel),
-                       (unsigned int ) ntohs (msg->paths),
+                       (unsigned int) ntohs (msg->paths),
                        0);
 }
 
@@ -987,62 +857,39 @@ static int
 check_get_peer (void *cls,
                 const struct GNUNET_CADET_LocalInfoPeer *message)
 {
-  struct GNUNET_CADET_Handle *h = cls;
-  const size_t msize = sizeof (struct GNUNET_CADET_LocalInfoPeer);
-  struct GNUNET_PeerIdentity *paths_array;
+  size_t msize = sizeof (struct GNUNET_CADET_LocalInfoPeer);
+  const struct GNUNET_PeerIdentity *paths_array;
   size_t esize;
   unsigned int epaths;
   unsigned int paths;
   unsigned int peers;
 
-  if (NULL == h->info_cb.peer_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for peer monitor message!\n");
-    goto clean_cls;
-  }
-
-  /* Verify message sanity */
   esize = ntohs (message->header.size);
   if (esize < msize)
   {
-    GNUNET_break_op (0);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   if (0 != ((esize - msize) % sizeof (struct GNUNET_PeerIdentity)))
   {
-    GNUNET_break_op (0);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
-
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   peers = (esize - msize) / sizeof (struct GNUNET_PeerIdentity);
-  epaths = (unsigned int) ntohs (message->paths);
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  epaths = ntohs (message->paths);
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   paths = 0;
-  for (int i = 0; i < peers; i++)
-  {
-    if (0 == memcmp (&paths_array[i], &message->destination,
+  for (unsigned int i = 0; i < peers; i++)
+    if (0 == memcmp (&paths_array[i],
+                     &message->destination,
                      sizeof (struct GNUNET_PeerIdentity)))
-    {
       paths++;
-    }
-  }
   if (paths != epaths)
   {
-    GNUNET_break_op (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "p:%u, e: %u\n", paths, epaths);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
-
-clean_cls:
-  h->info_cb.peer_cb = NULL;
-  h->info_cls = NULL;
-  return GNUNET_SYSERR;
 }
 
 
@@ -1057,22 +904,26 @@ handle_get_peer (void *cls,
                  const struct GNUNET_CADET_LocalInfoPeer *message)
 {
   struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_PeerIdentity *paths_array;
+  const struct GNUNET_PeerIdentity *paths_array;
   unsigned int paths;
   unsigned int path_length;
   int neighbor;
   unsigned int peers;
 
-  paths = (unsigned int) ntohs (message->paths);
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  if (NULL == h->info_cb.peer_cb)
+    return;
+  paths = ntohs (message->paths);
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   peers = (ntohs (message->header.size) - sizeof (*message))
           / sizeof (struct GNUNET_PeerIdentity);
   path_length = 0;
   neighbor = GNUNET_NO;
 
-  for (int i = 0; i < peers; i++)
+  for (unsigned int i = 0; i < peers; i++)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " %s\n", GNUNET_i2s (&paths_array[i]));
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                " %s\n",
+                GNUNET_i2s (&paths_array[i]));
     path_length++;
     if (0 == memcmp (&paths_array[i], &message->destination,
                      sizeof (struct GNUNET_PeerIdentity)))
@@ -1084,7 +935,7 @@ handle_get_peer (void *cls,
   }
 
   /* Call Callback with tunnel info. */
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   h->info_cb.peer_cb (h->info_cls,
                       &message->destination,
                       (int) ntohs (message->tunnel),
@@ -1095,40 +946,6 @@ handle_get_peer (void *cls,
 
 
 /**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param msg the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_tunnels (void *cls,
-                   const struct GNUNET_CADET_LocalInfoTunnel *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  uint16_t size;
-
-  if (NULL == h->info_cb.tunnels_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for tunnels monitor message!\n");
-    return GNUNET_SYSERR;
-  }
-
-  size = ntohs (msg->header.size);
-  if (sizeof (struct GNUNET_CADET_LocalInfoTunnel) > size)
-  {
-    h->info_cb.tunnels_cb (h->info_cls, NULL, 0, 0, 0, 0);
-    h->info_cb.tunnels_cb = NULL;
-    h->info_cls = NULL;
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
  * Process a local reply about info on all tunnels, pass info to the user.
  *
  * @param cls Closure (Cadet handle).
@@ -1140,6 +957,8 @@ handle_get_tunnels (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
 
+  if (NULL == h->info_cb.tunnels_cb)
+    return;
   h->info_cb.tunnels_cb (h->info_cls,
                          &msg->destination,
                          ntohl (msg->channels),
@@ -1162,28 +981,18 @@ static int
 check_get_tunnel (void *cls,
                   const struct GNUNET_CADET_LocalInfoTunnel *msg)
 {
-  struct GNUNET_CADET_Handle *h = cls;
   unsigned int ch_n;
   unsigned int c_n;
   size_t esize;
   size_t msize;
 
-  if (NULL == h->info_cb.tunnel_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for tunnel monitor message!\n");
-    goto clean_cls;
-  }
-
   /* Verify message sanity */
   msize = ntohs (msg->header.size);
   esize = sizeof (struct GNUNET_CADET_LocalInfoTunnel);
   if (esize > msize)
   {
-    GNUNET_break_op (0);
-    h->info_cb.tunnel_cb (h->info_cls,
-                          NULL, 0, 0, NULL, NULL, 0, 0);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
@@ -1198,17 +1007,9 @@ check_get_tunnel (void *cls,
                 (unsigned int) esize,
                 ch_n,
                 c_n);
-    h->info_cb.tunnel_cb (h->info_cls,
-                          NULL, 0, 0, NULL, NULL, 0, 0);
-    goto clean_cls;
+    return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
-
-clean_cls:
-  h->info_cb.tunnel_cb = NULL;
-  h->info_cls = NULL;
-  return GNUNET_SYSERR;
 }
 
 
@@ -1228,6 +1029,9 @@ handle_get_tunnel (void *cls,
   const struct GNUNET_CADET_ConnectionTunnelIdentifier *conns;
   const struct GNUNET_CADET_ChannelTunnelNumber *chns;
 
+  if (NULL == h->info_cb.tunnel_cb)
+    return;
+
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
 
@@ -1245,16 +1049,14 @@ handle_get_tunnel (void *cls,
 }
 
 
-
 /**
  * Reconnect to the service, retransmit all infomation to try to restore the
  * original state.
  *
  * @param h handle to the cadet
- * @return #GNUNET_YES in case of success, #GNUNET_NO otherwise (service down...)
  */
-static int
-do_reconnect (struct GNUNET_CADET_Handle *h)
+static void
+reconnect (struct GNUNET_CADET_Handle *h)
 {
   struct GNUNET_MQ_MessageHandler handlers[] = {
     GNUNET_MQ_hd_fixed_size (channel_created,
@@ -1273,32 +1075,25 @@ do_reconnect (struct GNUNET_CADET_Handle *h)
                              GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
                              struct GNUNET_CADET_LocalAck,
                              h),
-    GNUNET_MQ_hd_var_size (get_peers,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                           struct GNUNET_CADET_LocalInfoPeer,
-                           h),
+    GNUNET_MQ_hd_fixed_size (get_peers,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                             struct GNUNET_CADET_LocalInfoPeer,
+                             h),
     GNUNET_MQ_hd_var_size (get_peer,
                            GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
                            struct GNUNET_CADET_LocalInfoPeer,
                            h),
-    GNUNET_MQ_hd_var_size (get_tunnels,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                           struct GNUNET_CADET_LocalInfoTunnel,
-                           h),
+    GNUNET_MQ_hd_fixed_size (get_tunnels,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                             struct GNUNET_CADET_LocalInfoTunnel,
+                             h),
     GNUNET_MQ_hd_var_size (get_tunnel,
                            GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
                            struct GNUNET_CADET_LocalInfoTunnel,
                            h),
-  // FIXME
-//   GNUNET_MQ_hd_fixed_Y       size (channel_destroyed,
-//                            GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED,
-//                            struct GNUNET_CADET_ChannelDestroyMessage);
     GNUNET_MQ_handler_end ()
   };
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connecting to CADET\n");
-
-  GNUNET_assert (NULL == h->mq);
   h->mq = GNUNET_CLIENT_connect (h->cfg,
                                  "cadet",
                                  handlers,
@@ -1306,92 +1101,63 @@ do_reconnect (struct GNUNET_CADET_Handle *h)
                                  h);
   if (NULL == h->mq)
   {
-    reconnect (h);
-    return GNUNET_NO;
-  }
-  else
-  {
-    h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
+    schedule_reconnect (h);
+    return;
   }
-  return GNUNET_YES;
+  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
 }
 
+
 /**
- * Reconnect callback: tries to reconnect again after a failer previous
- * reconnecttion
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all channels that are still open.
  *
- * @param cls closure (cadet handle)
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param cid chanenl ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-reconnect_cbk (void *cls)
+static int
+destroy_channel_cb (void *cls,
+                    uint32_t cid,
+                    void *value)
 {
-  struct GNUNET_CADET_Handle *h = cls;
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Channel *ch = value;
 
-  h->reconnect_task = NULL;
-  do_reconnect (h);
+  if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "channel %X not destroyed\n",
+         ntohl (ch->ccn.channel_of_client));
+  }
+  destroy_channel (ch);
+  return GNUNET_OK;
 }
 
 
 /**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all ports that are still open.
  *
- * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param id port ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-reconnect (struct GNUNET_CADET_Handle *h)
-{
-  struct GNUNET_CADET_Channel *ch;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Requested RECONNECT, destroying all channels\n");
-  while (NULL != (ch = h->channels_head))
-    destroy_channel (ch, GNUNET_YES);
-  if (NULL == h->reconnect_task)
-    h->reconnect_task = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
-                                                      &reconnect_cbk, h);
-}
-
-
-/******************************************************************************/
-/**********************      API CALL DEFINITIONS     *************************/
-/******************************************************************************/
-
-struct GNUNET_CADET_Handle *
-GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      void *cls,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      const struct GNUNET_CADET_MessageHandler *handlers)
+static int
+destroy_port_cb (void *cls,
+                 const struct GNUNET_HashCode *id,
+                 void *value)
 {
-  struct GNUNET_CADET_Handle *h;
-
-  h = GNUNET_new (struct GNUNET_CADET_Handle);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GNUNET_CADET_connect() %p\n",
-       h);
-  h->cfg = cfg;
-  h->cleaner = cleaner;
-  h->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_YES);
-  do_reconnect (h);
-  if (h->mq == NULL)
-  {
-    GNUNET_break (0);
-    GNUNET_CADET_disconnect (h);
-    return NULL;
-  }
-  h->cls = cls;
-  h->message_handlers = handlers;
-  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
-  h->reconnect_task = NULL;
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Port *port = value;
 
-  /* count handlers */
-  for (h->n_handlers = 0;
-       handlers && handlers[h->n_handlers].type;
-       h->n_handlers++) ;
-  return h;
+  /* This is a warning, the app should have cleanly closed all open ports */
+  GNUNET_break (0);
+  GNUNET_CADET_close_port (port);
+  return GNUNET_OK;
 }
 
 
@@ -1406,58 +1172,16 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
 void
 GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
 {
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_Channel *aux;
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "CADET DISCONNECT\n");
-  ch = handle->channels_head;
-  while (NULL != ch)
-  {
-    aux = ch->next;
-    if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "channel %X not destroyed\n",
-           ntohl (ch->ccn.channel_of_client));
-    }
-    destroy_channel (ch,
-                     GNUNET_YES);
-    ch = aux;
-  }
-  while (NULL != (th = handle->th_head))
-  {
-    struct GNUNET_MessageHeader *msg;
-
-    /* Make sure it is an allowed packet (everything else should have been
-     * already canceled).
-     */
-    GNUNET_break (GNUNET_NO == th_is_payload (th));
-    msg = (struct GNUNET_MessageHeader *) &th[1];
-    switch (ntohs(msg->type))
-    {
-      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNELS:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS:
-        break;
-      default:
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_ERROR, "unexpected unsent msg %s\n",
-             GC_m2s (ntohs(msg->type)));
-    }
-
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
-  }
-
+  GNUNET_CONTAINER_multihashmap_iterate (handle->ports,
+                                         &destroy_port_cb,
+                                         handle);
+  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
+  handle->ports = NULL;
+  GNUNET_CONTAINER_multihashmap32_iterate (handle->channels,
+                                           &destroy_channel_cb,
+                                           handle);
+  GNUNET_CONTAINER_multihashmap32_destroy (handle->channels);
+  handle->channels = NULL;
   if (NULL != handle->mq)
   {
     GNUNET_MQ_destroy (handle->mq);
@@ -1465,58 +1189,15 @@ GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
   }
   if (NULL != handle->reconnect_task)
   {
-    GNUNET_SCHEDULER_cancel(handle->reconnect_task);
+    GNUNET_SCHEDULER_cancel (handle->reconnect_task);
     handle->reconnect_task = NULL;
   }
-
-  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
-  handle->ports = NULL;
   GNUNET_free (handle);
 }
 
 
 /**
- * Open a port to receive incomming channels.
- *
- * @param h CADET handle.
- * @param port Hash representing the port number.
- * @param new_channel Function called when an channel is received.
- * @param new_channel_cls Closure for @a new_channel.
- * @return Port handle.
- */
-struct GNUNET_CADET_Port *
-GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_InboundChannelNotificationHandler
-                            new_channel,
-                        void *new_channel_cls)
-{
-  struct GNUNET_CADET_PortMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_Port *p;
-
-  GNUNET_assert (NULL != new_channel);
-  p = GNUNET_new (struct GNUNET_CADET_Port);
-  p->cadet = h;
-  p->hash = GNUNET_new (struct GNUNET_HashCode);
-  *p->hash = *port;
-  p->handler = new_channel;
-  p->cls = new_channel_cls;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (h->ports,
-                                                    p->hash,
-                                                    p,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
-  msg->port = *p->hash;
-  GNUNET_MQ_send (h->mq, env);
-
-  return p;
-}
-
-/**
- * Close a port opened with @a GNUNET_CADET_open_port.
+ * Close a port opened with @a GNUNET_CADET_open_port().
  * The @a new_channel callback will no longer be called.
  *
  * @param p Port handle.
@@ -1527,110 +1208,45 @@ GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p)
   struct GNUNET_CADET_PortMessage *msg;
   struct GNUNET_MQ_Envelope *env;
 
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
-
-  msg->port = *p->hash;
-  GNUNET_MQ_send (p->cadet->mq, env);
-  GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports, p->hash, p);
-  GNUNET_free (p->hash);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
+  msg->port = p->id;
+  GNUNET_MQ_send (p->cadet->mq,
+                  env);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports,
+                                                       &p->id,
+                                                       p));
+  GNUNET_free_non_null (p->handlers);
   GNUNET_free (p);
 }
 
 
 /**
- * Create a new channel towards a remote peer.
+ * Destroy an existing channel.
  *
- * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
- * for this channel.
+ * The existing end callback for the channel will be called immediately.
+ * Any pending outgoing messages will be sent but no incoming messages will be
+ * accepted and no data callbacks will be called.
  *
- * @param h cadet handle
- * @param channel_ctx client's channel context to associate with the channel
- * @param peer peer identity the channel should go to
- * @param port Port hash (port number).
- * @param options CadetOption flag field, with all desired option bits set to 1.
- * @return handle to the channel
+ * @param channel Channel handle, becomes invalid after this call.
  */
-struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
-                            void *channel_ctx,
-                            const struct GNUNET_PeerIdentity *peer,
-                            const struct GNUNET_HashCode *port,
-                            enum GNUNET_CADET_ChannelOption options)
-{
-  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn.channel_of_client = htonl (0);
-  ch = create_channel (h, ccn);
-  ch->ctx = channel_ctx;
-  ch->peer = GNUNET_PEER_intern (peer);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating new channel to %s:%u at %p number %X\n",
-       GNUNET_i2s (peer),
-       port,
-       ch,
-       ntohl (ch->ccn.channel_of_client));
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  msg->ccn = ch->ccn;
-  msg->port = *port;
-  msg->peer = *peer;
-  msg->opt = htonl (options);
-  GNUNET_MQ_send (h->mq,
-                  env);
-  return ch;
-}
-
-
 void
 GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_CADET_Handle *h;
+  struct GNUNET_CADET_Handle *h = channel->cadet;
   struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
   struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_TransmitHandle *th;
-  struct GNUNET_CADET_TransmitHandle *next;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying channel\n");
-  h = channel->cadet;
-  for  (th = h->th_head; th != NULL; th = next)
+  if (NULL != h->mq)
   {
-    next = th->next;
-    if (th->channel == channel)
-    {
-      GNUNET_break (0);
-      if (GNUNET_YES == th_is_payload (th))
-      {
-        /* applications should cancel before destroying channel */
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "Channel destroyed without cancelling transmission requests\n");
-        th->notify (th->notify_cls, 0, NULL);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "no meta-traffic should be queued\n");
-      }
-      GNUNET_CONTAINER_DLL_remove (h->th_head,
-                                   h->th_tail,
-                                   th);
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-    }
+    env = GNUNET_MQ_msg (msg,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+    msg->ccn = channel->ccn;
+    GNUNET_MQ_send (h->mq,
+                    env);
   }
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  msg->ccn = channel->ccn;
-  GNUNET_MQ_send (h->mq,
-                  env);
-
-  destroy_channel (channel,
-                   GNUNET_YES);
+  destroy_channel (channel);
 }
 
 
@@ -1645,10 +1261,10 @@ GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
  */
 const union GNUNET_CADET_ChannelInfo *
 GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
-                              enum GNUNET_CADET_ChannelOption option, ...)
+                               enum GNUNET_CADET_ChannelOption option,
+                               ...)
 {
   static int bool_flag;
-  const union GNUNET_CADET_ChannelInfo *ret;
 
   switch (option)
   {
@@ -1659,74 +1275,15 @@ GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
         bool_flag = GNUNET_YES;
       else
         bool_flag = GNUNET_NO;
-      ret = (const union GNUNET_CADET_ChannelInfo *) &bool_flag;
+      return (const union GNUNET_CADET_ChannelInfo *) &bool_flag;
       break;
     case GNUNET_CADET_OPTION_PEER:
-      ret = (const union GNUNET_CADET_ChannelInfo *) GNUNET_PEER_resolve2 (channel->peer);
+      return (const union GNUNET_CADET_ChannelInfo *) &channel->peer;
       break;
     default:
       GNUNET_break (0);
       return NULL;
   }
-
-  return ret;
-}
-
-
-struct GNUNET_CADET_TransmitHandle *
-GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
-                                    int cork,
-                                    struct GNUNET_TIME_Relative maxdelay,
-                                    size_t notify_size,
-                                    GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                    void *notify_cls)
-{
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  GNUNET_assert (NULL != channel);
-  GNUNET_assert (NULL != notify);
-  GNUNET_assert (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE >= notify_size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "CADET NOTIFY TRANSMIT READY on channel %X allow_send is %u to %s with %u bytes\n",
-       channel->ccn,
-       channel->allow_send,
-       (ntohl (channel->ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-       ? "origin"
-       : "destination",
-       (unsigned int) notify_size);
-  if (GNUNET_TIME_UNIT_FOREVER_REL.rel_value_us != maxdelay.rel_value_us)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "CADET transmit ready timeout is deprected (has no effect)\n");
-  }
-
-  th = GNUNET_new (struct GNUNET_CADET_TransmitHandle);
-  th->channel = channel;
-  th->size = notify_size;
-  th->notify = notify;
-  th->notify_cls = notify_cls;
-  if (0 != channel->allow_send)
-    th->request_data_task
-      = GNUNET_SCHEDULER_add_now (&request_data,
-                                  th);
-  else
-    add_to_queue (channel->cadet,
-                  th);
-  return th;
-}
-
-
-void
-GNUNET_CADET_notify_transmit_ready_cancel (struct GNUNET_CADET_TransmitHandle *th)
-{
-  if (NULL != th->request_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (th->request_data_task);
-    th->request_data_task = NULL;
-  }
-  remove_from_queue (th);
-  GNUNET_free (th);
 }
 
 
@@ -1745,25 +1302,30 @@ GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel)
                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Sending ACK on channel %X\n",
-       channel->ccn.channel_of_client);
+       ntohl (channel->ccn.channel_of_client));
   msg->ccn = channel->ccn;
   GNUNET_MQ_send (channel->cadet->mq,
                   env);
 }
 
 
+/**
+ * Send message of @a type to CADET service of @a h
+ *
+ * @param h handle to CADET service
+ * @param type message type of trivial information request to send
+ */
 static void
-send_info_request (struct GNUNET_CADET_Handle *h, uint16_t type)
+send_info_request (struct GNUNET_CADET_Handle *h,
+                   uint16_t type)
 {
   struct GNUNET_MessageHeader *msg;
   struct GNUNET_MQ_Envelope *env;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " Sending %s monitor message to service\n",
-       GC_m2s(type));
-
-  env = GNUNET_MQ_msg (msg, type);
-  GNUNET_MQ_send (h->mq, env);
+  env = GNUNET_MQ_msg (msg,
+                       type);
+  GNUNET_MQ_send (h->mq,
+                  env);
 }
 
 
@@ -1777,8 +1339,8 @@ send_info_request (struct GNUNET_CADET_Handle *h, uint16_t type)
 void
 GNUNET_CADET_request_dump (struct GNUNET_CADET_Handle *h)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "requesting dump\n");
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP);
 }
 
 
@@ -1787,13 +1349,11 @@ GNUNET_CADET_request_dump (struct GNUNET_CADET_Handle *h)
  * The callback will be called for every peer known to the service.
  * Only one info request (of any kind) can be active at once.
  *
- *
  * WARNING: unstable API, likely to change in the future!
  *
  * @param h Handle to the cadet peer.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -1806,7 +1366,8 @@ GNUNET_CADET_get_peers (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
   h->info_cb.peers_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -1819,15 +1380,13 @@ GNUNET_CADET_get_peers (struct GNUNET_CADET_Handle *h,
  * WARNING: unstable API, likely to change in the future!
  *
  * @param h Cadet handle.
- *
- * @return Closure given to GNUNET_CADET_get_peers.
+ * @return Closure given to GNUNET_CADET_get_peers().
  */
 void *
 GNUNET_CADET_get_peers_cancel (struct GNUNET_CADET_Handle *h)
 {
-  void *cls;
+  void *cls = h->info_cls;
 
-  cls = h->info_cls;
   h->info_cb.peers_cb = NULL;
   h->info_cls = NULL;
   return cls;
@@ -1845,7 +1404,6 @@ GNUNET_CADET_get_peers_cancel (struct GNUNET_CADET_Handle *h)
  * @param id Peer whose tunnel to examine.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -1862,11 +1420,11 @@ GNUNET_CADET_get_peer (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
   msg->peer = *id;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   h->info_cb.peer_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -1883,7 +1441,6 @@ GNUNET_CADET_get_peer (struct GNUNET_CADET_Handle *h,
  * @param h Handle to the cadet peer.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -1896,7 +1453,8 @@ GNUNET_CADET_get_tunnels (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
   h->info_cb.tunnels_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -1907,23 +1465,19 @@ GNUNET_CADET_get_tunnels (struct GNUNET_CADET_Handle *h,
  * Cancel a monitor request. The monitor callback will not be called.
  *
  * @param h Cadet handle.
- *
- * @return Closure given to GNUNET_CADET_get_tunnels.
+ * @return Closure given to GNUNET_CADET_get_tunnels().
  */
 void *
 GNUNET_CADET_get_tunnels_cancel (struct GNUNET_CADET_Handle *h)
 {
-  void *cls;
+  void *cls = h->info_cls;
 
   h->info_cb.tunnels_cb = NULL;
-  cls = h->info_cls;
   h->info_cls = NULL;
-
   return cls;
 }
 
 
-
 /**
  * Request information about a tunnel of the running cadet peer.
  * The callback will be called for the tunnel once.
@@ -1935,7 +1489,6 @@ GNUNET_CADET_get_tunnels_cancel (struct GNUNET_CADET_Handle *h)
  * @param id Peer whose tunnel to examine.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -1952,11 +1505,11 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
   msg->peer = *id;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   h->info_cb.tunnel_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -1964,175 +1517,190 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
 
 
 /**
- * Request information about a specific channel of the running cadet peer.
- *
- * WARNING: unstable API, likely to change in the future!
- * FIXME Add destination option.
+ * Transitional function to convert an unsigned int port to a hash value.
+ * WARNING: local static value returned, NOT reentrant!
+ * WARNING: do not use this function for new code!
  *
- * @param h Handle to the cadet peer.
- * @param initiator ID of the owner of the channel.
- * @param channel_number Channel number.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
+ * @param port Numerical port (unsigned int format).
  *
- * @return #GNUNET_OK / #GNUNET_SYSERR
+ * @return A GNUNET_HashCode usable for the new CADET API.
  */
-int
-GNUNET_CADET_show_channel (struct GNUNET_CADET_Handle *h,
-                           struct GNUNET_PeerIdentity *initiator,
-                           unsigned int channel_number,
-                           GNUNET_CADET_ChannelCB callback,
-                           void *callback_cls)
+const struct GNUNET_HashCode *
+GC_u2h (uint32_t port)
 {
-  struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (NULL != h->info_cb.channel_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL);
-  msg->peer = *initiator;
-  msg->ccn.channel_of_client = htonl (channel_number);
-  GNUNET_MQ_send (h->mq, env);
+  static struct GNUNET_HashCode hash;
 
-  h->info_cb.channel_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "This is a transitional function, use proper crypto hashes as CADET ports\n");
+  GNUNET_CRYPTO_hash (&port,
+                      sizeof (port),
+                      &hash);
+  return &hash;
 }
 
 
 /**
- * Function called to notify a client about the connection
- * begin ready to queue more data.  "buf" will be
- * NULL and "size" zero if the connection was closed for
- * writing in the meantime.
+ * Connect to the MQ-based cadet service.
  *
- * @param cls closure
- * @param size number of bytes available in buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
+ * @param cfg Configuration to use.
+ *
+ * @return Handle to the cadet service NULL on error.
  */
-static size_t
-cadet_mq_ntr (void *cls, size_t size,
-             void *buf)
+struct GNUNET_CADET_Handle *
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct CadetMQState *state = GNUNET_MQ_impl_state (mq);
-  const struct GNUNET_MessageHeader *msg = GNUNET_MQ_impl_current (mq);
-  uint16_t msize;
+  struct GNUNET_CADET_Handle *h;
 
-  state->th = NULL;
-  if (NULL == buf)
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "GNUNET_CADET_connect()\n");
+  h = GNUNET_new (struct GNUNET_CADET_Handle);
+  h->cfg = cfg;
+  h->ports = GNUNET_CONTAINER_multihashmap_create (4,
+                                                   GNUNET_YES);
+  h->channels = GNUNET_CONTAINER_multihashmap32_create (4);
+  reconnect (h);
+  if (NULL == h->mq)
   {
-    GNUNET_MQ_inject_error (mq, GNUNET_MQ_ERROR_WRITE);
-    return 0;
+    GNUNET_break (0);
+    GNUNET_CADET_disconnect (h);
+    return NULL;
   }
-  msize = ntohs (msg->size);
-  GNUNET_assert (msize <= size);
-  GNUNET_memcpy (buf, msg, msize);
-  GNUNET_MQ_impl_send_continue (mq);
-  return msize;
+  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
+  h->reconnect_task = NULL;
+
+  return h;
 }
 
 
 /**
- * Signature of functions implementing the
- * sending functionality of a message queue.
+ * Open a port to receive incomming MQ-based channels.
  *
- * @param mq the message queue
- * @param msg the message to send
- * @param impl_state state of the implementation
+ * @param h CADET handle.
+ * @param port Hash identifying the port.
+ * @param connects Function called when an incoming channel is connected.
+ * @param connects_cls Closure for the @a connects handler.
+ * @param window_changes Function called when the transmit window size changes.
+ * @param disconnects Function called when a channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Port handle.
  */
-static void
-cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
-                    const struct GNUNET_MessageHeader *msg,
-                    void *impl_state)
+struct GNUNET_CADET_Port *
+GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
+                        const struct GNUNET_HashCode *port,
+                        GNUNET_CADET_ConnectEventHandler connects,
+                        void * connects_cls,
+                        GNUNET_CADET_WindowSizeEventHandler window_changes,
+                        GNUNET_CADET_DisconnectEventHandler disconnects,
+                        const struct GNUNET_MQ_MessageHandler *handlers)
 {
-  struct CadetMQState *state = impl_state;
-
-  GNUNET_assert (NULL == state->th);
-  state->th =
-      GNUNET_CADET_notify_transmit_ready (state->channel,
-                                         /* FIXME: add option for corking */
-                                         GNUNET_NO,
-                                         GNUNET_TIME_UNIT_FOREVER_REL,
-                                         ntohs (msg->size),
-                                         &cadet_mq_ntr, mq);
-
-}
+  struct GNUNET_CADET_PortMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_Port *p;
 
+  GNUNET_assert (NULL != connects);
+  GNUNET_assert (NULL != disconnects);
 
-/**
- * Signature of functions implementing the
- * destruction of a message queue.
- * Implementations must not free 'mq', but should
- * take care of 'impl_state'.
- *
- * @param mq the message queue to destroy
- * @param impl_state state of the implementation
- */
-static void
-cadet_mq_destroy_impl (struct GNUNET_MQ_Handle *mq,
-                       void *impl_state)
-{
-  struct CadetMQState *state = impl_state;
+  p = GNUNET_new (struct GNUNET_CADET_Port);
+  p->cadet = h;
+  p->id = *port;
+  p->connects = connects;
+  p->cls = connects_cls;
+  p->window_changes = window_changes;
+  p->disconnects = disconnects;
+  p->handlers = GNUNET_MQ_copy_handlers (handlers);
 
-  if (NULL != state->th)
-    GNUNET_CADET_notify_transmit_ready_cancel (state->th);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (h->ports,
+                                                    &p->id,
+                                                    p,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
-  GNUNET_free (state);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
+  msg->port = p->id;
+  GNUNET_MQ_send (h->mq,
+                  env);
+  return p;
 }
 
 
 /**
- * Create a message queue for a cadet channel.
- * The message queue can only be used to transmit messages,
- * not to receive them.
+ * Create a new channel towards a remote peer.
  *
- * @param channel the channel to create the message qeue for
- * @return a message queue to messages over the channel
+ * If the destination port is not open by any peer or the destination peer
+ * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
+ * for this channel.
+ *
+ * @param h CADET handle.
+ * @param channel_cls Closure for the channel. It's given to:
+ *                    - The disconnect handler @a disconnects
+ *                    - Each message type callback in @a handlers
+ * @param destination Peer identity the channel should go to.
+ * @param port Identification of the destination port.
+ * @param options CadetOption flag field, with all desired option bits set to 1.
+ * @param window_changes Function called when the transmit window size changes.
+ * @param disconnects Function called when the channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Handle to the channel.
  */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_mq_create (struct GNUNET_CADET_Channel *channel)
+struct GNUNET_CADET_Channel *
+GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
+                             void *channel_cls,
+                             const struct GNUNET_PeerIdentity *destination,
+                             const struct GNUNET_HashCode *port,
+                             enum GNUNET_CADET_ChannelOption options,
+                             GNUNET_CADET_WindowSizeEventHandler window_changes,
+                             GNUNET_CADET_DisconnectEventHandler disconnects,
+                             const struct GNUNET_MQ_MessageHandler *handlers)
 {
-  struct GNUNET_MQ_Handle *mq;
-  struct CadetMQState *state;
-
-  state = GNUNET_new (struct CadetMQState);
-  state->channel = channel;
-
-  mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
-                                      &cadet_mq_destroy_impl,
-                                      NULL, /* FIXME: cancel impl. */
-                                      state,
-                                      NULL, /* no msg handlers */
-                                      NULL, /* no err handlers */
-                                      NULL); /* no handler cls */
-  return mq;
+  struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  GNUNET_assert (NULL != disconnects);
+  ch = create_channel (h,
+                       NULL);
+  ch->ctx = channel_cls;
+  ch->peer = *destination;
+  ch->options = options;
+  ch->window_changes = window_changes;
+  ch->disconnects = disconnects;
+
+  /* Create MQ for channel */
+  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
+                                          &cadet_mq_destroy_impl,
+                                          &cadet_mq_cancel_impl,
+                                          ch,
+                                          handlers,
+                                          &cadet_mq_error_handler,
+                                          ch);
+  GNUNET_MQ_set_handlers_closure (ch->mq, channel_cls);
+
+  /* Request channel creation to service */
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  msg->ccn = ch->ccn;
+  msg->port = *port;
+  msg->peer = *destination;
+  msg->opt = htonl (options);
+  GNUNET_MQ_send (h->mq,
+                  env);
+  return ch;
 }
 
 
 /**
- * Transitional function to convert an unsigned int port to a hash value.
- * WARNING: local static value returned, NOT reentrant!
- * WARNING: do not use this function for new code!
+ * Obtain the message queue for a connected peer.
  *
- * @param port Numerical port (unsigned int format).
+ * @param channel The channel handle from which to get the MQ.
  *
- * @return A GNUNET_HashCode usable for the new CADET API.
+ * @return NULL if @a channel is not yet connected.
  */
-const struct GNUNET_HashCode *
-GC_u2h (uint32_t port)
+struct GNUNET_MQ_Handle *
+GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel)
 {
-  static struct GNUNET_HashCode hash;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "This is a transitional function, "
-              "use proper crypto hashes as CADET ports\n");
-  GNUNET_CRYPTO_hash (&port, sizeof (port), &hash);
-
-  return &hash;
+  return channel->mq;
 }
+
+/* end of cadet_api.c */
diff --git a/src/cadet/cadet_common.c b/src/cadet/cadet_common.c
deleted file mode 100644
index 95a3144e4..000000000
--- a/src/cadet/cadet_common.c
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_common.c
- * @brief CADET helper functions
- * @author Bartlomiej Polot
- */
-
-#include "cadet.h"
-
-/**
- * @brief Translate a fwd variable into a string representation, for logging.
- *
- * @param fwd Is FWD? (#GNUNET_YES or #GNUNET_NO)
- *
- * @return String representing FWD or BCK.
- */
-char *
-GC_f2s (int fwd)
-{
-  if (GNUNET_YES == fwd)
-  {
-    return "FWD";
-  }
-  else if (GNUNET_NO == fwd)
-  {
-    return "BCK";
-  }
-  else
-  {
-    /* Not an error, can happen with CONNECTION_BROKEN messages. */
-    return "\???";
-  }
-}
-
-
-/**
- * Test if @a bigger is larger than @a smaller.
- * Considers the case that @a bigger just overflowed
- * and is thus tiny while @a smaller is still below
- * `UINT32_MAX`.
- */
-int
-GC_is_pid_bigger (uint32_t bigger,
-                  uint32_t smaller)
-{
-    return (PID_OVERFLOW (smaller, bigger) ||
-            ( (bigger > smaller) &&
-              (! PID_OVERFLOW (bigger, smaller))) );
-}
-
-
-uint32_t
-GC_max_pid (uint32_t a, uint32_t b)
-{
-  if (GC_is_pid_bigger(a, b))
-    return a;
-  return b;
-}
-
-
-uint32_t
-GC_min_pid (uint32_t a, uint32_t b)
-{
-  if (GC_is_pid_bigger(a, b))
-    return b;
-  return a;
-}
-
-
-/**
- * Allocate a string with a hexdump of any binary data.
- *
- * @param bin Arbitrary binary data.
- * @param len Length of @a bin in bytes.
- * @param output Where to write the output (if *output be NULL it's allocated).
- *
- * @return The size of the output.
- */
-size_t
-GC_bin2s (void *bin, unsigned int len, char **output)
-{
-  char *data = bin;
-  char *buf;
-  unsigned int s_len;
-  unsigned int i;
-
-  s_len = 2 * len + 1;
-  if (NULL == *output)
-    *output = GNUNET_malloc (s_len);
-  buf = *output;
-
-  for (i = 0; i < len; i++)
-  {
-    SPRINTF (&buf[2 * i], "%2X", data[i]);
-  }
-  buf[s_len - 1] = '\0';
-
-  return s_len;
-}
-
-
-#if !defined(GNUNET_CULL_LOGGING)
-const char *
-GC_m2s (uint16_t m)
-{
-  static char buf[2][16];
-  static int idx;
-  const char *s;
-
-  idx = (idx + 1) % 2;
-  switch (m)
-  {
-    /**
-     * Used to mark the "payload" of a non-payload message.
-     */
-    case 0:
-      s = "retransmit";
-      break;
-
-      /**
-       * Request the creation of a path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-      s = "CONN_CREAT";
-      break;
-
-      /**
-       * Request the modification of an existing path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-      s = "CONN_ACK";
-      break;
-
-      /**
-       * Notify that a connection of a path is no longer valid
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-      s = "CONN_BRKN";
-      break;
-
-      /**
-       * At some point, the route will spontaneously change
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_PATH_CHANGED_UNIMPLEMENTED:
-      s = "PATH_CHNGD";
-      break;
-
-      /**
-       * Transport payload data.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      s = "DATA";
-      break;
-
-    /**
-     * Confirm receipt of payload data.
-     */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      s = "DATA_ACK";
-      break;
-
-      /**
-       * Key exchange message.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      s = "KX";
-      break;
-
-      /**
-       * Encrypted.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      s = "ENCRYPTED";
-      break;
-
-      /**
-       * Request the destuction of a path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      s = "CONN_DSTRY";
-      break;
-
-      /**
-       * ACK for a data packet.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-      s = "ACK";
-      break;
-
-      /**
-       * POLL for ACK.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-      s = "POLL";
-      break;
-
-      /**
-       * Announce origin is still alive.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE:
-      s = "KEEPALIVE";
-      break;
-
-      /**
-       * Open port
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN:
-      s = "OPEN_PORT";
-      break;
-
-      /**
-       * Close port
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE:
-      s = "CLOSE_PORT";
-      break;
-
-      /**
-       * Ask the cadet service to create a new tunnel
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      s = "CHAN_CREAT";
-      break;
-
-      /**
-       * Ask the cadet service to destroy a tunnel
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      s = "CHAN_DSTRY";
-      break;
-
-      /**
-       * Confirm the creation of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      s = "CHAN_ACK";
-      break;
-
-      /**
-       * Confirm the creation of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      s = "CHAN_NACK";
-      break;
-
-      /**
-       * Local payload traffic
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA:
-      s = "LOC_DATA";
-      break;
-
-      /**
-       * Local ACK for data.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK:
-      s = "LOC_ACK";
-      break;
-
-      /**
-       * Local monitoring of channels.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNELS:
-      s = "INFO_CHANS";
-      break;
-
-      /**
-       * Local monitoring of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL:
-      s = "INFO_CHAN";
-      break;
-
-      /**
-       * Local monitoring of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS:
-      s = "INFO_TUNS";
-      break;
-
-      /**
-       * Local monitoring of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL:
-      s = "INFO_TUN";
-      break;
-
-      /**
-       * Local information about all connections of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CONNECTIONS:
-      s = "INFO_CONNS";
-      break;
-
-      /**
-       * Local information of service about a specific connection.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CONNECTION:
-      s = "INFO_CONN";
-      break;
-
-      /**
-       * Local information about all peers known to the service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS:
-      s = "INFO_PEERS";
-      break;
-
-      /**
-       * Local information of service about a specific peer.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER:
-      s = "INFO_PEER";
-      break;
-
-      /**
-       * Traffic (net-cat style) used by the Command Line Interface.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CLI:
-      s = "CLI";
-      break;
-
-      /**
-       * Debug request.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP:
-      s = "INFO_DUMP";
-      break;
-
-      /**
-       * Used to mark the "payload" of a non-payload message.
-       */
-    case UINT16_MAX:
-      s = "      N/A";
-      break;
-
-
-    default:
-      SPRINTF (buf[idx], "{UNK: %5u}", m);
-      return buf[idx];
-  }
-  SPRINTF (buf[idx], "{%10s}", s);
-  return buf[idx];
-}
-#else
-const char *
-GC_m2s (uint16_t m)
-{
-  return "";
-}
-#endif
diff --git a/src/cadet/cadet_path.c b/src/cadet/cadet_path.c
deleted file mode 100644
index 79a498805..000000000
--- a/src/cadet/cadet_path.c
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001 - 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_path.c
- * @brief Path handling functions
- * @author Bartlomiej Polot
- */
-
-#include "cadet.h"
-#include "cadet_path.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-pth",__VA_ARGS__)
-
-
-/**
- * @brief Destroy a path after some time has past.
- * Removes the path from the peer (must not be used for direct paths).
- *
- * @param cls Closure (path to destroy).
- */
-static void
-path_destroy_delayed (void *cls)
-{
-  struct CadetPeerPath *path = cls;
-  struct CadetPeer *peer;
-
-  path->path_delete = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroy delayed %p (%u)\n",
-       path,
-       path->length);
-  GNUNET_assert (2 < path->length);
-  peer = GCP_get_short (path->peers[path->length - 1],
-                        GNUNET_NO);
-  GNUNET_assert (NULL != peer);
-  GCP_remove_path (peer, path);
-}
-
-
-/**
- * Create a new path
- *
- * @param length How many hops will the path have.
- * @return A newly allocated path with a peer array of the specified length.
- */
-struct CadetPeerPath *
-path_new (unsigned int length)
-{
-  struct CadetPeerPath *p;
-
-  p = GNUNET_new (struct CadetPeerPath);
-  if (length > 0)
-  {
-    p->length = length;
-    p->peers = GNUNET_malloc (length * sizeof (GNUNET_PEER_Id));
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "New path %p (%u)\n", p, p->length);
-  return p;
-}
-
-
-/**
- * Invert the path
- *
- * @param path the path to invert
- */
-void
-path_invert (struct CadetPeerPath *path)
-{
-  GNUNET_PEER_Id aux;
-  unsigned int i;
-
-  for (i = 0; i < path->length / 2; i++)
-  {
-    aux = path->peers[i];
-    path->peers[i] = path->peers[path->length - i - 1];
-    path->peers[path->length - i - 1] = aux;
-  }
-}
-
-
-/**
- * Duplicate a path, incrementing short peer's rc.
- *
- * @param path The path to duplicate.
- */
-struct CadetPeerPath *
-path_duplicate (const struct CadetPeerPath *path)
-{
-  struct CadetPeerPath *aux;
-  unsigned int i;
-
-  aux = path_new (path->length);
-  GNUNET_memcpy (aux->peers,
-          path->peers,
-          path->length * sizeof (GNUNET_PEER_Id));
-  for (i = 0; i < aux->length; i++)
-    GNUNET_PEER_change_rc (aux->peers[i], 1);
-  return aux;
-}
-
-
-/**
- * Get the length of a path.
- *
- * @param path The path to measure, with the local peer at any point of it.
- *
- * @return Number of hops to reach destination.
- *         UINT_MAX in case the peer is not in the path.
- */
-unsigned int
-path_get_length (struct CadetPeerPath *path)
-{
-  if (NULL == path)
-    return UINT_MAX;
-  return path->length;
-}
-
-
-
-/**
- * Mark path as invalid: keep it aroud for a while to avoid trying it in a loop.
- *
- * Never invalidates a two-hop (direct) path, only a core handler can do that.
- *
- * Rationale: DHT_get sometimes returns bad cached results, for instance,
- * on a locally cached result where the PUT followed a path that is no longer
- * current. The path must remain "known and marked as invalid" for a while.
- *
- * @param p Path to invalidate.
- */
-void
-path_invalidate (struct CadetPeerPath *p)
-{
-  if (NULL != p->path_delete)
-    return;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Invalidating path %p (%u)\n",
-       p,
-       p->length);
-  p->path_delete
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &path_destroy_delayed, p);
-}
-
-
-/**
- * Builds a path from a PeerIdentity array.
- *
- * @param peers PeerIdentity array.
- * @param size Size of the @c peers array.
- * @param myid ID of local peer, to find @c own_pos.
- * @param own_pos Output parameter: own position in the path.
- *
- * @return Fixed and shortened path.
- */
-struct CadetPeerPath *
-path_build_from_peer_ids (struct GNUNET_PeerIdentity *peers,
-                          unsigned int size,
-                          GNUNET_PEER_Id myid,
-                          unsigned int *own_pos)
-{
-  struct CadetPeerPath *path;
-  GNUNET_PEER_Id shortid;
-  unsigned int i;
-  unsigned int j;
-  unsigned int offset;
-
-  /* Create path */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Creating path...\n");
-  path = path_new (size);
-  *own_pos = 0;
-  offset = 0;
-  for (i = 0; i < size; i++)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  - %u: taking %s\n",
-         i, GNUNET_i2s (&peers[i]));
-    shortid = GNUNET_PEER_intern (&peers[i]);
-
-    /* Check for loops / duplicates */
-    for (j = 0; j < i - offset; j++)
-    {
-      if (path->peers[j] == shortid)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "    already exists at pos %u\n", j);
-        offset = i - j;
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "    offset now %u\n", offset);
-        GNUNET_PEER_change_rc (shortid, -1);
-      }
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "    storing at %u\n", i - offset);
-    path->peers[i - offset] = shortid;
-    if (path->peers[i - offset] == myid)
-      *own_pos = i - offset;
-  }
-  path->length -= offset;
-
-  if (path->peers[*own_pos] != myid)
-  {
-    /* create path: self not found in path through self */
-    GNUNET_break_op (0);
-    path_destroy (path);
-    return NULL;
-  }
-
-  return path;
-}
-
-
-/**
- * Test if two paths are equivalent (equal or revese of each other).
- *
- * @param p1 First path
- * @param p2 Second path
- *
- * @return #GNUNET_YES if both paths are equivalent
- *         #GNUNET_NO otherwise
- */
-int
-path_equivalent (const struct CadetPeerPath *p1,
-                 const struct CadetPeerPath *p2)
-{
-  unsigned int i;
-  unsigned int l;
-  unsigned int half;
-
-  if (NULL == p1 || NULL == p2)
-    return GNUNET_NO;
-
-  if (p1->length != p2->length)
-    return GNUNET_NO;
-
-  l = p1->length;
-  if (0 == memcmp (p1->peers, p2->peers, sizeof (p1->peers[0]) * l))
-    return GNUNET_YES;
-
-  half = l / 2;
-  l = l - 1;
-  for (i = 0; i <= half; i++)
-    if (p1->peers[i] != p2->peers[l - i])
-      return GNUNET_NO;
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Test if a path is valid (or at least not known to be invalid).
- *
- * @param path Path to test.
- *
- * @return #GNUNET_YES If the path is valid or unknown,
- *         #GNUNET_NO If the path is known to be invalid.
- */
-int
-path_is_valid (const struct CadetPeerPath *path)
-{
-  return (NULL == path->path_delete);
-}
-
-
-/**
- * Destroy the path and free any allocated resources linked to it
- *
- * @param p the path to destroy
- *
- * @return #GNUNET_OK on success
- */
-int
-path_destroy (struct CadetPeerPath *p)
-{
-  if (NULL == p)
-    return GNUNET_OK;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying path %p (%u)\n",
-       p,
-       p->length);
-  GNUNET_PEER_decrement_rcs (p->peers, p->length);
-  GNUNET_free_non_null (p->peers);
-  if (NULL != p->path_delete)
-    GNUNET_SCHEDULER_cancel (p->path_delete);
-  GNUNET_free (p);
-  return GNUNET_OK;
-}
-
-
-/**
- * Compare two paths.
- *
- * @param p1 First path.
- * @param p2 Second path.
- *
- * @return > 0 if p1 is longer, or the first differing PEER_Id is higher on p1.
- *         < 0 if p2 is longer, or the first differing PEER_Id is higher on p2.
- *         0 if they are identical.
- */
-int
-path_cmp (const struct CadetPeerPath *p1,
-          const struct CadetPeerPath *p2)
-{
-  if (p1->length > p2->length)
-    return 1;
-
-  if (p1->length < p2->length)
-    return -1;
-
-  return memcmp (p1->peers,
-                 p2->peers,
-                 sizeof (GNUNET_PEER_Id) * p1->length);
-}
-
-
-char *
-path_2s (struct CadetPeerPath *p)
-{
-  char *s;
-  char *old;
-  unsigned int i;
-
-  old = GNUNET_strdup ("");
-  for (i = 0; i < p->length; i++)
-  {
-    GNUNET_asprintf (&s, "%s %s",
-                     old, GNUNET_i2s (GNUNET_PEER_resolve2 (p->peers[i])));
-    GNUNET_free_non_null (old);
-    old = s;
-  }
-  return old;
-}
-
-
-void
-path_debug (struct CadetPeerPath *p)
-{
-  unsigned int i;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "PATH:\n");
-  for (i = 0; i < p->length; i++)
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  %s\n",
-                GNUNET_i2s (GNUNET_PEER_resolve2 (p->peers[i])));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "END\n");
-}
diff --git a/src/cadet/cadet_path.h b/src/cadet/cadet_path.h
deleted file mode 100644
index bb68eec42..000000000
--- a/src/cadet/cadet_path.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001 - 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_path.h
- * @brief Path handling functions
- * @author Bartlomiej Polot
- */
-
-#ifndef CADET_PATH_H_
-#define CADET_PATH_H_
-
-#ifdef __cplusplus
-extern "C"
-{
-  #if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-
-/******************************************************************************/
-/************************      DATA STRUCTURES     ****************************/
-/******************************************************************************/
-
-/**
- * Information regarding a possible path to reach a single peer
- */
-struct CadetPeerPath
-{
-
-    /**
-     * Linked list
-     */
-  struct CadetPeerPath *next;
-  struct CadetPeerPath *prev;
-
-    /**
-     * List of all the peers that form the path from origin to target.
-     */
-  GNUNET_PEER_Id *peers;
-
-    /**
-     * Number of peers (hops) in the path
-     */
-  unsigned int length;
-
-    /**
-     * User defined data store.
-     */
-  struct CadetConnection *c;
-
-    /**
-     * Path's score, how reliable is the path.
-     */
-//   int score;
-
-  /**
-   * Task to delete the path.
-   * We tried it, it didn't work, don't try again in a while.
-   */
-  struct GNUNET_SCHEDULER_Task * path_delete;
-
-};
-
-/******************************************************************************/
-/*************************        FUNCTIONS       *****************************/
-/******************************************************************************/
-
-/**
- * Create a new path.
- *
- * @param length How many hops will the path have.
- *
- * @return A newly allocated path with a peer array of the specified length.
- */
-struct CadetPeerPath *
-path_new (unsigned int length);
-
-
-/**
- * Invert the path.
- *
- * @param path The path to invert.
- */
-void
-path_invert (struct CadetPeerPath *path);
-
-
-/**
- * Duplicate a path, incrementing short peer's rc.
- *
- * @param path The path to duplicate.
- */
-struct CadetPeerPath *
-path_duplicate (const struct CadetPeerPath *path);
-
-
-/**
- * Get the length of a path.
- *
- * @param path The path to measure, with the local peer at any point of it.
- *
- * @return Number of hops to reach destination.
- *         UINT_MAX in case the peer is not in the path.
- */
-unsigned int
-path_get_length (struct CadetPeerPath *path);
-
-/**
- * Mark path as invalid: keep it aroud for a while to avoid trying it in a loop.
- *
- * DHT_get sometimes returns bad cached results, for instance, on a locally
- * cached result where the PUT followed a path that is no longer current.
- *
- * @param p Path to invalidate.
- */
-void
-path_invalidate (struct CadetPeerPath *p);
-
-/**
- * Test if two paths are equivalent (equal or revese of each other).
- *
- * @param p1 First path
- * @param p2 Second path
- *
- * @return GNUNET_YES if both paths are equivalent
- *         GNUNET_NO otherwise
- */
-int
-path_equivalent (const struct CadetPeerPath *p1,
-                 const struct CadetPeerPath *p2);
-
-/**
- * Test if a path is valid (or at least not known to be invalid).
- *
- * @param path Path to test.
- *
- * @return #GNUNET_YES If the path is valid or unknown,
- *         #GNUNET_NO If the path is known to be invalid.
- */
-int
-path_is_valid (const struct CadetPeerPath *path);
-
-/**
- * Destroy the path and free any allocated resources linked to it
- *
- * @param p the path to destroy
- *
- * @return GNUNET_OK on success
- */
-int
-path_destroy (struct CadetPeerPath *p);
-
-/**
- * Compare two paths.
- *
- * @param p1 First path.
- * @param p2 Second path.
- *
- * @return > 0 if p1 is longer, or the first differing PEER_Id is higher on p1.
- *         < 0 if p2 is longer, or the first differing PEER_Id is higher on p2.
- *         0 if they are identical.
- */
-int
-path_cmp (const struct CadetPeerPath *p1, const struct CadetPeerPath *p2);
-
-/**
- * Builds a path from a PeerIdentity array.
- *
- * @param peers PeerIdentity array.
- * @param size Size of the @c peers array.
- * @param myid ID of local peer, to find @c own_pos.
- * @param own_pos Output parameter: own position in the path.
- *
- * @return Fixed and shortened path.
- */
-struct CadetPeerPath *
-path_build_from_peer_ids (struct GNUNET_PeerIdentity *peers,
-                          unsigned int size,
-                          GNUNET_PEER_Id myid,
-                          unsigned int *own_pos);
-
-/**
- * Path -> allocated one line string. Caller must free.
- *
- * @param p Path.
- */
-char *
-path_2s (struct CadetPeerPath *p);
-
-/**
- * Print info about the path for debug.
- *
- * @param p Path to debug.
- */
-void
-path_debug (struct CadetPeerPath *p);
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-  #endif
-  #ifdef __cplusplus
-}
-#endif
-
-
-/* ifndef CADET_PATH_H */
-#endif
diff --git a/src/cadet/cadet_protocol.h b/src/cadet/cadet_protocol.h
index 5ec34f7d7..de0cec5d0 100644
--- a/src/cadet/cadet_protocol.h
+++ b/src/cadet/cadet_protocol.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001 - 2011 GNUnet e.V.
+     Copyright (C) 2007 - 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,8 +19,10 @@
 */
 
 /**
- * @author Bartlomiej Polot
  * @file cadet/cadet_protocol.h
+ * @brief P2P messages used by CADET
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
 
 #ifndef CADET_PROTOCOL_H_
@@ -67,9 +69,12 @@ struct GNUNET_CADET_ConnectionCreateMessage
   struct GNUNET_MessageHeader header;
 
   /**
-   * For alignment.
+   * Connection options in network byte order.
+   * #GNUNET_CADET_OPTION_DEFAULT for buffered;
+   * #GNUNET_CADET_OPTION_NOBUFFER for unbuffered.
+   * Other flags are ignored and should not be set at this level.
    */
-  uint32_t reserved GNUNET_PACKED;
+  uint32_t options GNUNET_PACKED;
 
   /**
    * ID of the connection
@@ -205,7 +210,9 @@ enum GNUNET_CADET_KX_Flags {
 struct GNUNET_CADET_TunnelKeyExchangeMessage
 {
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX.
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX or
+   * #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH as part
+   * of `struct GNUNET_CADET_TunnelKeyExchangeAuthMessage`.
    */
   struct GNUNET_MessageHeader header;
 
@@ -234,50 +241,36 @@ struct GNUNET_CADET_TunnelKeyExchangeMessage
    */
   struct GNUNET_CRYPTO_EcdhePublicKey ratchet_key;
 
-#ifdef NEW_CADET
-  /**
-   * Proof that sender could compute the 3-DH, in lieu of a signature.
-   */
-  struct GNUNET_HashCode triple_dh_proof;
-#endif
 };
 
 
 /**
- * Axolotl tunnel message.
+ * Message for a Key eXchange for a tunnel, with authentication.
+ * Used as a response to the initial KX as well as for rekeying.
  */
-struct GNUNET_CADET_TunnelEncryptedMessage
+struct GNUNET_CADET_TunnelKeyExchangeAuthMessage
 {
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED
-   */
-  struct GNUNET_MessageHeader header;
 
-#if NEW_CADET
   /**
-   * Reserved, for alignment.
+   * Message header with key material.
    */
-  uint32_t reserved GNUNET_PACKED;
-#else
-  /**
-   * Maximum packet ID authorized.
-   */
-  struct CadetEncryptedMessageIdentifier cemi;
-#endif
+  struct GNUNET_CADET_TunnelKeyExchangeMessage kx;
 
   /**
-   * ID of the connection.
+   * KDF-proof that sender could compute the 3-DH, used in lieu of a
+   * signature or payload data.
    */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+  struct GNUNET_HashCode auth;
+
+};
 
-  /**
-   * MAC of the encrypted message, used to verify message integrity.
-   * Everything after this value  will be encrypted with the header key
-   * and authenticated.
-   */
-  struct GNUNET_ShortHashCode hmac;
 
-  /**************** AX_HEADER start ****************/
+/**
+ * Encrypted axolotl header with numbers that identify which
+ * keys in which ratchet are to be used to decrypt the body.
+ */
+struct GNUNET_CADET_AxHeader
+{
 
   /**
    * Number of messages sent with the current ratchet key.
@@ -294,68 +287,47 @@ struct GNUNET_CADET_TunnelEncryptedMessage
    */
   struct GNUNET_CRYPTO_EcdhePublicKey DHRs;
 
-  /**************** AX_HEADER  end  ****************/
-
-  /**
-   * Encrypted content follows.
-   */
 };
 
 
-#ifndef NEW_CADET
-
 /**
- * Message to query a peer about its Flow Control status regarding a tunnel.
- *
- * It is NOT yet clear if we need this.
+ * Axolotl-encrypted tunnel message with application payload.
  */
-struct GNUNET_CADET_ConnectionHopByHopPollMessage
+struct GNUNET_CADET_TunnelEncryptedMessage
 {
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED
    */
   struct GNUNET_MessageHeader header;
 
   /**
-   * Last packet sent.
+   * Reserved, for alignment.
    */
-  struct CadetEncryptedMessageIdentifier cemi;
+  uint32_t reserved GNUNET_PACKED;
 
   /**
    * ID of the connection.
    */
   struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 
-};
-
-
-/**
- * Message to acknowledge cadet encrypted traffic, used for
- * flow-control on a hop-by-hop basis on the connection-level. Note
- * that we do use the @e cemi from the tunnel layer as the connection
- * layer's header is included/shared with the tunnel layer messages,
- * and we only do flow control for the payload.
- */
-struct GNUNET_CADET_ConnectionEncryptedAckMessage
-{
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK
+   * MAC of the encrypted message, used to verify message integrity.
+   * Everything after this value  will be encrypted with the header key
+   * and authenticated.
    */
-  struct GNUNET_MessageHeader header;
+  struct GNUNET_ShortHashCode hmac;
 
   /**
-   * Maximum packet ID authorized.
+   * Axolotl-header that specifies which keys to use in which ratchet
+   * to decrypt the body that follows.
    */
-  struct CadetEncryptedMessageIdentifier cemi_max;
+  struct GNUNET_CADET_AxHeader ax_header;
 
   /**
-   * ID of the connection.
+   * Encrypted content follows.
    */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 };
 
-#endif
-
 
 /******************************************************************************/
 /*******************************   CHANNEL  ***********************************/
@@ -378,9 +350,9 @@ struct GNUNET_CADET_ChannelOpenMessage
   uint32_t opt GNUNET_PACKED;
 
   /**
-   * Destination port.
+   * Hash of destination port and listener.
    */
-  struct GNUNET_HashCode port;
+  struct GNUNET_HashCode h_port;
 
   /**
    * ID of the channel within the tunnel.
@@ -390,93 +362,56 @@ struct GNUNET_CADET_ChannelOpenMessage
 
 
 /**
- * Message to manage a Channel
- * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK,
- * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY).
+ * Message to acknowledge opening a channel of type
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
  */
-struct GNUNET_CADET_ChannelManageMessage
+struct GNUNET_CADET_ChannelOpenAckMessage
 {
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK or
-   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK
    */
   struct GNUNET_MessageHeader header;
 
-#ifdef NEW_CADET
   /**
    * For alignment.
    */
   uint32_t reserved GNUNET_PACKED;
-#endif
 
   /**
    * ID of the channel
    */
   struct GNUNET_CADET_ChannelTunnelNumber ctn;
-};
-
-
-#ifndef NEW_CADET
-
-/**
- * Message for cadet data traffic.
- */
-struct GNUNET_CADET_ChannelAppDataMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_UNICAST,
-   *       #GNUNET_MESSAGE_TYPE_CADET_TO_ORIGIN
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique ID of the payload message
-   */
-  /* NEW: struct ChannelMessageIdentifier */
-  uint32_t mid GNUNET_PACKED;
 
   /**
-   * ID of the channel
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Payload follows
+   * Port number of the channel, used to prove to the
+   * initiator that the receiver knows the port.
    */
+  struct GNUNET_HashCode port;
 };
 
 
 /**
- * Message to acknowledge end-to-end data.
+ * Message to destroy a channel of type
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY.
  */
-struct GNUNET_CADET_ChannelDataAckMessage
+struct GNUNET_CADET_ChannelDestroyMessage
 {
   /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY
    */
   struct GNUNET_MessageHeader header;
 
   /**
-   * ID of the channel
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Bitfield of already-received newer messages
-   * pid +  1 @ LSB
-   * pid + 64 @ MSB
+   * For alignment.
    */
-  uint64_t futures GNUNET_PACKED;
+  uint32_t reserved GNUNET_PACKED;
 
   /**
-   * Last message ID received.
+   * ID of the channel
    */
-  /* NEW: struct ChannelMessageIdentifier */
-  uint32_t mid GNUNET_PACKED;
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
 };
 
-#else
-
 
 /**
  * Number used to uniquely identify messages in a CADET Channel.
@@ -532,21 +467,21 @@ struct GNUNET_CADET_ChannelDataAckMessage
   struct GNUNET_CADET_ChannelTunnelNumber ctn;
 
   /**
-   * Bitfield of already-received messages past @e mid.
-   * pid +  1 @ LSB
-   * pid + 64 @ MSB
+   * Bitfield of already-received newer messages.  Note that bit 0
+   * corresponds to @e mid + 1.
+   *
+   * pid +  0 @ LSB
+   * pid + 63 @ MSB
    */
   uint64_t futures GNUNET_PACKED;
 
   /**
-   * Last message ID received.
+   * Next message ID expected.
    */
   struct ChannelMessageIdentifier mid;
 };
 
 
-#endif
-
 GNUNET_NETWORK_STRUCT_END
 
 #if 0                           /* keep Emacsens' auto-indent happy */
diff --git a/src/cadet/cadet_test_lib.c b/src/cadet/cadet_test_lib.c
index 9a70dad49..1df6bff0d 100644
--- a/src/cadet/cadet_test_lib.c
+++ b/src/cadet/cadet_test_lib.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -27,6 +27,7 @@
 #include "cadet_test_lib.h"
 #include "gnunet_cadet_service.h"
 
+
 /**
  * Test context for a CADET Test.
  */
@@ -40,7 +41,7 @@ struct GNUNET_CADET_TEST_Context
   /**
    * Array of handles to the CADET for each peer.
    */
-  struct GNUNET_CADET_Handle **cadetes;
+  struct GNUNET_CADET_Handle **cadets;
 
   /**
    * Operation associated with the connection to the CADET.
@@ -48,6 +49,11 @@ struct GNUNET_CADET_TEST_Context
   struct GNUNET_TESTBED_Operation **ops;
 
   /**
+   * Number of peers running, size of the arrays above.
+   */
+  unsigned int num_peers;
+
+  /**
    * Main function of the test to run once all CADETs are available.
    */
   GNUNET_CADET_TEST_AppMain app_main;
@@ -58,30 +64,35 @@ struct GNUNET_CADET_TEST_Context
   void *app_main_cls;
 
   /**
-   * Number of peers running, size of the arrays above.
+   * Handler for incoming tunnels.
    */
-  unsigned int num_peers;
+  GNUNET_CADET_ConnectEventHandler connects;
 
   /**
-   * Handler for incoming tunnels.
+   * Function called when the transmit window size changes.
    */
-  GNUNET_CADET_InboundChannelNotificationHandler *new_channel;
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
 
   /**
    * Cleaner for destroyed incoming tunnels.
    */
-  GNUNET_CADET_ChannelEndHandler *cleaner;
+  GNUNET_CADET_DisconnectEventHandler disconnects;
 
   /**
    * Message handlers.
    */
-  struct GNUNET_CADET_MessageHandler* handlers;
+  struct GNUNET_MQ_MessageHandler *handlers;
 
   /**
    * Application ports.
    */
   const struct GNUNET_HashCode **ports;
 
+  /**
+   * Number of ports in #ports.
+   */
+  unsigned int port_count;
+
 };
 
 
@@ -94,6 +105,11 @@ struct GNUNET_CADET_TEST_AdapterContext
    * Peer number for the particular peer.
    */
   unsigned int peer;
+
+  /**
+   * Port handlers for open ports.
+   */
+  struct GNUNET_CADET_Port **ports;
  
   /**
    * General context.
@@ -114,26 +130,28 @@ struct GNUNET_CADET_TEST_AdapterContext
  */
 static void *
 cadet_connect_adapter (void *cls,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg)
+                       const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
   struct GNUNET_CADET_TEST_Context *ctx = actx->ctx;
   struct GNUNET_CADET_Handle *h;
+  unsigned int i;
 
-  h = GNUNET_CADET_connect (cfg,
-                           (void *) (long) actx->peer,
-                           ctx->cleaner,
-                           ctx->handlers);
+  h = GNUNET_CADET_connect (cfg);
   if (NULL == ctx->ports)
     return h;
 
-  for (int i = 0; NULL != ctx->ports[i]; i++)
+  actx->ports = GNUNET_new_array (ctx->port_count, struct GNUNET_CADET_Port *);
+  for (i = 0; i < ctx->port_count; i++)
   {
-    (void ) GNUNET_CADET_open_port (h, ctx->ports[i],
-                                    ctx->new_channel,
-                                    (void *) (long) actx->peer);
+    actx->ports[i] = GNUNET_CADET_open_port (h,
+                                             ctx->ports[i],
+                                             ctx->connects,
+                                             (void *) (long) actx->peer,
+                                             ctx->window_changes,
+                                             ctx->disconnects,
+                                             ctx->handlers);
   }
-
   return h;
 }
 
@@ -152,6 +170,15 @@ cadet_disconnect_adapter (void *cls,
   struct GNUNET_CADET_Handle *cadet = op_result;
   struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
 
+  if (NULL != actx->ports)
+  {
+    for (int i = 0; i < actx->ctx->port_count; i++)
+    {
+      GNUNET_CADET_close_port (actx->ports[i]);
+      actx->ports[i] = NULL;
+    }
+    GNUNET_free (actx->ports);
+  }
   GNUNET_free (actx);
   GNUNET_CADET_disconnect (cadet);
 }
@@ -186,18 +213,18 @@ cadet_connect_cb (void *cls,
   for (i = 0; i < ctx->num_peers; i++)
     if (op == ctx->ops[i])
     {
-      ctx->cadetes[i] = ca_result;
+      ctx->cadets[i] = ca_result;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO, "...cadet %u connected\n", i);
     }
   for (i = 0; i < ctx->num_peers; i++)
-    if (NULL == ctx->cadetes[i])
+    if (NULL == ctx->cadets[i])
       return; /* still some CADET connections missing */
   /* all CADET connections ready! */
   ctx->app_main (ctx->app_main_cls,
                  ctx,
                  ctx->num_peers,
                  ctx->peers,
-                 ctx->cadetes);
+                 ctx->cadets);
 }
 
 
@@ -213,7 +240,7 @@ GNUNET_CADET_TEST_cleanup (struct GNUNET_CADET_TEST_Context *ctx)
     ctx->ops[i] = NULL;
   }
   GNUNET_free (ctx->ops);
-  GNUNET_free (ctx->cadetes);
+  GNUNET_free (ctx->cadets);
   GNUNET_free (ctx);
   GNUNET_SCHEDULER_shutdown ();
 }
@@ -243,12 +270,23 @@ cadet_test_run (void *cls,
   struct GNUNET_CADET_TEST_Context *ctx = cls;
   unsigned int i;
 
+  if (0 != links_failed)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Some links failed (%u), ending\n",
+                links_failed);
+    exit (2);
+  }
+
   if  (num_peers != ctx->num_peers)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Peers started %u/%u, ending\n",
                 num_peers, ctx->num_peers);
     exit (1);
   }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testbed up, %u peers and %u links\n",
+              num_peers, links_succeeded);
   ctx->peers = peers;
   for (i = 0; i < num_peers; i++)
   {
@@ -270,31 +308,52 @@ cadet_test_run (void *cls,
 }
 
 
+/**
+ * Run a test using the given name, configuration file and number of peers.
+ * All cadet callbacks will receive the peer number (long) as the closure.
+ *
+ * @param testname Name of the test (for logging).
+ * @param cfgfile Name of the configuration file.
+ * @param num_peers Number of peers to start.
+ * @param tmain Main function to run once the testbed is ready.
+ * @param tmain_cls Closure for @a tmain.
+ * @param connects Handler for incoming channels.
+ * @param window_changes Handler for the window size change notification.
+ * @param disconnects Cleaner for destroyed incoming channels.
+ * @param handlers Message handlers.
+ * @param ports Ports the peers offer, NULL-terminated.
+ */
 void
-GNUNET_CADET_TEST_run (const char *testname,
-                      const char *cfgname,
-                      unsigned int num_peers,
-                      GNUNET_CADET_TEST_AppMain tmain,
-                      void *tmain_cls,
-                      GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      struct GNUNET_CADET_MessageHandler* handlers,
-                      const struct GNUNET_HashCode **ports)
+GNUNET_CADET_TEST_ruN (const char *testname,
+                       const char *cfgfile,
+                       unsigned int num_peers,
+                       GNUNET_CADET_TEST_AppMain tmain,
+                       void *tmain_cls,
+                       GNUNET_CADET_ConnectEventHandler connects,
+                       GNUNET_CADET_WindowSizeEventHandler window_changes,
+                       GNUNET_CADET_DisconnectEventHandler disconnects,
+                       struct GNUNET_MQ_MessageHandler *handlers,
+                       const struct GNUNET_HashCode **ports)
 {
   struct GNUNET_CADET_TEST_Context *ctx;
 
   ctx = GNUNET_new (struct GNUNET_CADET_TEST_Context);
   ctx->num_peers = num_peers;
-  ctx->ops = GNUNET_malloc (num_peers * sizeof (struct GNUNET_TESTBED_Operation *));
-  ctx->cadetes = GNUNET_malloc (num_peers * sizeof (struct GNUNET_CADET_Handle *));
+  ctx->ops = GNUNET_new_array (num_peers, struct GNUNET_TESTBED_Operation *);
+  ctx->cadets = GNUNET_new_array (num_peers, struct GNUNET_CADET_Handle *);
   ctx->app_main = tmain;
   ctx->app_main_cls = tmain_cls;
-  ctx->new_channel = new_channel;
-  ctx->cleaner = cleaner;
-  ctx->handlers = handlers;
+  ctx->connects = connects;
+  ctx->window_changes = window_changes;
+  ctx->disconnects = disconnects;
+  ctx->handlers = GNUNET_MQ_copy_handlers (handlers);
   ctx->ports = ports;
+  ctx->port_count = 0;
+  while (NULL != ctx->ports[ctx->port_count])
+    ctx->port_count++;
+
   GNUNET_TESTBED_test_run (testname,
-                           cfgname,
+                           cfgfile,
                            num_peers,
                            0LL, NULL, NULL,
                            &cadet_test_run, ctx);
diff --git a/src/cadet/cadet_test_lib.h b/src/cadet/cadet_test_lib.h
index 464977d42..4b3a6b18d 100644
--- a/src/cadet/cadet_test_lib.h
+++ b/src/cadet/cadet_test_lib.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012,2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -49,41 +49,41 @@ struct GNUNET_CADET_TEST_Context;
  * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
  * @param num_peers Number of peers that are running.
  * @param peers Array of peers.
- * @param cadetes Handle to each of the CADETs of the peers.
+ * @param cadets Handle to each of the CADETs of the peers.
  */
 typedef void (*GNUNET_CADET_TEST_AppMain) (void *cls,
                                           struct GNUNET_CADET_TEST_Context *ctx,
                                           unsigned int num_peers,
                                           struct GNUNET_TESTBED_Peer **peers,
-                                          struct GNUNET_CADET_Handle **cadetes);
+                                          struct GNUNET_CADET_Handle **cadets);
 
 
 /**
- * Run a test using the given name, configuration file and number of
- * peers.
- * All cadet callbacks will receive the peer number as the closure.
+ * Run a test using the given name, configuration file and number of peers.
+ * All cadet callbacks will receive the peer number (long) as the closure.
  *
  * @param testname Name of the test (for logging).
- * @param cfgname Name of the configuration file.
+ * @param cfgfile Name of the configuration file.
  * @param num_peers Number of peers to start.
  * @param tmain Main function to run once the testbed is ready.
- * @param tmain_cls Closure for 'tmain'.
- * @param new_channel Handler for incoming tunnels.
- * @param cleaner Cleaner for destroyed incoming tunnels.
+ * @param tmain_cls Closure for @a tmain.
+ * @param connects Handler for incoming channels.
+ * @param window_changes Handler for the window size change notification.
+ * @param disconnects Cleaner for destroyed incoming channels.
  * @param handlers Message handlers.
  * @param ports Ports the peers offer, NULL-terminated.
  */
 void
-GNUNET_CADET_TEST_run (const char *testname,
-                      const char *cfgname,
-                      unsigned int num_peers,
-                      GNUNET_CADET_TEST_AppMain tmain,
-                      void *tmain_cls,
-                      GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      struct GNUNET_CADET_MessageHandler* handlers,
-                      const struct GNUNET_HashCode **ports);
-
+GNUNET_CADET_TEST_ruN (const char *testname,
+                       const char *cfgfile,
+                       unsigned int num_peers,
+                       GNUNET_CADET_TEST_AppMain tmain,
+                       void *tmain_cls,
+                       GNUNET_CADET_ConnectEventHandler connects,
+                       GNUNET_CADET_WindowSizeEventHandler window_changes,
+                       GNUNET_CADET_DisconnectEventHandler disconnects,
+                       struct GNUNET_MQ_MessageHandler *handlers,
+                       const struct GNUNET_HashCode **ports);
 
 /**
  * Clean up the testbed.
diff --git a/src/cadet/desirability_table.c b/src/cadet/desirability_table.c
new file mode 100644
index 000000000..21ec3e388
--- /dev/null
+++ b/src/cadet/desirability_table.c
@@ -0,0 +1,35 @@
+/* This file is in the public domain. */
+
+/**
+ * @brief Program to simulate results from #GCP_get_desirability_of_path()
+ * for various plausible inputs.
+ * @author Christian Grothoff
+ */
+#include <stdio.h>
+
+int
+main ()
+{
+  for (unsigned int num_alts=1; num_alts<10; num_alts++)
+    for (unsigned int off=0; off<10; off++)
+      for (double delta=-(int) off;delta<=5;delta += 0.25)
+      {
+        double weight_alts;
+
+        if (delta <= - 1.0)
+          weight_alts = - 1.0 * num_alts / delta; /* discount alternative paths */
+        else if (delta >= 1.0)
+          weight_alts = 1.0 * num_alts * delta; /* overcount alternative paths */
+        else
+          weight_alts = 1.0 * num_alts; /* count alternative paths normally */
+
+        fprintf (stderr,
+                 "Paths: %u  Offset: %u  Delta: %5.2f  SCORE: %f\n",
+                 num_alts,
+                 off,
+                 delta,
+                 ((off + 1.0) / (weight_alts * weight_alts)));
+      }
+
+
+}
diff --git a/src/cadet/gnunet-cadet-profiler.c b/src/cadet/gnunet-cadet-profiler.c
index d688dc60b..15da05b6d 100644
--- a/src/cadet/gnunet-cadet-profiler.c
+++ b/src/cadet/gnunet-cadet-profiler.c
@@ -778,7 +778,9 @@ pong_handler (void *cls, struct GNUNET_CADET_Channel *channel,
   latency = GNUNET_TIME_absolute_get_duration (send_time);
   r = ntohl (msg->round_number);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "%u <- %u (%u) latency: %s\n",
-              get_index (peer), get_index (peer->dest), ntohl (msg->counter),
+              get_index (peer),
+              get_index (peer->dest),
+              (uint32_t) ntohl (msg->counter),
               GNUNET_STRINGS_relative_time_to_string (latency, GNUNET_NO));
 
   /* Online variance calculation */
diff --git a/src/cadet/gnunet-cadet.c b/src/cadet/gnunet-cadet.c
index 80010ec54..675e7faf0 100644
--- a/src/cadet/gnunet-cadet.c
+++ b/src/cadet/gnunet-cadet.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,6 +22,7 @@
  * @file cadet/gnunet-cadet.c
  * @brief Print information about cadet tunnels and peers.
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
@@ -30,11 +31,6 @@
 
 
 /**
- * Option -m.
- */
-static int monitor_mode;
-
-/**
  * Option -P.
  */
 static int request_peers;
@@ -100,11 +96,6 @@ static char *target_id;
 static char *target_port = "default";
 
 /**
- * Data pending in netcat mode.
- */
-static size_t data_size;
-
-/**
  * Cadet handle.
  */
 static struct GNUNET_CADET_Handle *mh;
@@ -115,11 +106,6 @@ static struct GNUNET_CADET_Handle *mh;
 static struct GNUNET_CADET_Channel *ch;
 
 /**
- * Transmit handle.
- */
-static struct GNUNET_CADET_TransmitHandle *th;
-
-/**
  * HashCode of the given port string
  */
 static struct GNUNET_HashCode porthash;
@@ -130,11 +116,6 @@ static struct GNUNET_HashCode porthash;
 struct GNUNET_CADET_Port *lp;
 
 /**
- * Shutdown task handle.
- */
-static struct GNUNET_SCHEDULER_Task *sd;
-
-/**
  * Task for reading from stdin.
  */
 static struct GNUNET_SCHEDULER_Task *rd_task;
@@ -145,6 +126,9 @@ static struct GNUNET_SCHEDULER_Task *rd_task;
 static struct GNUNET_SCHEDULER_Task *job;
 
 
+/**
+ * Wait for input on STDIO and send it out over the #ch.
+ */
 static void
 listen_stdio (void);
 
@@ -214,22 +198,11 @@ shutdown_task (void *cls)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Shutdown\n");
-  if (NULL != th)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
-    th = NULL;
-  }
   if (NULL != ch)
   {
     GNUNET_CADET_channel_destroy (ch);
     ch = NULL;
   }
-  else if (NULL != target_id) {
-    // FIXME: would be nicer to have proper NACK support from cadet_api
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Connection refused to %s\n",
-                target_id);
-  }
   if (NULL != mh)
   {
     GNUNET_CADET_disconnect (mh);
@@ -254,42 +227,38 @@ shutdown_task (void *cls)
 
 
 /**
- * Function called to notify a client about the connection
- * begin ready to queue more data.  "buf" will be
- * NULL and "size" zero if the connection was closed for
- * writing in the meantime.
- *
- * FIXME
+ * Task run in stdio mode, after some data is available at stdin.
  *
- * @param cls closure
- * @param size number of bytes available in buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
+ * @param cls Closure (unused).
  */
-static size_t
-data_ready (void *cls, size_t size, void *buf)
+static void
+read_stdio (void *cls)
 {
+  struct GNUNET_MQ_Envelope *env;
   struct GNUNET_MessageHeader *msg;
-  size_t total_size;
-
-  th = NULL;
+  char buf[60000];
+  ssize_t data_size;
 
-  if (NULL == buf || 0 == size)
+  rd_task = NULL;
+  data_size = read (0,
+                    buf,
+                    60000);
+  if (data_size < 1)
   {
     GNUNET_SCHEDULER_shutdown();
-    return 0;
+    return;
   }
-
-  total_size = data_size + sizeof (struct GNUNET_MessageHeader);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "sending %u bytes\n",
+              "Read %u bytes from stdio\n",
               (unsigned int) data_size);
-  GNUNET_assert (size >= total_size);
-
-  msg = buf;
-  msg->size = htons (total_size);
-  msg->type = htons (GNUNET_MESSAGE_TYPE_CADET_CLI);
-  GNUNET_memcpy (&msg[1], cls, data_size);
+  env = GNUNET_MQ_msg_extra (msg,
+                             data_size,
+                             GNUNET_MESSAGE_TYPE_CADET_CLI);
+  GNUNET_memcpy (&msg[1],
+                 buf,
+                 data_size);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (ch),
+                  env);
   if (GNUNET_NO == echo)
   {
     listen_stdio ();
@@ -298,53 +267,27 @@ data_ready (void *cls, size_t size, void *buf)
   {
     echo_time = GNUNET_TIME_absolute_get ();
   }
-
-  return total_size;
 }
 
 
 /**
- * Task run in stdio mode, after some data is available at stdin.
- *
- * @param cls Closure (unused).
+ * Wait for input on STDIO and send it out over the #ch.
  */
 static void
-read_stdio (void *cls)
-{
-  static char buf[60000];
-
-  data_size = read (0, buf, 60000);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "stdio read %u bytes\n",
-              (unsigned int) data_size);
-  if (data_size < 1)
-  {
-    GNUNET_SCHEDULER_shutdown();
-    return;
-  }
-  GNUNET_assert (NULL == th);
-  th = GNUNET_CADET_notify_transmit_ready (ch, GNUNET_NO,
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           sizeof (struct GNUNET_MessageHeader)
-                                           + data_size,
-                                           &data_ready, buf);
-}
-
-
-/**
- * Start listening to stdin
- */
-static void
-listen_stdio (void)
+listen_stdio ()
 {
   struct GNUNET_NETWORK_FDSet *rs;
 
+  /* FIXME: why use 'rs' here, seems overly complicated... */
   rs = GNUNET_NETWORK_fdset_create ();
-  GNUNET_NETWORK_fdset_set_native (rs, 0);
+  GNUNET_NETWORK_fdset_set_native (rs,
+                                   0); /* STDIN */
   rd_task = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
-                                         GNUNET_TIME_UNIT_FOREVER_REL,
-                                         rs, NULL,
-                                         &read_stdio, NULL);
+                                         GNUNET_TIME_UNIT_FOREVER_REL,
+                                         rs,
+                                         NULL,
+                                         &read_stdio,
+                                         NULL);
   GNUNET_NETWORK_fdset_destroy (rs);
 }
 
@@ -355,32 +298,17 @@ listen_stdio (void)
  *
  * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
+ * @param cls closure
  * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
  */
 static void
 channel_ended (void *cls,
-               const struct GNUNET_CADET_Channel *channel,
-               void *channel_ctx)
+               const struct GNUNET_CADET_Channel *channel)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Channel ended!\n");
-  if (channel != ch)
-  {
-    GNUNET_break (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "ended: %p, expected: %p\n", channel, ch);
-  }
-  else
-  {
-    ch = NULL;
-  }
-  if (NULL != th)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
-    th = NULL;
-  }
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Channel ended!\n");
+  GNUNET_assert (channel == ch);
+  ch = NULL;
   GNUNET_SCHEDULER_shutdown ();
 }
 
@@ -397,65 +325,27 @@ channel_ended (void *cls,
  * @param cls closure
  * @param channel new handle to the channel
  * @param initiator peer that started the channel
- * @param port Port this channel is for.
- * @param options CadetOption flag field, with all active option bits set to 1.
- *
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
+ * @return initial channel context for the channel, we use @a channel
  */
 static void *
 channel_incoming (void *cls,
                   struct GNUNET_CADET_Channel *channel,
-                  const struct GNUNET_PeerIdentity *initiator,
-                  const struct GNUNET_HashCode *port,
-                  enum GNUNET_CADET_ChannelOption options)
+                  const struct GNUNET_PeerIdentity *initiator)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-              "Connected from %s\n",
+              "Incoming connection from %s\n",
               GNUNET_i2s_full (initiator));
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Incoming channel %p on port %s\n",
-              channel, GNUNET_h2s (port));
-  if (NULL != ch)
-  {
-    GNUNET_break (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "A channel already exists (%p)\n", ch);
-    /*
-     * From now on multiple channels will be sending data to us
-     * making the service of this command unpredictable in its
-     * current implementation. So for now let's just bail out.
-     */
-    GNUNET_SCHEDULER_shutdown();
-    return NULL;
-  }
-  if (NULL == listen_port)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Not listening to channels\n");
-    return NULL;
-  }
-#if 0
-  // Closing the listen port currently breaks open connections.
-  // Is this an intentional departure from POSIX socket behavior?
-  //
-  if (NULL != lp) {
-    /* Now that we have our circuit up and running, let's not
-     * get confused by further incoming connect requests.
-     */
-    GNUNET_CADET_close_port (lp);
-    lp = NULL;
-  }
-#endif
+  GNUNET_assert (NULL == ch);
+  GNUNET_assert (NULL != lp);
+  GNUNET_CADET_close_port (lp);
+  lp = NULL;
   ch = channel;
   if (GNUNET_NO == echo)
-  {
     listen_stdio ();
-    return NULL;
-  }
-  data_size = 0;
-  return NULL;
+  return channel;
 }
 
+
 /**
  * @brief Send an echo request to the remote peer.
  *
@@ -464,13 +354,16 @@ channel_incoming (void *cls,
 static void
 send_echo (void *cls)
 {
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+
+  echo_task = NULL;
   if (NULL == ch)
     return;
-  GNUNET_assert (NULL == th);
-  th = GNUNET_CADET_notify_transmit_ready (ch, GNUNET_NO,
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           sizeof (struct GNUNET_MessageHeader),
-                                           &data_ready, NULL);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_CLI);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (ch),
+                  env);
 }
 
 
@@ -483,44 +376,23 @@ static void
 request_dump (void *cls)
 {
   GNUNET_CADET_request_dump (mh);
-  GNUNET_SCHEDULER_cancel (sd);
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                &shutdown_task, NULL);
+  GNUNET_SCHEDULER_shutdown ();
 }
 
 
 /**
- * Call CADET's monitor API, get info of one connection.
+ * Check data message sanity. Does nothing so far (all messages are OK).
  *
  * @param cls Closure (unused).
+ * @param message The message to check.
+ * @return #GNUNET_OK to keep the channel open,
+ *         #GNUNET_SYSERR to close it (signal serious error).
  */
-static void
-create_channel (void *cls)
+static int
+check_data (void *cls,
+            const struct GNUNET_MessageHeader *message)
 {
-  struct GNUNET_PeerIdentity pid;
-  enum GNUNET_CADET_ChannelOption opt;
-
-  GNUNET_assert (NULL == ch);
-
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_eddsa_public_key_from_string (target_id,
-                                                  strlen (target_id),
-                                                  &pid.public_key))
-  {
-    FPRINTF (stderr,
-             _("Invalid target `%s'\n"),
-             target_id);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connecting to `%s'\n", target_id);
-  opt = GNUNET_CADET_OPTION_DEFAULT | GNUNET_CADET_OPTION_RELIABLE;
-  GNUNET_CRYPTO_hash (target_port, strlen(target_port), &porthash);
-  ch = GNUNET_CADET_channel_create (mh, NULL, &pid, &porthash, opt);
-  if (GNUNET_NO == echo)
-    listen_stdio ();
-  else
-    echo_task = GNUNET_SCHEDULER_add_now (&send_echo, NULL);
+  return GNUNET_OK; /* all is well-formed */
 }
 
 
@@ -531,42 +403,36 @@ create_channel (void *cls)
  * in order to receive the next message. This doesn't need to be immediate:
  * can be delayed if some processing is done on the message.
  *
- * @param cls Closure (set from #GNUNET_CADET_connect).
- * @param channel Connection to the other end.
- * @param channel_ctx Place to store local state associated with the channel.
+ * @param cls NULL
  * @param message The actual message.
- * @return #GNUNET_OK to keep the channel open,
- *         #GNUNET_SYSERR to close it (signal serious error).
  */
-static int
-data_callback (void *cls,
-       struct GNUNET_CADET_Channel *channel,
-       void **channel_ctx,
-       const struct GNUNET_MessageHeader *message)
+static void
+handle_data (void *cls,
+             const struct GNUNET_MessageHeader *message)
 {
+  size_t payload_size = ntohs (message->size) - sizeof (*message);
   uint16_t len;
   ssize_t done;
   uint16_t off;
   const char *buf;
-  GNUNET_break (ch == channel);
-  GNUNET_CADET_receive_done (channel);
 
+  GNUNET_CADET_receive_done (ch);
   if (GNUNET_YES == echo)
   {
     if (NULL != listen_port)
     {
-      /* Just listening to echo incoming messages*/
-      if (NULL != th)
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Last echo reply not yet sent, dropping current reply.\n");
-        return GNUNET_OK;
-      }
-      th = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                               GNUNET_TIME_UNIT_FOREVER_REL,
-                                               sizeof (struct GNUNET_MessageHeader),
-                                               &data_ready, NULL);
-      return GNUNET_OK;
+      struct GNUNET_MQ_Envelope *env;
+      struct GNUNET_MessageHeader *msg;
+
+      env = GNUNET_MQ_msg_extra (msg,
+                                 payload_size,
+                                 GNUNET_MESSAGE_TYPE_CADET_CLI);
+      GNUNET_memcpy (&msg[1],
+                     &message[1],
+                     payload_size);
+      GNUNET_MQ_send (GNUNET_CADET_get_mq (ch),
+                      env);
+      return;
     }
     else
     {
@@ -574,30 +440,37 @@ data_callback (void *cls,
 
       latency = GNUNET_TIME_absolute_get_duration (echo_time);
       echo_time = GNUNET_TIME_UNIT_FOREVER_ABS;
-      FPRINTF (stdout, "time: %s\n",
-               GNUNET_STRINGS_relative_time_to_string (latency, GNUNET_NO));
+      GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+                  "time: %s\n",
+                  GNUNET_STRINGS_relative_time_to_string (latency,
+                                                          GNUNET_NO));
       echo_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                                &send_echo, NULL);
+                                                &send_echo,
+                                                NULL);
     }
   }
 
   len = ntohs (message->size) - sizeof (*message);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got %u bytes\n", len);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Got %u bytes\n",
+              len);
   buf = (const char *) &message[1];
   off = 0;
   while (off < len)
   {
-    done = write (1, &buf[off], len - off);
+    done = write (1,
+                  &buf[off],
+                  len - off);
     if (done <= 0)
     {
       if (-1 == done)
         GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
                              "write");
-      return GNUNET_SYSERR;
+      GNUNET_SCHEDULER_shutdown ();
+      return;
     }
     off += done;
   }
-  return GNUNET_OK;
 }
 
 
@@ -623,16 +496,17 @@ peers_callback (void *cls,
 {
   if (NULL == peer)
   {
-    if (GNUNET_YES != monitor_mode)
-    {
-      GNUNET_SCHEDULER_shutdown();
-    }
+    GNUNET_SCHEDULER_shutdown();
     return;
   }
-  FPRINTF (stdout, "%s tunnel: %c, paths: %u\n",
-           GNUNET_i2s_full (peer), tunnel ? 'Y' : 'N', n_paths);
+  FPRINTF (stdout,
+           "%s tunnel: %c, paths: %u\n",
+           GNUNET_i2s_full (peer),
+           tunnel ? 'Y' : 'N',
+           n_paths);
 }
 
+
 /**
  * Method called to retrieve information about a specific peer
  * known to the service.
@@ -652,19 +526,26 @@ peer_callback (void *cls,
                int tunnel,
                int neighbor,
                unsigned int n_paths,
-               struct GNUNET_PeerIdentity *paths)
+               const struct GNUNET_PeerIdentity *paths)
 {
   unsigned int i;
-  struct GNUNET_PeerIdentity *p;
+  const struct GNUNET_PeerIdentity *p;
 
-  FPRINTF (stdout, "%s [TUNNEL: %s, NEIGHBOR: %s, PATHS: %u]\n",
+  FPRINTF (stdout,
+           "%s [TUNNEL: %s, NEIGHBOR: %s, PATHS: %u]\n",
            GNUNET_i2s_full (peer),
-           tunnel ? "Y" : "N", neighbor ? "Y" : "N", n_paths);
+           tunnel ? "Y" : "N",
+           neighbor ? "Y" : "N",
+           n_paths);
   p = paths;
   for (i = 0; i < n_paths && NULL != p;)
   {
-    FPRINTF (stdout, "%s ", GNUNET_i2s (p));
-    if (0 == memcmp (p, peer, sizeof (*p)))
+    FPRINTF (stdout,
+             "%s ",
+             GNUNET_i2s (p));
+    if (0 == memcmp (p,
+                     peer,
+                     sizeof (*p)))
     {
       FPRINTF (stdout, "\n");
       i++;
@@ -696,16 +577,16 @@ tunnels_callback (void *cls,
 {
   if (NULL == peer)
   {
-    if (GNUNET_YES != monitor_mode)
-    {
-      GNUNET_SCHEDULER_shutdown();
-    }
+    GNUNET_SCHEDULER_shutdown();
     return;
   }
-  FPRINTF (stdout, "%s [ENC: %s, CON: %s] CHs: %u, CONNs: %u\n",
+  FPRINTF (stdout,
+           "%s [ENC: %s, CON: %s] CHs: %u, CONNs: %u\n",
            GNUNET_i2s_full (peer),
-           enc_2s (estate), conn_2s (cstate),
-           channels, connections);
+           enc_2s (estate),
+           conn_2s (cstate),
+           channels,
+           connections);
 }
 
 
@@ -746,11 +627,7 @@ tunnel_callback (void *cls,
     FPRINTF (stdout, "\tencryption state: %s\n", enc_2s (estate));
     FPRINTF (stdout, "\tconnection state: %s\n", conn_2s (cstate));
   }
-  if (GNUNET_YES != monitor_mode)
-  {
-    GNUNET_SCHEDULER_shutdown ();
-  }
-  return;
+  GNUNET_SCHEDULER_shutdown ();
 }
 
 
@@ -874,93 +751,158 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
-  static const struct GNUNET_CADET_MessageHandler handlers[] = {
-    {&data_callback, GNUNET_MESSAGE_TYPE_CADET_CLI, 0},
-    {NULL, 0, 0} /* FIXME add option to monitor msg types */
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (data,
+                           GNUNET_MESSAGE_TYPE_CADET_CLI,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_handler_end ()
   };
 
   /* FIXME add option to monitor apps */
 
   target_id = args[0];
-  if (target_id && args[1]) target_port = args[1];
+  if (target_id && args[1])
+    target_port = args[1];
 
   if ( (0 != (request_peers | request_tunnels)
-        || 0 != monitor_mode
         || NULL != tunnel_id
         || NULL != conn_id
         || NULL != channel_id)
        && target_id != NULL)
   {
     FPRINTF (stderr,
-             _("You must NOT give a TARGET "
-               "when using 'request all' options\n"));
+             _("Extra arguments are not applicable "
+               "in combination with this option.\n"));
     return;
   }
 
   if (GNUNET_YES == dump)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "requesting debug dump\n");
-    GNUNET_SCHEDULER_add_now (&request_dump, NULL);
-  }
-  else if (NULL != target_id)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Creating channel to %s\n",
-                target_id);
-    GNUNET_SCHEDULER_add_now (&create_channel, NULL);
+                "Requesting debug dump\n");
+    job = GNUNET_SCHEDULER_add_now (&request_dump,
+                                    NULL);
   }
   else if (NULL != peer_id)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show peer\n");
-    job = GNUNET_SCHEDULER_add_now (&show_peer, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show peer\n");
+    job = GNUNET_SCHEDULER_add_now (&show_peer,
+                                    NULL);
   }
   else if (NULL != tunnel_id)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show tunnel\n");
-    job = GNUNET_SCHEDULER_add_now (&show_tunnel, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show tunnel\n");
+    job = GNUNET_SCHEDULER_add_now (&show_tunnel,
+                                    NULL);
   }
   else if (NULL != channel_id)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show channel\n");
-    job = GNUNET_SCHEDULER_add_now (&show_channel, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show channel\n");
+    job = GNUNET_SCHEDULER_add_now (&show_channel,
+                                    NULL);
   }
   else if (NULL != conn_id)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show connection\n");
-    job = GNUNET_SCHEDULER_add_now (&show_connection, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show connection\n");
+    job = GNUNET_SCHEDULER_add_now (&show_connection,
+                                    NULL);
   }
   else if (GNUNET_YES == request_peers)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show all peers\n");
-    job = GNUNET_SCHEDULER_add_now (&get_peers, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show all peers\n");
+    job = GNUNET_SCHEDULER_add_now (&get_peers,
+                                    NULL);
   }
   else if (GNUNET_YES == request_tunnels)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Show all tunnels\n");
-    job = GNUNET_SCHEDULER_add_now (&get_tunnels, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Show all tunnels\n");
+    job = GNUNET_SCHEDULER_add_now (&get_tunnels,
+                                    NULL);
   }
-  else if (NULL == listen_port)
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Connecting to CADET service\n");
+  mh = GNUNET_CADET_connect (cfg);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  if (NULL == mh)
   {
-    FPRINTF (stderr, "No action requested\n");
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connecting to CADET service\n");
-  mh = GNUNET_CADET_connect (cfg,
-                            NULL, /* cls */
-                            &channel_ended, /* cleaner */
-                            handlers);
-  if (NULL == mh)
-    GNUNET_SCHEDULER_add_now (&shutdown_task, NULL);
-  else
-    sd = GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-
   if (NULL != listen_port)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Opening CADET listen port\n");
-    GNUNET_CRYPTO_hash (listen_port, strlen(listen_port), &porthash);
-    lp = GNUNET_CADET_open_port (mh, &porthash, &channel_incoming, NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Opening CADET listen port\n");
+    GNUNET_CRYPTO_hash (listen_port,
+                        strlen (listen_port),
+                        &porthash);
+    lp = GNUNET_CADET_open_port (mh,
+                                 &porthash,
+                                 &channel_incoming,
+                                 NULL,
+                                 NULL /* window changes */,
+                                 &channel_ended,
+                                 handlers);
+  }
+  if (NULL != target_id)
+  {
+    struct GNUNET_PeerIdentity pid;
+    enum GNUNET_CADET_ChannelOption opt;
+
+    if (GNUNET_OK !=
+        GNUNET_CRYPTO_eddsa_public_key_from_string (target_id,
+                                                    strlen (target_id),
+                                                    &pid.public_key))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+                  _("Invalid target `%s'\n"),
+                  target_id);
+      GNUNET_SCHEDULER_shutdown ();
+      return;
+    }
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Connecting to `%s:%s'\n",
+                target_id,
+                target_port);
+    opt = GNUNET_CADET_OPTION_DEFAULT | GNUNET_CADET_OPTION_RELIABLE;
+    GNUNET_CRYPTO_hash (target_port,
+                        strlen(target_port),
+                        &porthash);
+    ch = GNUNET_CADET_channel_create (mh,
+                                      NULL,
+                                      &pid,
+                                      &porthash,
+                                      opt,
+                                      NULL /* window changes */,
+                                      &channel_ended,
+                                      handlers);
+    if (GNUNET_YES == echo)
+    {
+      echo_task = GNUNET_SCHEDULER_add_now (&send_echo,
+                                            NULL);
+    }
+    else
+    {
+      listen_stdio ();
+    }
+  }
+
+  if ( (NULL == lp) &&
+       (NULL == job) &&
+       (NULL == ch) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+                _("No action requested\n"));
+    GNUNET_SCHEDULER_shutdown ();
+    return;
   }
 }
 
@@ -973,51 +915,70 @@ run (void *cls,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
   int res;
-  const char helpstr[] = "Create channels and retreive info about cadets status.";
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-//     {'a', "channel", "TUNNEL_ID:CHANNEL_ID",
-//      gettext_noop ("provide information about a particular channel"),
-//      GNUNET_YES, &GNUNET_GETOPT_set_string, &channel_id},
-    {'C', "connection", "CONNECTION_ID",
-     gettext_noop ("provide information about a particular connection"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &conn_id},
-    {'e', "echo", NULL,
-     gettext_noop ("activate echo mode"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &echo},
-    {'d', "dump", NULL,
-     gettext_noop ("dump debug information to STDERR"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &dump},
-//     {'m', "monitor", NULL,
-//      gettext_noop ("provide information about all events (continuously)"),
-//      GNUNET_NO, &GNUNET_GETOPT_set_one, &monitor_mode},
-    {'o', "open-port", NULL,
-     gettext_noop ("port to listen to"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &listen_port},
-    {'p', "peer", "PEER_ID",
-     gettext_noop ("provide information about a patricular peer"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &peer_id},
-    {'P', "peers", NULL,
-      gettext_noop ("provide information about all peers"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &request_peers},
-    {'t', "tunnel", "TUNNEL_ID",
-     gettext_noop ("provide information about a particular tunnel"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &tunnel_id},
-    {'T', "tunnels", NULL,
-     gettext_noop ("provide information about all tunnels"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &request_tunnels},
+  const char helpstr[] = "Create tunnels and retrieve info about CADET's status.";
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    /* I would use the terminology 'circuit' here...  --lynX */
+    GNUNET_GETOPT_option_string ('C',
+                                 "connection",
+                                 "CONNECTION_ID",
+                                 gettext_noop ("Provide information about a particular connection"),
+                                 &conn_id),
+
+    GNUNET_GETOPT_option_flag ('e',
+                                  "echo",
+                                  gettext_noop ("Activate echo mode"),
+                                  &echo), 
+
+    GNUNET_GETOPT_option_flag ('d',
+                                  "dump",
+                                  gettext_noop ("Dump debug information to STDERR"),
+                                  &dump),
+
+    GNUNET_GETOPT_option_string ('o',
+                                 "open-port",
+                                 "SHARED_SECRET",
+                                 gettext_noop ("Listen for connections using a shared secret among sender and recipient"),
+                                 &listen_port),
+
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "peer",
+                                 "PEER_ID",
+                                 gettext_noop ("Provide information about a patricular peer"),
+                                 &peer_id),
+
+
+    GNUNET_GETOPT_option_flag ('P',
+                                  "peers",
+                                  gettext_noop ("Provide information about all peers"),
+                                  &request_peers),
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "tunnel",
+                                 "TUNNEL_ID",
+                                 gettext_noop ("Provide information about a particular tunnel"),
+                                 &tunnel_id),
+
+
+    GNUNET_GETOPT_option_flag ('T',
+                                  "tunnels",
+                                  gettext_noop ("Provide information about all tunnels"),
+                                  &request_tunnels),
 
     GNUNET_GETOPT_OPTION_END
   };
 
-  monitor_mode = GNUNET_NO;
-
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
 
-  res = GNUNET_PROGRAM_run (argc, argv, "gnunet-cadet (OPTIONS | TARGET PORT)",
+  res = GNUNET_PROGRAM_run (argc, argv,
+                            "gnunet-cadet (OPTIONS | PEER_ID SHARED_SECRET)",
                             gettext_noop (helpstr),
                             options, &run, NULL);
 
@@ -1025,8 +986,7 @@ main (int argc, char *const *argv)
 
   if (GNUNET_OK == res)
     return 0;
-  else
-    return 1;
+  return 1;
 }
 
 /* end of gnunet-cadet.c */
diff --git a/src/cadet/gnunet-service-cadet-new.c b/src/cadet/gnunet-service-cadet-new.c
deleted file mode 100644
index 97489f3fd..000000000
--- a/src/cadet/gnunet-service-cadet-new.c
+++ /dev/null
@@ -1,1428 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new.c
- * @brief GNUnet CADET service with encryption
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * Dictionary:
- * - peer: other cadet instance. If there is direct connection it's a neighbor.
- * - path: series of directly connected peer from one peer to another.
- * - connection: path which is being used in a tunnel.
- * - tunnel: encrypted connection to a peer, neighbor or not.
- * - channel: logical link between two clients, on the same or different peers.
- *            have properties like reliability.
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_core.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-#define LOG(level, ...) GNUNET_log (level,__VA_ARGS__)
-
-
-/**
- * Struct containing information about a client of the service
- */
-struct CadetClient
-{
-  /**
-   * Linked list next
-   */
-  struct CadetClient *next;
-
-  /**
-   * Linked list prev
-   */
-  struct CadetClient *prev;
-
-  /**
-   * Tunnels that belong to this client, indexed by local id,
-   * value is a `struct CadetChannel`.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
-
-  /**
-   * Handle to communicate with the client
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Client handle.
-   */
-  struct GNUNET_SERVICE_Client *client;
-
-  /**
-   * Ports that this client has declared interest in.
-   * Indexed by port, contains *Client.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-  /**
-   * Channel ID to use for the next incoming channel for this client.
-   * Wraps around (in theory).
-   */
-  struct GNUNET_CADET_ClientChannelNumber next_ccn;
-
-  /**
-   * ID of the client, mainly for debug messages. Purely internal to this file.
-   */
-  unsigned int id;
-};
-
-/******************************************************************************/
-/***********************      GLOBAL VARIABLES     ****************************/
-/******************************************************************************/
-
-/****************************** Global variables ******************************/
-
-/**
- * Handle to our configuration.
- */
-const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Handle to the statistics service.
- */
-struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Handle to communicate with ATS.
- */
-struct GNUNET_ATS_ConnectivityHandle *ats_ch;
-
-/**
- * Local peer own ID.
- */
-struct GNUNET_PeerIdentity my_full_id;
-
-/**
- * Own private key.
- */
-struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
-
-/**
- * Signal that shutdown is happening: prevent recovery measures.
- */
-int shutting_down;
-
-/**
- * DLL with all the clients, head.
- */
-static struct CadetClient *clients_head;
-
-/**
- * DLL with all the clients, tail.
- */
-static struct CadetClient *clients_tail;
-
-/**
- * Next ID to assign to a client.
- */
-static unsigned int next_client_id;
-
-/**
- * All ports clients of this peer have opened.
- */
-struct GNUNET_CONTAINER_MultiHashMap *open_ports;
-
-/**
- * Map from ports to channels where the ports were closed at the
- * time we got the inbound connection.
- * Indexed by port, contains `struct CadetChannel`.
- */
-struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
-
-/**
- * Map from PIDs to `struct CadetPeer` entries.
- */
-struct GNUNET_CONTAINER_MultiPeerMap *peers;
-
-/**
- * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
- * hash codes to `struct CadetConnection` objects.
- */
-struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-/**
- * How many messages are needed to trigger an AXOLOTL ratchet advance.
- */
-unsigned long long ratchet_messages;
-
-/**
- * How long until we trigger a ratched advance due to time.
- */
-struct GNUNET_TIME_Relative ratchet_time;
-
-
-/**
- * Send a message to a client.
- *
- * @param c client to get the message
- * @param env envelope with the message
- */
-void
-GSC_send_to_client (struct CadetClient *c,
-                    struct GNUNET_MQ_Envelope *env)
-{
-  GNUNET_MQ_send (c->mq,
-                  env);
-}
-
-
-/**
- * Return identifier for a client as a string.
- *
- * @param c client to identify
- * @return string for debugging
- */
-const char *
-GSC_2s (struct CadetClient *c)
-{
-  static char buf[32];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Client(%u)",
-                   c->id);
-  return buf;
-}
-
-
-/**
- * Lookup channel of client @a c by @a ccn.
- *
- * @param c client to look in
- * @param ccn channel ID to look up
- * @return NULL if no such channel exists
- */
-static struct CadetChannel *
-lookup_channel (struct CadetClient *c,
-                struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  return GNUNET_CONTAINER_multihashmap32_get (c->channels,
-                                              ntohl (ccn.channel_of_client));
-}
-
-
-/**
- * Obtain the next LID to use for incoming connections to
- * the given client.
- *
- * @param c client handle
- */
-static struct GNUNET_CADET_ClientChannelNumber
-client_get_next_ccn (struct CadetClient *c)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn = c->next_ccn;
-
-  /* increment until we have a free one... */
-  while (NULL !=
-         lookup_channel (c,
-                         ccn))
-  {
-    ccn.channel_of_client
-      = htonl (1 + (ntohl (ccn.channel_of_client)));
-    if (ntohl (ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-      ccn.channel_of_client = htonl (0);
-  }
-  c->next_ccn.channel_of_client
-    = htonl (1 + (ntohl (ccn.channel_of_client)));
-  return ccn;
-}
-
-
-/**
- * Bind incoming channel to this client, and notify client about
- * incoming connection.  Caller is responsible for notifying the other
- * peer about our acceptance of the channel.
- *
- * @param c client to bind to
- * @param ch channel to be bound
- * @param dest peer that establishes the connection
- * @param port port number
- * @param options options
- * @return local channel number assigned to the new client
- */
-struct GNUNET_CADET_ClientChannelNumber
-GSC_bind (struct CadetClient *c,
-          struct CadetChannel *ch,
-          struct CadetPeer *dest,
-          const struct GNUNET_HashCode *port,
-          uint32_t options)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalChannelCreateMessage *cm;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn = client_get_next_ccn (c);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
-                                                      ntohl (ccn.channel_of_client),
-                                                      ch,
-                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Accepting incoming %s from %s on open port %s (%u), assigning ccn %X\n",
-       GCCH_2s (ch),
-       GCP_2s (dest),
-       GNUNET_h2s (port),
-       ntohl (options),
-       ntohl (ccn.channel_of_client));
-  /* notify local client about incoming connection! */
-  env = GNUNET_MQ_msg (cm,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  cm->ccn = ccn;
-  cm->port = *port;
-  cm->opt = htonl (options);
-  cm->peer = *GCP_get_id (dest);
-  GSC_send_to_client (c,
-                      env);
-  return ccn;
-}
-
-
-/**
- * Callback invoked on all peers to destroy all tunnels
- * that may still exist.
- *
- * @param cls NULL
- * @param pid identify of a peer
- * @param value a `struct CadetPeer` that may still have a tunnel
- * @return #GNUNET_OK (iterate over all entries)
- */
-static int
-destroy_tunnels_now (void *cls,
-                     const struct GNUNET_PeerIdentity *pid,
-                     void *value)
-{
-  struct CadetPeer *cp = value;
-  struct CadetTunnel *t = GCP_get_tunnel (cp,
-                                          GNUNET_NO);
-
-  if (NULL != t)
-    GCT_destroy_tunnel_now (t);
-  return GNUNET_OK;
-}
-
-
-/**
- * Callback invoked on all peers to destroy all tunnels
- * that may still exist.
- *
- * @param cls NULL
- * @param pid identify of a peer
- * @param value a `struct CadetPeer` that may still have a tunnel
- * @return #GNUNET_OK (iterate over all entries)
- */
-static int
-destroy_paths_now (void *cls,
-                   const struct GNUNET_PeerIdentity *pid,
-                   void *value)
-{
-  struct CadetPeer *cp = value;
-
-  GCP_drop_owned_paths (cp);
-  return GNUNET_OK;
-}
-
-
-/**
- * Shutdown everything once the clients have disconnected.
- */
-static void
-shutdown_rest ()
-{
-  if (NULL != stats)
-  {
-    GNUNET_STATISTICS_destroy (stats,
-                               GNUNET_NO);
-    stats = NULL;
-  }
-  if (NULL != open_ports)
-  {
-    GNUNET_CONTAINER_multihashmap_destroy (open_ports);
-    open_ports = NULL;
-  }
-  if (NULL != loose_channels)
-  {
-    GNUNET_CONTAINER_multihashmap_destroy (loose_channels);
-    loose_channels = NULL;
-  }
-  /* Destroy tunnels.  Note that all channels must be destroyed first! */
-  GCP_iterate_all (&destroy_tunnels_now,
-                   NULL);
-  /* All tunnels, channels, connections and CORE must be down before this point. */
-  GCP_iterate_all (&destroy_paths_now,
-                   NULL);
-  /* All paths, tunnels, channels, connections and CORE must be down before this point. */
-  GCP_destroy_all_peers ();
-  if (NULL != peers)
-  {
-    GNUNET_CONTAINER_multipeermap_destroy (peers);
-    peers = NULL;
-  }
-  if (NULL != connections)
-  {
-    GNUNET_CONTAINER_multishortmap_destroy (connections);
-    connections = NULL;
-  }
-  if (NULL != ats_ch)
-  {
-    GNUNET_ATS_connectivity_done (ats_ch);
-    ats_ch = NULL;
-  }
-  GCD_shutdown ();
-  GCH_shutdown ();
-  GNUNET_free_non_null (my_private_key);
-  my_private_key = NULL;
-}
-
-
-/**
- * Task run during shutdown.
- *
- * @param cls unused
- */
-static void
-shutdown_task (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Shutting down\n");
-  shutting_down = GNUNET_YES;
-  GCO_shutdown ();
-  if (NULL == clients_head)
-    shutdown_rest ();
-}
-
-
-/**
- * We had a remote connection @a value to port @a port before
- * client @a cls opened port @a port.  Bind them now.
- *
- * @param cls the `struct CadetClient`
- * @param port the port
- * @param value the `struct CadetChannel`
- * @return #GNUNET_YES (iterate over all such channels)
- */
-static int
-bind_loose_channel (void *cls,
-                    const struct GNUNET_HashCode *port,
-                    void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch = value;
-
-  GCCH_bind (ch,
-             c);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
-                                                       port,
-                                                       value));
-  return GNUNET_YES;
-}
-
-
-/**
- * Handle port open request.  Creates a mapping from the
- * port to the respective client and checks whether we have
- * loose channels trying to bind to the port.  If so, those
- * are bound.
- *
- * @param cls Identification of the client.
- * @param pmsg The actual message.
- */
-static void
-handle_port_open (void *cls,
-                  const struct GNUNET_CADET_PortMessage *pmsg)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Open port %s requested by %s\n",
-       GNUNET_h2s (&pmsg->port),
-       GSC_2s (c));
-  if (NULL == c->ports)
-    c->ports = GNUNET_CONTAINER_multihashmap_create (4,
-                                                      GNUNET_NO);
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (c->ports,
-                                         &pmsg->port,
-                                         c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  (void) GNUNET_CONTAINER_multihashmap_put (open_ports,
-                                            &pmsg->port,
-                                            c,
-                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-  GNUNET_CONTAINER_multihashmap_get_multiple (loose_channels,
-                                              &pmsg->port,
-                                              &bind_loose_channel,
-                                              c);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for port close requests.  Marks this port as closed
- * (unless of course we have another client with the same port
- * open).  Note that existing channels accepted on the port are
- * not affected.
- *
- * @param cls Identification of the client.
- * @param pmsg The actual message.
- */
-static void
-handle_port_close (void *cls,
-                   const struct GNUNET_CADET_PortMessage *pmsg)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing port %s as requested by %s\n",
-       GNUNET_h2s (&pmsg->port),
-       GSC_2s (c));
-  if (GNUNET_YES !=
-      GNUNET_CONTAINER_multihashmap_remove (c->ports,
-                                            &pmsg->port,
-                                            c))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
-                                                       &pmsg->port,
-                                                       c));
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for requests for us creating a new channel to another peer and port.
- *
- * @param cls Identification of the client.
- * @param tcm The actual message.
- */
-static void
-handle_channel_create (void *cls,
-                       const struct GNUNET_CADET_LocalChannelCreateMessage *tcm)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  if (ntohl (tcm->ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    /* Channel ID not in allowed range. */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  ch = lookup_channel (c,
-                       tcm->ccn);
-  if (NULL != ch)
-  {
-    /* Channel ID already in use. Not allowed. */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "New channel to %s at port %s requested by %s\n",
-       GNUNET_i2s (&tcm->peer),
-       GNUNET_h2s (&tcm->port),
-       GSC_2s (c));
-
-  /* Create channel */
-  ch = GCCH_channel_local_new (c,
-                               tcm->ccn,
-                               GCP_get (&tcm->peer,
-                                        GNUNET_YES),
-                               &tcm->port,
-                               ntohl (tcm->opt));
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
-                                                      ntohl (tcm->ccn.channel_of_client),
-                                                      ch,
-                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for requests of destroying an existing channel.
- *
- * @param cls client identification of the client
- * @param msg the actual message
- */
-static void
-handle_channel_destroy (void *cls,
-                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Client attempted to destroy unknown channel */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "%s is destroying %s\n",
-       GSC_2s(c),
-       GCCH_2s (ch));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         ntohl (msg->ccn.channel_of_client),
-                                                         ch));
-  GCCH_channel_local_destroy (ch,
-                              c,
-                              msg->ccn);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Check for client traffic data message is well-formed.
- *
- * @param cls identification of the client
- * @param msg the actual message
- * @return #GNUNET_OK if @a msg is OK, #GNUNET_SYSERR if not
- */
-static int
-check_local_data (void *cls,
-                  const struct GNUNET_CADET_LocalData *msg)
-{
-  size_t payload_size;
-  size_t payload_claimed_size;
-  const char *buf;
-  struct GNUNET_MessageHeader pa;
-
-  /* FIXME: what is the format we shall allow for @a msg?
-     ONE payload item or multiple? Seems current cadet_api
-     at least in theory allows more than one. Next-gen
-     cadet_api will likely no more, so we could then
-     simplify this mess again. */
-  /* Sanity check for message size */
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  buf = (const char *) &msg[1];
-  while (payload_size >= sizeof (struct GNUNET_MessageHeader))
-  {
-    /* need to memcpy() for alignment */
-    GNUNET_memcpy (&pa,
-                   buf,
-                   sizeof (pa));
-    payload_claimed_size = ntohs (pa.size);
-    if ( (payload_size < payload_claimed_size) ||
-         (payload_claimed_size < sizeof (struct GNUNET_MessageHeader)) ||
-         (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size) )
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Local data of %u total size had sub-message %u at %u with %u bytes\n",
-           ntohs (msg->header.size),
-           ntohs (pa.type),
-           (unsigned int) (buf - (const char *) &msg[1]),
-           (unsigned int) payload_claimed_size);
-      return GNUNET_SYSERR;
-    }
-    payload_size -= payload_claimed_size;
-    buf += payload_claimed_size;
-  }
-  if (0 != payload_size)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for client payload traffic to be send on a channel to
- * another peer.
- *
- * @param cls identification of the client
- * @param msg the actual message
- */
-static void
-handle_local_data (void *cls,
-                   const struct GNUNET_CADET_LocalData *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-  size_t payload_size;
-  const char *buf;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Channel does not exist! */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  buf = (const char *) &msg[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received %u bytes payload from %s for %s\n",
-       (unsigned int) payload_size,
-       GSC_2s (c),
-       GCCH_2s (ch));
-  if (GNUNET_OK !=
-      GCCH_handle_local_data (ch,
-                              msg->ccn,
-                              buf,
-                              payload_size))
-  {
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for client's ACKs for payload traffic.
- *
- * @param cls identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_local_ack (void *cls,
-                  const struct GNUNET_CADET_LocalAck *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Channel does not exist! */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got a local ACK from %s for %s\n",
-       GSC_2s(c),
-       GCCH_2s (ch));
-  GCCH_handle_local_ack (ch,
-                         msg->ccn);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all peers to send a monitoring client info about each peer.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_peers_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity *peer,
-                        void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoPeer *msg;
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  msg->destination = *peer;
-  msg->paths = htons (GCP_count_paths (p));
-  msg->tunnel = htons (NULL != GCP_get_tunnel (p,
-                                               GNUNET_NO));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO PEERS request.
- *
- * @param cls Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_peers (void *cls,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *reply;
-
-  GCP_iterate_all (&get_all_peers_iterator,
-                   c);
-  env = GNUNET_MQ_msg (reply,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all paths of a peer to build an InfoPeer message.
- * Message contains blocks of peers, first not included.
- *
- * @param cls message queue for transmission
- * @param path Path itself
- * @param off offset of the peer on @a path
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-static int
-path_info_iterator (void *cls,
-                    struct CadetPeerPath *path,
-                    unsigned int off)
-{
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *resp;
-  struct GNUNET_PeerIdentity *id;
-  uint16_t path_size;
-  unsigned int i;
-  unsigned int path_length;
-
-  path_length = GCPP_get_length (path);
-  path_size = sizeof (struct GNUNET_PeerIdentity) * (path_length - 1);
-  if (sizeof (*resp) + path_size > UINT16_MAX)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Path of %u entries is too long for info message\n",
-         path_length);
-    return GNUNET_YES;
-  }
-  env = GNUNET_MQ_msg_extra (resp,
-                             path_size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
-  id = (struct GNUNET_PeerIdentity *) &resp[1];
-
-  /* Don't copy first peer.  First peer is always the local one.  Last
-   * peer is always the destination (leave as 0, EOL).
-   */
-  for (i = 0; i < off; i++)
-    id[i] = *GCP_get_id (GCPP_get_peer_at_offset (path,
-                                                  i + 1));
-  GNUNET_MQ_send (mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's SHOW_PEER request.
- *
- * @param cls Identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_show_peer (void *cls,
-                  const struct GNUNET_CADET_LocalInfo *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *resp;
-
-  p = GCP_get (&msg->peer,
-               GNUNET_NO);
-  if (NULL != p)
-    GCP_iterate_paths (p,
-                       &path_info_iterator,
-                       c->mq);
-  /* Send message with 0/0 to indicate the end */
-  env = GNUNET_MQ_msg (resp,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER_END);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all tunnels to send a monitoring client info about each tunnel.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value a `struct CadetPeer`
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_tunnels_iterator (void *cls,
-                          const struct GNUNET_PeerIdentity *peer,
-                          void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoTunnel *msg;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL == t)
-    return GNUNET_YES;
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  msg->destination = *peer;
-  msg->channels = htonl (GCT_count_channels (t));
-  msg->connections = htonl (GCT_count_any_connections (t));
-  msg->cstate = htons (0);
-  msg->estate = htons ((uint16_t) GCT_get_estate (t));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS request.
- *
- * @param cls client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_info_tunnels (void *cls,
-                     const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *reply;
-
-  GCP_iterate_all (&get_all_tunnels_iterator,
-                   c);
-  env = GNUNET_MQ_msg (reply,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Update the message with information about the connection.
- *
- * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
- * @param c a connection about which we should store information in @a cls
- */
-static void
-iter_connection (void *cls,
-                 struct CadetConnection *c)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
-
-  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  h[msg->connections++] = *(GCC_get_id (c));
-}
-
-
-/**
- * Update the message with information about the channel.
- *
- * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
- * @param ch a channel about which we should store information in @a cls
- */
-static void
-iter_channel (void *cls,
-              struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  struct GNUNET_CADET_ChannelTunnelNumber *chn
-    = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
-
-  chn[msg->channels++] = GCCH_get_id (ch);
-}
-
-
-/**
- * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL request.
- *
- * @param cls Identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_info_tunnel (void *cls,
-                    const struct GNUNET_CADET_LocalInfo *msg)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoTunnel *resp;
-  struct CadetTunnel *t;
-  struct CadetPeer *p;
-  unsigned int ch_n;
-  unsigned int c_n;
-
-  p = GCP_get (&msg->peer,
-               GNUNET_NO);
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL == t)
-  {
-    /* We don't know the tunnel */
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_LocalInfoTunnel *warn;
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Tunnel to %s unknown\n",
-         GNUNET_i2s_full (&msg->peer));
-    env = GNUNET_MQ_msg (warn,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-    warn->destination = msg->peer;
-    GNUNET_MQ_send (c->mq,
-                    env);
-    GNUNET_SERVICE_client_continue (c->client);
-    return;
-  }
-
-  /* Initialize context */
-  ch_n = GCT_count_channels (t);
-  c_n = GCT_count_any_connections (t);
-  env = GNUNET_MQ_msg_extra (resp,
-                             c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier) +
-                             ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber),
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-  resp->destination = msg->peer;
-  /* Do not reorder! #iter_channel needs counters in HBO! */
-  GCT_iterate_connections (t,
-                           &iter_connection,
-                           resp);
-  GCT_iterate_channels (t,
-                        &iter_channel,
-                        resp);
-  resp->connections = htonl (resp->connections);
-  resp->channels = htonl (resp->channels);
-  resp->cstate = htons (0);
-  resp->estate = htons (GCT_get_estate (t));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all peers to dump info for each peer.
- *
- * @param cls Closure (unused).
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-show_peer_iterator (void *cls,
-                    const struct GNUNET_PeerIdentity *peer,
-                    void *value)
-{
-  struct CadetPeer *p = value;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL != t)
-    GCT_debug (t,
-               GNUNET_ERROR_TYPE_ERROR);
-  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO_DUMP request.
- *
- * @param cls Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_info_dump (void *cls,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Received dump info request from client %u\n",
-       c->id);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "*************************** DUMP START ***************************\n");
-  for (struct CadetClient *ci = clients_head;
-       NULL != ci;
-       ci = ci->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "Client %u (%p), handle: %p, ports: %u, channels: %u\n",
-         ci->id,
-         ci,
-         ci->client,
-         (NULL != c->ports)
-         ? GNUNET_CONTAINER_multihashmap_size (ci->ports)
-         : 0,
-         GNUNET_CONTAINER_multihashmap32_size (ci->channels));
-  }
-  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
-  GCP_iterate_all (&show_peer_iterator,
-                   NULL);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "**************************** DUMP END ****************************\n");
-
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-
-/**
- * Callback called when a client connects to the service.
- *
- * @param cls closure for the service
- * @param client the new client that connected to the service
- * @param mq the message queue used to send messages to the client
- * @return @a c
- */
-static void *
-client_connect_cb (void *cls,
-                   struct GNUNET_SERVICE_Client *client,
-                   struct GNUNET_MQ_Handle *mq)
-{
-  struct CadetClient *c;
-
-  c = GNUNET_new (struct CadetClient);
-  c->client = client;
-  c->mq = mq;
-  c->id = next_client_id++; /* overflow not important: just for debug */
-  c->channels
-    = GNUNET_CONTAINER_multihashmap32_create (32);
-  GNUNET_CONTAINER_DLL_insert (clients_head,
-                               clients_tail,
-                               c);
-  GNUNET_STATISTICS_update (stats,
-                            "# clients",
-                            +1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s connected\n",
-       GSC_2s (c));
-  return c;
-}
-
-
-/**
- * A channel was destroyed by the other peer. Tell our client.
- *
- * @param c client that lost a channel
- * @param ccn channel identification number for the client
- * @param ch the channel object
- */
-void
-GSC_handle_remote_channel_destroy (struct CadetClient *c,
-                                   struct GNUNET_CADET_ClientChannelNumber ccn,
-                                   struct CadetChannel *ch)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalChannelDestroyMessage *tdm;
-
-  env = GNUNET_MQ_msg (tdm,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  tdm->ccn = ccn;
-  GSC_send_to_client (c,
-                      env);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         ntohl (ccn.channel_of_client),
-                                                         ch));
-}
-
-
-/**
- * Iterator for deleting each channel whose client endpoint disconnected.
- *
- * @param cls Closure (client that has disconnected).
- * @param key The local channel id in host byte order
- * @param value The value stored at the key (channel to destroy).
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-channel_destroy_iterator (void *cls,
-                          uint32_t key,
-                          void *value)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct CadetChannel *ch = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying %s, due to %s disconnecting.\n",
-       GCCH_2s (ch),
-       GSC_2s (c));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         key,
-                                                         ch));
-  ccn.channel_of_client = htonl (key);
-  GCCH_channel_local_destroy (ch,
-                              c,
-                              ccn);
-  return GNUNET_OK;
-}
-
-
-/**
- * Remove client's ports from the global hashmap on disconnect.
- *
- * @param cls Closure (unused).
- * @param key the port.
- * @param value the `struct CadetClient` to remove
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-client_release_ports (void *cls,
-                      const struct GNUNET_HashCode *key,
-                      void *value)
-{
-  struct CadetClient *c = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing port %s due to %s disconnect.\n",
-       GNUNET_h2s (key),
-       GSC_2s (c));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
-                                                       key,
-                                                       value));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
-                                                       key,
-                                                       value));
-  return GNUNET_OK;
-}
-
-
-/**
- * Callback called when a client disconnected from the service
- *
- * @param cls closure for the service
- * @param client the client that disconnected
- * @param internal_cls should be equal to @a c
- */
-static void
-client_disconnect_cb (void *cls,
-                      struct GNUNET_SERVICE_Client *client,
-                      void *internal_cls)
-{
-  struct CadetClient *c = internal_cls;
-
-  GNUNET_assert (c->client == client);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s is disconnecting.\n",
-       GSC_2s (c));
-  if (NULL != c->channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->channels,
-                                             &channel_destroy_iterator,
-                                             c);
-    GNUNET_CONTAINER_multihashmap32_destroy (c->channels);
-  }
-  if (NULL != c->ports)
-  {
-    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
-                                           &client_release_ports,
-                                           c);
-    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
-  }
-  GNUNET_CONTAINER_DLL_remove (clients_head,
-                               clients_tail,
-                               c);
-  GNUNET_STATISTICS_update (stats,
-                            "# clients",
-                            -1,
-                            GNUNET_NO);
-  GNUNET_free (c);
-  if ( (NULL == clients_head) &&
-       (GNUNET_YES == shutting_down) )
-    shutdown_rest ();
-}
-
-
-/**
- * Setup CADET internals.
- *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *c,
-     struct GNUNET_SERVICE_Handle *service)
-{
-  cfg = c;
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "RATCHET_MESSAGES",
-                                             &ratchet_messages))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_MESSAGES",
-                               "needs to be a number");
-    ratchet_messages = 64;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "RATCHET_TIME",
-                                           &ratchet_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_TIME",
-                               "need delay value");
-    ratchet_time = GNUNET_TIME_UNIT_HOURS;
-  }
-
-  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
-  if (NULL == my_private_key)
-  {
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key,
-                                      &my_full_id.public_key);
-  stats = GNUNET_STATISTICS_create ("cadet",
-                                    c);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  ats_ch = GNUNET_ATS_connectivity_init (c);
-  /* FIXME: optimize code to allow GNUNET_YES here! */
-  open_ports = GNUNET_CONTAINER_multihashmap_create (16,
-                                                     GNUNET_NO);
-  loose_channels = GNUNET_CONTAINER_multihashmap_create (16,
-                                                         GNUNET_NO);
-  peers = GNUNET_CONTAINER_multipeermap_create (16,
-                                                GNUNET_YES);
-  connections = GNUNET_CONTAINER_multishortmap_create (256,
-                                                       GNUNET_YES);
-  GCH_init (c);
-  GCD_init (c);
-  GCO_init (c);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "CADET started for peer %s\n",
-              GNUNET_i2s (&my_full_id));
-
-}
-
-
-/**
- * Define "main" method using service macro.
- */
-GNUNET_SERVICE_MAIN
-("cadet",
- GNUNET_SERVICE_OPTION_NONE,
- &run,
- &client_connect_cb,
- &client_disconnect_cb,
- NULL,
- GNUNET_MQ_hd_fixed_size (port_open,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
-                          struct GNUNET_CADET_PortMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (port_close,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
-                          struct GNUNET_CADET_PortMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (channel_create,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
-                          struct GNUNET_CADET_LocalChannelCreateMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (channel_destroy,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
-                          struct GNUNET_CADET_LocalChannelDestroyMessage,
-                          NULL),
- GNUNET_MQ_hd_var_size (local_data,
-                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
-                        struct GNUNET_CADET_LocalData,
-                        NULL),
- GNUNET_MQ_hd_fixed_size (local_ack,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-                          struct GNUNET_CADET_LocalAck,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (get_peers,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (show_peer,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-                          struct GNUNET_CADET_LocalInfo,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_tunnels,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_tunnel,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-                          struct GNUNET_CADET_LocalInfo,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_dump,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_handler_end ());
-
-/* end of gnunet-service-cadet-new.c */
diff --git a/src/cadet/gnunet-service-cadet-new_channel.c b/src/cadet/gnunet-service-cadet-new_channel.c
deleted file mode 100644
index 0c6241817..000000000
--- a/src/cadet/gnunet-service-cadet-new_channel.c
+++ /dev/null
@@ -1,1613 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_channel.c
- * @brief logical links between CADET clients
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - FIXME: send ACKs back to loopback clients!
- *
- * - introduce shutdown so we can have half-closed channels, modify
- *   destroy to include MID to have FIN-ACK equivalents, etc.
- * - estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
- * - check that '0xFFULL' really is sufficient for flow control!
- * - revisit handling of 'unreliable' traffic!
- * - revisit handling of 'out-of-order' option, especially in combination with/without 'reliable'.
- * - figure out flow control without ACKs (unreliable traffic!)
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-#define LOG(level,...) GNUNET_log_from (level,"cadet-chn",__VA_ARGS__)
-
-/**
- * How long do we initially wait before retransmitting?
- */
-#define CADET_INITIAL_RETRANSMIT_TIME GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
-
-/**
- * How long do we wait before dropping state about incoming
- * connection to closed port?
- */
-#define TIMEOUT_CLOSED_PORT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
-
-
-/**
- * All the states a connection can be in.
- */
-enum CadetChannelState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_CHANNEL_NEW,
-
-  /**
-   * Connection create message sent, waiting for ACK.
-   */
-  CADET_CHANNEL_OPEN_SENT,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CHANNEL_READY
-};
-
-
-/**
- * Info needed to retry a message in case it gets lost.
- * Note that we DO use this structure also for unreliable
- * messages.
- */
-struct CadetReliableMessage
-{
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetReliableMessage *next;
-
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetReliableMessage *prev;
-
-  /**
-   * Which channel is this message in?
-   */
-  struct CadetChannel *ch;
-
-  /**
-   * Entry in the tunnels queue for this message, NULL if it has left
-   * the tunnel.  Used to cancel transmission in case we receive an
-   * ACK in time.
-   */
-  struct CadetTunnelQueueEntry *qe;
-
-  /**
-   * How soon should we retry if we fail to get an ACK?
-   * Messages in the queue are sorted by this value.
-   */
-  struct GNUNET_TIME_Absolute next_retry;
-
-  /**
-   * How long do we wait for an ACK after transmission?
-   * Use for the back-off calculation.
-   */
-  struct GNUNET_TIME_Relative retry_delay;
-
-  /**
-   * Data message we are trying to send.
-   */
-  struct GNUNET_CADET_ChannelAppDataMessage *data_message;
-
-};
-
-
-/**
- * List of received out-of-order data messages.
- */
-struct CadetOutOfOrderMessage
-{
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetOutOfOrderMessage *next;
-
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetOutOfOrderMessage *prev;
-
-  /**
-   * ID of the message (messages up to this point needed
-   * before we give this one to the client).
-   */
-  struct ChannelMessageIdentifier mid;
-
-  /**
-   * The envelope with the payload of the out-of-order message
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-};
-
-
-/**
- * Client endpoint of a `struct CadetChannel`.  A channel may be a
- * loopback channel, in which case it has two of these endpoints.
- * Note that flow control also is required in both directions.
- */
-struct CadetChannelClient
-{
-  /**
-   * Client handle.  Not by itself sufficient to designate
-   * the client endpoint, as the same client handle may
-   * be used for both the owner and the destination, and
-   * we thus also need the channel ID to identify the client.
-   */
-  struct CadetClient *c;
-
-  /**
-   * Head of DLL of messages received out of order or while client was unready.
-   */
-  struct CadetOutOfOrderMessage *head_recv;
-
-  /**
-   * Tail DLL of messages received out of order or while client was unready.
-   */
-  struct CadetOutOfOrderMessage *tail_recv;
-
-  /**
-   * Local tunnel number for this client.
-   * (if owner >= #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI,
-   *  otherwise < #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-   */
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  /**
-   * Can we send data to the client?
-   */
-  int client_ready;
-
-};
-
-
-/**
- * Struct containing all information regarding a channel to a remote client.
- */
-struct CadetChannel
-{
-  /**
-   * Tunnel this channel is in.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Client owner of the tunnel, if any.
-   * (Used if this channel represends the initiating end of the tunnel.)
-   */
-  struct CadetChannelClient *owner;
-
-  /**
-   * Client destination of the tunnel, if any.
-   * (Used if this channel represents the listening end of the tunnel.)
-   */
-  struct CadetChannelClient *dest;
-
-  /**
-   * Last entry in the tunnel's queue relating to control messages
-   * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN or
-   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK).  Used to cancel
-   * transmission in case we receive updated information.
-   */
-  struct CadetTunnelQueueEntry *last_control_qe;
-
-  /**
-   * Head of DLL of messages sent and not yet ACK'd.
-   */
-  struct CadetReliableMessage *head_sent;
-
-  /**
-   * Tail of DLL of messages sent and not yet ACK'd.
-   */
-  struct CadetReliableMessage *tail_sent;
-
-  /**
-   * Task to resend/poll in case no ACK is received.
-   */
-  struct GNUNET_SCHEDULER_Task *retry_control_task;
-
-  /**
-   * Task to resend/poll in case no ACK is received.
-   */
-  struct GNUNET_SCHEDULER_Task *retry_data_task;
-
-  /**
-   * Last time the channel was used
-   */
-  struct GNUNET_TIME_Absolute timestamp;
-
-  /**
-   * Destination port of the channel.
-   */
-  struct GNUNET_HashCode port;
-
-  /**
-   * Counter for exponential backoff.
-   */
-  struct GNUNET_TIME_Relative retry_time;
-
-  /**
-   * How long does it usually take to get an ACK.
-   */
-  struct GNUNET_TIME_Relative expected_delay;
-
-  /**
-   * Bitfield of already-received messages past @e mid_recv.
-   */
-  uint64_t mid_futures;
-
-  /**
-   * Next MID expected for incoming traffic.
-   */
-  struct ChannelMessageIdentifier mid_recv;
-
-  /**
-   * Next MID to use for outgoing traffic.
-   */
-  struct ChannelMessageIdentifier mid_send;
-
-  /**
-   * Total (reliable) messages pending ACK for this channel.
-   */
-  unsigned int pending_messages;
-
-  /**
-   * Maximum (reliable) messages pending ACK for this channel
-   * before we throttle the client.
-   */
-  unsigned int max_pending_messages;
-
-  /**
-   * Number identifying this channel in its tunnel.
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Channel state.
-   */
-  enum CadetChannelState state;
-
-  /**
-   * Is the tunnel bufferless (minimum latency)?
-   */
-  int nobuffer;
-
-  /**
-   * Is the tunnel reliable?
-   */
-  int reliable;
-
-  /**
-   * Is the tunnel out-of-order?
-   */
-  int out_of_order;
-
-  /**
-   * Is this channel a loopback channel, where the destination is us again?
-   */
-  int is_loopback;
-
-  /**
-   * Flag to signal the destruction of the channel.  If this is set to
-   * #GNUNET_YES the channel will be destroyed once the queue is
-   * empty.
-   */
-  int destroy;
-
-};
-
-
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch)
-{
-  static char buf[128];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Channel %s:%s ctn:%X(%X/%X)",
-                   (GNUNET_YES == ch->is_loopback)
-                   ? "loopback"
-                   : GNUNET_i2s (GCP_get_id (GCT_get_destination (ch->t))),
-                   GNUNET_h2s (&ch->port),
-                   ch->ctn,
-                   (NULL == ch->owner) ? 0 : ntohl (ch->owner->ccn.channel_of_client),
-                   (NULL == ch->dest) ? 0 : ntohl (ch->dest->ccn.channel_of_client));
-  return buf;
-}
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch)
-{
-  return ch->ctn;
-}
-
-
-/**
- * Release memory associated with @a ccc
- *
- * @param ccc data structure to clean up
- */
-static void
-free_channel_client (struct CadetChannelClient *ccc)
-{
-  struct CadetOutOfOrderMessage *com;
-
-  while (NULL != (com = ccc->head_recv))
-  {
-    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                 ccc->tail_recv,
-                                 com);
-    GNUNET_MQ_discard (com->env);
-    GNUNET_free (com);
-  }
-  GNUNET_free (ccc);
-}
-
-
-/**
- * Destroy the given channel.
- *
- * @param ch channel to destroy
- */
-static void
-channel_destroy (struct CadetChannel *ch)
-{
-  struct CadetReliableMessage *crm;
-
-  while (NULL != (crm = ch->head_sent))
-  {
-    GNUNET_assert (ch == crm->ch);
-    if (NULL != crm->qe)
-    {
-      GCT_send_cancel (crm->qe);
-      crm->qe = NULL;
-    }
-    GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                                 ch->tail_sent,
-                                 crm);
-    GNUNET_free (crm->data_message);
-    GNUNET_free (crm);
-  }
-  if (NULL != ch->owner)
-  {
-    free_channel_client (ch->owner);
-    ch->owner = NULL;
-  }
-  if (NULL != ch->dest)
-  {
-    free_channel_client (ch->dest);
-    ch->dest = NULL;
-  }
-  if (NULL != ch->last_control_qe)
-  {
-    GCT_send_cancel (ch->last_control_qe);
-    ch->last_control_qe = NULL;
-  }
-  if (NULL != ch->retry_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task = NULL;
-  }
-  if (NULL != ch->retry_control_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-    ch->retry_control_task = NULL;
-  }
-  if (GNUNET_NO == ch->is_loopback)
-  {
-    GCT_remove_channel (ch->t,
-                        ch,
-                        ch->ctn);
-    ch->t = NULL;
-  }
-  GNUNET_free (ch);
-}
-
-
-/**
- * Send a channel create message.
- *
- * @param cls Channel for which to send.
- */
-static void
-send_channel_open (void *cls);
-
-
-/**
- * Function called once the tunnel confirms that we sent the
- * create message.  Delays for a bit until we retry.
- *
- * @param cls our `struct CadetChannel`.
- */
-static void
-channel_open_sent_cb (void *cls)
-{
-  struct CadetChannel *ch = cls;
-
-  GNUNET_assert (NULL != ch->last_control_qe);
-  ch->last_control_qe = NULL;
-  ch->retry_time = GNUNET_TIME_STD_BACKOFF (ch->retry_time);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sent CADET_CHANNEL_OPEN on %s, retrying in %s\n",
-       GCCH_2s (ch),
-       GNUNET_STRINGS_relative_time_to_string (ch->retry_time,
-                                               GNUNET_YES));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_delayed (ch->retry_time,
-                                    &send_channel_open,
-                                    ch);
-}
-
-
-/**
- * Send a channel open message.
- *
- * @param cls Channel for which to send.
- */
-static void
-send_channel_open (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct GNUNET_CADET_ChannelOpenMessage msgcc;
-  uint32_t options;
-
-  ch->retry_control_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CHANNEL_OPEN message for %s\n",
-       GCCH_2s (ch));
-  options = 0;
-  if (ch->nobuffer)
-    options |= GNUNET_CADET_OPTION_NOBUFFER;
-  if (ch->reliable)
-    options |= GNUNET_CADET_OPTION_RELIABLE;
-  if (ch->out_of_order)
-    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
-  msgcc.header.size = htons (sizeof (msgcc));
-  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
-  msgcc.opt = htonl (options);
-  msgcc.port = ch->port;
-  msgcc.ctn = ch->ctn;
-  ch->state = CADET_CHANNEL_OPEN_SENT;
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msgcc.header,
-                                  &channel_open_sent_cb,
-                                  ch);
-}
-
-
-/**
- * Function called once and only once after a channel was bound
- * to its tunnel via #GCT_add_channel() is ready for transmission.
- * Note that this is only the case for channels that this peer
- * initiates, as for incoming channels we assume that they are
- * ready for transmission immediately upon receiving the open
- * message.  Used to bootstrap the #GCT_send() process.
- *
- * @param ch the channel for which the tunnel is now ready
- */
-void
-GCCH_tunnel_up (struct CadetChannel *ch)
-{
-  GNUNET_assert (NULL == ch->retry_control_task);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel up, sending CHANNEL_OPEN on %s now\n",
-       GCCH_2s (ch));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_now (&send_channel_open,
-                                ch);
-}
-
-
-/**
- * Create a new channel.
- *
- * @param owner local client owning the channel
- * @param ccn local number of this channel at the @a owner
- * @param destination peer to which we should build the channel
- * @param port desired port at @a destination
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_local_new (struct CadetClient *owner,
-                        struct GNUNET_CADET_ClientChannelNumber ccn,
-                        struct CadetPeer *destination,
-                        const struct GNUNET_HashCode *port,
-                        uint32_t options)
-{
-  struct CadetChannel *ch;
-  struct CadetChannelClient *ccco;
-
-  ccco = GNUNET_new (struct CadetChannelClient);
-  ccco->c = owner;
-  ccco->ccn = ccn;
-  ccco->client_ready = GNUNET_YES;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->mid_recv.mid = htonl (1); /* The OPEN_ACK counts as message 0! */
-  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
-  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
-  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
-  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
-  ch->owner = ccco;
-  ch->port = *port;
-  if (0 == memcmp (&my_full_id,
-                   GCP_get_id (destination),
-                   sizeof (struct GNUNET_PeerIdentity)))
-  {
-    struct CadetClient *c;
-
-    ch->is_loopback = GNUNET_YES;
-    c = GNUNET_CONTAINER_multihashmap_get (open_ports,
-                                           port);
-    if (NULL == c)
-    {
-      /* port closed, wait for it to possibly open */
-      (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
-                                                port,
-                                                ch,
-                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Created loose incoming loopback channel to port %s\n",
-           GNUNET_h2s (&ch->port));
-    }
-    else
-    {
-      ch->dest = GNUNET_new (struct CadetChannelClient);
-      ch->dest->c = c;
-      ch->dest->client_ready = GNUNET_YES;
-      GCCH_bind (ch,
-                 ch->dest->c);
-    }
-  }
-  else
-  {
-    ch->t = GCP_get_tunnel (destination,
-                            GNUNET_YES);
-    ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
-    ch->ctn = GCT_add_channel (ch->t,
-                               ch);
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Created channel to port %s at peer %s for %s using %s\n",
-       GNUNET_h2s (port),
-       GCP_2s (destination),
-       GSC_2s (owner),
-       (GNUNET_YES == ch->is_loopback) ? "loopback" : GCT_2s (ch->t));
-  return ch;
-}
-
-
-/**
- * We had an incoming channel to a port that is closed.
- * It has not been opened for a while, drop it.
- *
- * @param cls the channel to drop
- */
-static void
-timeout_closed_cb (void *cls)
-{
-  struct CadetChannel *ch = cls;
-
-  ch->retry_control_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing incoming channel to port %s from peer %s due to timeout\n",
-       GNUNET_h2s (&ch->port),
-       GCP_2s (GCT_get_destination (ch->t)));
-  channel_destroy (ch);
-}
-
-
-/**
- * Create a new channel based on a request coming in over the network.
- *
- * @param t tunnel to the remote peer
- * @param ctn identifier of this channel in the tunnel
- * @param port desired local port
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_incoming_new (struct CadetTunnel *t,
-                           struct GNUNET_CADET_ChannelTunnelNumber ctn,
-                           const struct GNUNET_HashCode *port,
-                           uint32_t options)
-{
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->port = *port;
-  ch->t = t;
-  ch->ctn = ctn;
-  ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
-  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
-  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
-  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
-  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-
-  c = GNUNET_CONTAINER_multihashmap_get (open_ports,
-                                         port);
-  if (NULL == c)
-  {
-    /* port closed, wait for it to possibly open */
-    (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
-                                              port,
-                                              ch,
-                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-    ch->retry_control_task
-      = GNUNET_SCHEDULER_add_delayed (TIMEOUT_CLOSED_PORT,
-                                      &timeout_closed_cb,
-                                      ch);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Created loose incoming channel to port %s from peer %s\n",
-         GNUNET_h2s (&ch->port),
-         GCP_2s (GCT_get_destination (ch->t)));
-  }
-  else
-  {
-    GCCH_bind (ch,
-               c);
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-  return ch;
-}
-
-
-/**
- * Function called once the tunnel confirms that we sent the
- * ACK message.  Just remembers it was sent, we do not expect
- * ACKs for ACKs ;-).
- *
- * @param cls our `struct CadetChannel`.
- */
-static void
-send_ack_cb (void *cls)
-{
-  struct CadetChannel *ch = cls;
-
-  GNUNET_assert (NULL != ch->last_control_qe);
-  ch->last_control_qe = NULL;
-}
-
-
-/**
- * Compute and send the current #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK to the other peer.
- *
- * @param ch channel to send the #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK for
- */
-static void
-send_channel_data_ack (struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_ChannelDataAckMessage msg;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->ctn;
-  msg.mid.mid = htonl (ntohl (ch->mid_recv.mid) - 1);
-  msg.futures = GNUNET_htonll (ch->mid_futures);
-  if (NULL != ch->last_control_qe)
-    GCT_send_cancel (ch->last_control_qe);
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msg.header,
-                                  &send_ack_cb,
-                                  ch);
-}
-
-
-/**
- * Send our initial #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK to the client confirming that the
- * connection is up.
- *
- * @param cls the `struct CadetChannel`
- */
-static void
-send_open_ack (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CHANNEL_OPEN_ACK on %s\n",
-       GCCH_2s (ch));
-  ch->retry_control_task = NULL;
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.reserved = htonl (0);
-  msg.ctn = ch->ctn;
-  if (NULL != ch->last_control_qe)
-    GCT_send_cancel (ch->last_control_qe);
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msg.header,
-                                  &send_ack_cb,
-                                  ch);
-}
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
- * this channel.  If the binding was successful, (re)transmit the
- * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
- *
- * @param ch channel that got the duplicate open
- */
-void
-GCCH_handle_duplicate_open (struct CadetChannel *ch)
-{
-  if (NULL == ch->dest)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Ignoring duplicate channel OPEN on %s: port is closed\n",
-         GCCH_2s (ch));
-    return;
-  }
-  if (NULL != ch->retry_control_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Ignoring duplicate channel OPEN on %s: control message is pending\n",
-         GCCH_2s (ch));
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Retransmitting OPEN_ACK on %s\n",
-       GCCH_2s (ch));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_now (&send_open_ack,
-                                ch);
-}
-
-
-/**
- * Send a #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK to the client to solicit more messages.
- *
- * @param ch channel the ack is for
- * @param to_owner #GNUNET_YES to send to owner,
- *                 #GNUNET_NO to send to dest
- */
-static void
-send_ack_to_client (struct CadetChannel *ch,
-                    int to_owner)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalAck *ack;
-  struct CadetChannelClient *ccc;
-
-  env = GNUNET_MQ_msg (ack,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
-  ack->ccn = ccc->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X\n",
-       GSC_2s (ccc->c),
-       (GNUNET_YES == to_owner) ? "owner" : "dest",
-       ntohl (ack->ccn.channel_of_client));
-  GSC_send_to_client (ccc->c,
-                      env);
-}
-
-
-/**
- * A client is bound to the port that we have a channel
- * open to.  Send the acknowledgement for the connection
- * request and establish the link with the client.
- *
- * @param ch open incoming channel
- * @param c client listening on the respective port
- */
-void
-GCCH_bind (struct CadetChannel *ch,
-           struct CadetClient *c)
-{
-  uint32_t options;
-  struct CadetChannelClient *cccd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Binding %s from %s to port %s of %s\n",
-       GCCH_2s (ch),
-       GCT_2s (ch->t),
-       GNUNET_h2s (&ch->port),
-       GSC_2s (c));
-  if (NULL != ch->retry_control_task)
-  {
-    /* there might be a timeout task here */
-    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-    ch->retry_control_task = NULL;
-  }
-  options = 0;
-  if (ch->nobuffer)
-    options |= GNUNET_CADET_OPTION_NOBUFFER;
-  if (ch->reliable)
-    options |= GNUNET_CADET_OPTION_RELIABLE;
-  if (ch->out_of_order)
-    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
-  cccd = GNUNET_new (struct CadetChannelClient);
-  ch->dest = cccd;
-  cccd->c = c;
-  cccd->client_ready = GNUNET_YES;
-  cccd->ccn = GSC_bind (c,
-                        ch,
-                        (GNUNET_YES == ch->is_loopback)
-                        ? GCP_get (&my_full_id,
-                                   GNUNET_YES)
-                        : GCT_get_destination (ch->t),
-                        &ch->port,
-                        options);
-  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
-                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  ch->mid_recv.mid = htonl (1); /* The OPEN counts as message 0! */
-  if (GNUNET_YES == ch->is_loopback)
-  {
-    ch->state = CADET_CHANNEL_OPEN_SENT;
-    GCCH_handle_channel_open_ack (ch);
-  }
-  else
-  {
-    /* notify other peer that we accepted the connection */
-    ch->retry_control_task
-      = GNUNET_SCHEDULER_add_now (&send_open_ack,
-                                  ch);
-  }
-  /* give client it's initial supply of ACKs */
-  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
-                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  for (unsigned int i=0;i<ch->max_pending_messages;i++)
-    send_ack_to_client (ch,
-                        GNUNET_NO);
-}
-
-
-/**
- * Destroy locally created channel.  Called by the local client, so no
- * need to tell the client.
- *
- * @param ch channel to destroy
- * @param c client that caused the destruction
- * @param ccn client number of the client @a c
- */
-void
-GCCH_channel_local_destroy (struct CadetChannel *ch,
-                            struct CadetClient *c,
-                            struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s asks for destruction of %s\n",
-       GSC_2s (c),
-       GCCH_2s (ch));
-  GNUNET_assert (NULL != c);
-  if ( (NULL != ch->owner) &&
-       (c == ch->owner->c) &&
-       (ccn.channel_of_client == ch->owner->ccn.channel_of_client) )
-  {
-    free_channel_client (ch->owner);
-    ch->owner = NULL;
-  }
-  else if ( (NULL != ch->dest) &&
-            (c == ch->dest->c) &&
-            (ccn.channel_of_client == ch->dest->ccn.channel_of_client) )
-  {
-    free_channel_client (ch->dest);
-    ch->dest = NULL;
-  }
-  else
-  {
-    GNUNET_assert (0);
-  }
-
-  if (GNUNET_YES == ch->destroy)
-  {
-    /* other end already destroyed, with the local client gone, no need
-       to finish transmissions, just destroy immediately. */
-    channel_destroy (ch);
-    return;
-  }
-  if ( (NULL != ch->head_sent) ||
-       (NULL != ch->owner) ||
-       (NULL != ch->dest) )
-  {
-    /* Wait for other end to destroy us as well,
-       and otherwise allow send queue to be transmitted first */
-    ch->destroy = GNUNET_YES;
-    return;
-  }
-  /* If the we ever sent the CHANNEL_CREATE, we need to send a destroy message. */
-  if (CADET_CHANNEL_NEW != ch->state)
-    GCT_send_channel_destroy (ch->t,
-                              ch->ctn);
-  /* Nothing left to do, just finish destruction */
-  channel_destroy (ch);
-}
-
-
-/**
- * We got an acknowledgement for the creation of the channel
- * (the port is open on the other side). Begin transmissions.
- *
- * @param ch channel to destroy
- */
-void
-GCCH_handle_channel_open_ack (struct CadetChannel *ch)
-{
-  switch (ch->state)
-  {
-  case CADET_CHANNEL_NEW:
-    /* this should be impossible */
-    GNUNET_break (0);
-    break;
-  case CADET_CHANNEL_OPEN_SENT:
-    if (NULL == ch->owner)
-    {
-      /* We're not the owner, wrong direction! */
-      GNUNET_break_op (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Received CHANNEL_OPEN_ACK for waiting %s, entering READY state\n",
-         GCCH_2s (ch));
-    if (NULL != ch->retry_control_task) /* can be NULL if ch->is_loopback */
-    {
-      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-      ch->retry_control_task = NULL;
-    }
-    ch->state = CADET_CHANNEL_READY;
-    /* On first connect, send client as many ACKs as we allow messages
-       to be buffered! */
-    for (unsigned int i=0;i<ch->max_pending_messages;i++)
-      send_ack_to_client (ch,
-                          GNUNET_YES);
-    break;
-  case CADET_CHANNEL_READY:
-    /* duplicate ACK, maybe we retried the CREATE. Ignore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Received duplicate channel OPEN_ACK for %s\n",
-         GCCH_2s (ch));
-    GNUNET_STATISTICS_update (stats,
-                              "# duplicate CREATE_ACKs",
-                              1,
-                              GNUNET_NO);
-    break;
-  }
-}
-
-
-/**
- * Test if element @a e1 comes before element @a e2.
- *
- * TODO: use opportunity to create generic list insertion sort
- * logic in container!
- *
- * @param cls closure, our `struct CadetChannel`
- * @param e1 an element of to sort
- * @param e2 another element to sort
- * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
- */
-static int
-is_before (void *cls,
-           void *e1,
-           void *e2)
-{
-  struct CadetOutOfOrderMessage *m1 = e1;
-  struct CadetOutOfOrderMessage *m2 = e2;
-  uint32_t v1 = ntohl (m1->mid.mid);
-  uint32_t v2 = ntohl (m2->mid.mid);
-  uint32_t delta;
-
-  delta = v1 - v2;
-  if (delta > (uint32_t) INT_MAX)
-  {
-    /* in overflow range, we can safely assume we wrapped around */
-    return GNUNET_NO;
-  }
-  else
-  {
-    return GNUNET_YES;
-  }
-}
-
-
-/**
- * We got payload data for a channel.  Pass it on to the client
- * and send an ACK to the other end (once flow control allows it!)
- *
- * @param ch channel that got data
- * @param msg message that was received
- */
-void
-GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
-                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalData *ld;
-  struct CadetChannelClient *ccc;
-  struct CadetOutOfOrderMessage *com;
-  size_t payload_size;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  if ( (GNUNET_YES == ch->destroy) &&
-       (NULL == ch->owner) &&
-       (NULL == ch->dest) )
-  {
-    /* This client is gone, but we still have messages to send to
-       the other end (which is why @a ch is not yet dead).  However,
-       we cannot pass messages to our client anymore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Dropping incoming payload on %s as this end is already closed\n",
-         GCCH_2s (ch));
-    /* FIXME: send back ACK/NACK/Closed notification
-       to stop retransmissions! */
-    return;
-  }
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  env = GNUNET_MQ_msg_extra (ld,
-                             payload_size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  ld->ccn = (NULL == ch->dest) ? ch->owner->ccn : ch->dest->ccn;
-  GNUNET_memcpy (&ld[1],
-                 &msg[1],
-                 payload_size);
-  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
-  if ( (GNUNET_YES == ccc->client_ready) &&
-       ( (GNUNET_YES == ch->out_of_order) ||
-         (msg->mid.mid == ch->mid_recv.mid) ) )
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Giving %u bytes of payload from %s to client %s\n",
-         (unsigned int) payload_size,
-         GCCH_2s (ch),
-         GSC_2s (ccc->c));
-    ccc->client_ready = GNUNET_NO;
-    GSC_send_to_client (ccc->c,
-                        env);
-    ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
-    ch->mid_futures >>= 1;
-  }
-  else
-  {
-    /* FIXME-SECURITY: if the element is WAY too far ahead,
-       drop it (can't buffer too much!) */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Queuing %s payload of %u bytes on %s (mid %u, need %u first)\n",
-         (GNUNET_YES == ccc->client_ready)
-         ? "out-of-order"
-         : "client-not-ready",
-         (unsigned int) payload_size,
-         GCCH_2s (ch),
-         ntohl (msg->mid.mid),
-         ntohl (ch->mid_recv.mid));
-
-    com = GNUNET_new (struct CadetOutOfOrderMessage);
-    com->mid = msg->mid;
-    com->env = env;
-    /* sort into list ordered by "is_before" */
-    if ( (NULL == ccc->head_recv) ||
-         (GNUNET_YES == is_before (ch,
-                                   com,
-                                   ccc->head_recv)) )
-    {
-      GNUNET_CONTAINER_DLL_insert (ccc->head_recv,
-                                   ccc->tail_recv,
-                                   com);
-    }
-    else
-    {
-      struct CadetOutOfOrderMessage *pos;
-
-      for (pos = ccc->head_recv;
-           NULL != pos;
-           pos = pos->next)
-      {
-        if (GNUNET_YES !=
-            is_before (NULL,
-                       pos,
-                       com))
-          break;
-      }
-      if (NULL == pos)
-        GNUNET_CONTAINER_DLL_insert_tail (ccc->head_recv,
-                                          ccc->tail_recv,
-                                          com);
-      else
-        GNUNET_CONTAINER_DLL_insert_after (ccc->head_recv,
-                                           ccc->tail_recv,
-                                           com,
-                                           pos->prev);
-    }
-  }
-}
-
-
-/**
- * We got an acknowledgement for payload data for a channel.
- * Possibly resume transmissions.
- *
- * @param ch channel that got the ack
- * @param ack details about what was received
- */
-void
-GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
-                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack)
-{
-  struct CadetReliableMessage *crm;
-
-  GNUNET_break (GNUNET_NO == ch->is_loopback);
-  if (GNUNET_NO == ch->reliable)
-  {
-    /* not expecting ACKs on unreliable channel, odd */
-    GNUNET_break_op (0);
-    return;
-  }
-  for (crm = ch->head_sent;
-        NULL != crm;
-       crm = crm->next)
-    if (ack->mid.mid == crm->data_message->mid.mid)
-      break;
-  if (NULL == crm)
-  {
-    /* ACK for message we already dropped, might have been a
-       duplicate ACK? Ignore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Duplicate DATA_ACK on %s, ignoring\n",
-         GCCH_2s (ch));
-    GNUNET_STATISTICS_update (stats,
-                              "# duplicate DATA_ACKs",
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  GNUNET_free (crm->data_message);
-  GNUNET_free (crm);
-  ch->pending_messages--;
-  send_ack_to_client (ch,
-                      (NULL == ch->owner)
-                      ? GNUNET_NO
-                      : GNUNET_YES);
-  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
-       GCCH_2s (ch),
-       (unsigned int) ntohl (ack->mid.mid),
-       ch->pending_messages);
-  send_ack_to_client (ch,
-                      (NULL == ch->owner)
-                      ? GNUNET_NO
-                      : GNUNET_YES);
-}
-
-
-/**
- * Destroy channel, based on the other peer closing the
- * connection.  Also needs to remove this channel from
- * the tunnel.
- *
- * @param ch channel to destroy
- */
-void
-GCCH_handle_remote_destroy (struct CadetChannel *ch)
-{
-  struct CadetChannelClient *ccc;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received remote channel DESTROY for %s\n",
-       GCCH_2s (ch));
-  if (GNUNET_YES == ch->destroy)
-  {
-    /* Local client already gone, this is instant-death. */
-    channel_destroy (ch);
-    return;
-  }
-  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
-  if (NULL != ccc->head_recv)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Lost end of transmission due to remote shutdown on %s\n",
-         GCCH_2s (ch));
-    /* FIXME: change API to notify client about truncated transmission! */
-  }
-  ch->destroy = GNUNET_YES;
-  GSC_handle_remote_channel_destroy (ccc->c,
-                                     ccc->ccn,
-                                     ch);
-  channel_destroy (ch);
-}
-
-
-/**
- * Function called once the tunnel has sent one of our messages.
- * If the message is unreliable, simply frees the `crm`. If the
- * message was reliable, calculate retransmission time and
- * wait for ACK (or retransmit).
- *
- * @param cls the `struct CadetReliableMessage` that was sent
- */
-static void
-data_sent_cb (void *cls);
-
-
-/**
- * We need to retry a transmission, the last one took too long to
- * be acknowledged.
- *
- * @param cls the `struct CadetChannel` where we need to retransmit
- */
-static void
-retry_transmission (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct CadetReliableMessage *crm = ch->head_sent;
-
-  ch->retry_data_task = NULL;
-  GNUNET_assert (NULL == crm->qe);
-  crm->qe = GCT_send (ch->t,
-                      &crm->data_message->header,
-                      &data_sent_cb,
-                      crm);
-}
-
-
-/**
- * Function called once the tunnel has sent one of our messages.
- * If the message is unreliable, simply frees the `crm`. If the
- * message was reliable, calculate retransmission time and
- * wait for ACK (or retransmit).
- *
- * @param cls the `struct CadetReliableMessage` that was sent
- */
-static void
-data_sent_cb (void *cls)
-{
-  struct CadetReliableMessage *crm = cls;
-  struct CadetChannel *ch = crm->ch;
-  struct CadetReliableMessage *off;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  crm->qe = NULL;
-  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  if (GNUNET_NO == ch->reliable)
-  {
-    GNUNET_free (crm);
-    ch->pending_messages--;
-    send_ack_to_client (ch,
-                        (NULL == ch->owner)
-                        ? GNUNET_NO
-                        : GNUNET_YES);
-    return;
-  }
-  if (0 == crm->retry_delay.rel_value_us)
-    crm->retry_delay = ch->expected_delay;
-  crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
-
-  /* find position for re-insertion into the DLL */
-  if ( (NULL == ch->head_sent) ||
-       (crm->next_retry.abs_value_us < ch->head_sent->next_retry.abs_value_us) )
-  {
-    /* insert at HEAD, also (re)schedule retry task! */
-    GNUNET_CONTAINER_DLL_insert (ch->head_sent,
-                                 ch->tail_sent,
-                                 crm);
-    if (NULL != ch->retry_data_task)
-      GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task
-      = GNUNET_SCHEDULER_add_delayed (crm->retry_delay,
-                                      &retry_transmission,
-                                      ch);
-    return;
-  }
-  for (off = ch->head_sent; NULL != off; off = off->next)
-    if (crm->next_retry.abs_value_us < off->next_retry.abs_value_us)
-      break;
-  if (NULL == off)
-  {
-    /* insert at tail */
-    GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent,
-                                      ch->tail_sent,
-                                      crm);
-  }
-  else
-  {
-    /* insert before off */
-    GNUNET_CONTAINER_DLL_insert_after (ch->head_sent,
-                                       ch->tail_sent,
-                                       off->prev,
-                                       crm);
-  }
-}
-
-
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if
- * channel is reliable and send an ACK to the client if there is still
- * buffer space in the tunnel.
- *
- * @param ch Channel.
- * @param sender_ccn ccn of the sender
- * @param buf payload to transmit.
- * @param buf_len number of bytes in @a buf
- * @return #GNUNET_OK if everything goes well,
- *         #GNUNET_SYSERR in case of an error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
-                        const char *buf,
-                        size_t buf_len)
-{
-  struct CadetReliableMessage *crm;
-
-  if (ch->pending_messages > ch->max_pending_messages)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  ch->pending_messages++;
-
-  if (GNUNET_YES == ch->is_loopback)
-  {
-    struct CadetChannelClient *receiver;
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_LocalData *ld;
-    int to_owner;
-
-    env = GNUNET_MQ_msg_extra (ld,
-                               buf_len,
-                               GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-    if (sender_ccn.channel_of_client ==
-        ch->owner->ccn.channel_of_client)
-    {
-      receiver = ch->dest;
-      to_owner = GNUNET_NO;
-    }
-    else
-    {
-      GNUNET_assert (sender_ccn.channel_of_client ==
-                     ch->dest->ccn.channel_of_client);
-      receiver = ch->owner;
-      to_owner = GNUNET_YES;
-    }
-    ld->ccn = receiver->ccn;
-    GNUNET_memcpy (&ld[1],
-                   buf,
-                   buf_len);
-    /* FIXME: this does not provide for flow control! */
-    GSC_send_to_client (receiver->c,
-                        env);
-    send_ack_to_client (ch,
-                        to_owner);
-    return GNUNET_OK;
-  }
-
-  /* Everything is correct, send the message. */
-  crm = GNUNET_malloc (sizeof (*crm));
-  crm->ch = ch;
-  crm->data_message = GNUNET_malloc (sizeof (struct GNUNET_CADET_ChannelAppDataMessage)
-                                     + buf_len);
-  crm->data_message->header.size = htons (sizeof (struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
-  crm->data_message->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
-  ch->mid_send.mid = htonl (ntohl (ch->mid_send.mid) + 1);
-  crm->data_message->mid = ch->mid_send;
-  crm->data_message->ctn = ch->ctn;
-  GNUNET_memcpy (&crm->data_message[1],
-                 buf,
-                 buf_len);
-  GNUNET_CONTAINER_DLL_insert (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending %u bytes from local client to %s\n",
-       buf_len,
-       GCCH_2s (ch));
-  crm->qe = GCT_send (ch->t,
-                      &crm->data_message->header,
-                      &data_sent_cb,
-                      crm);
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle ACK from client on local channel.  Means the client is ready
- * for more data, see if we have any for it.
- *
- * @param ch channel to destroy
- * @param client_ccn ccn of the client sending the ack
- */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch,
-                       struct GNUNET_CADET_ClientChannelNumber client_ccn)
-{
-  struct CadetChannelClient *ccc;
-  struct CadetOutOfOrderMessage *com;
-
-  if ( (NULL != ch->owner) &&
-       (ch->owner->ccn.channel_of_client == client_ccn.channel_of_client) )
-    ccc = ch->owner;
-  else if ( (NULL != ch->dest) &&
-            (ch->dest->ccn.channel_of_client == client_ccn.channel_of_client) )
-    ccc = ch->dest;
-  else
-    GNUNET_assert (0);
-  ccc->client_ready = GNUNET_YES;
-  com = ccc->head_recv;
-  if (NULL == com)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got LOCAL_ACK, %s-%X ready to receive more data (but none pending)!\n",
-         GSC_2s (ccc->c),
-         ntohl (ccc->ccn.channel_of_client));
-    return; /* none pending */
-  }
-  if ( (com->mid.mid != ch->mid_recv.mid) &&
-       (GNUNET_NO == ch->out_of_order) )
-    return; /* missing next one in-order */
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got LOCAL ACK, passing payload message to %s-%X on %s\n",
-              GSC_2s (ccc->c),
-              ntohl (ccc->ccn.channel_of_client),
-              GCCH_2s (ch));
-
-  /* all good, pass next message to client */
-  GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                               ccc->tail_recv,
-                               com);
-  /* FIXME: if unreliable, this is not aggressive
-     enough, as it would be OK to have lost some! */
-  ch->mid_recv.mid = htonl (1 + ntohl (com->mid.mid));
-  ch->mid_futures >>= 1; /* equivalent to division by 2 */
-  ccc->client_ready = GNUNET_NO;
-  GSC_send_to_client (ccc->c,
-                      com->env);
-  GNUNET_free (com);
-  if ( (0xFFULL == (ch->mid_futures & 0xFFULL)) &&
-       (GNUNET_YES == ch->reliable) )
-  {
-    /* The next 15 messages were also already received (0xFF), this
-       suggests that the sender may be blocked on flow control
-       urgently waiting for an ACK from us. (As we have an inherent
-       maximum of 64 bits, and 15 is getting too close for comfort.)
-       So we should send one now. */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sender on %s likely blocked on flow-control, sending ACK now.\n",
-                GCCH_2s (ch));
-    if (GNUNET_YES == ch->reliable)
-      send_channel_data_ack (ch);
-  }
-
-  if (NULL != ccc->head_recv)
-    return;
-  if (GNUNET_NO == ch->destroy)
-    return;
-  GCT_send_channel_destroy (ch->t,
-                            ch->ctn);
-  channel_destroy (ch);
-}
-
-
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
-
-
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch,
-            enum GNUNET_ErrorType level)
-{
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-chn",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  if (NULL == ch)
-  {
-    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
-    return;
-  }
-  LOG2 (level,
-        "CHN %s:%X (%p)\n",
-        GCT_2s (ch->t),
-        ch->ctn,
-        ch);
-  if (NULL != ch->owner)
-  {
-    LOG2 (level,
-          "CHN origin %s ready %s local-id: %u\n",
-          GSC_2s (ch->owner->c),
-          ch->owner->client_ready ? "YES" : "NO",
-          ntohl (ch->owner->ccn.channel_of_client));
-  }
-  if (NULL != ch->dest)
-  {
-    LOG2 (level,
-          "CHN destination %s ready %s local-id: %u\n",
-          GSC_2s (ch->dest->c),
-          ch->dest->client_ready ? "YES" : "NO",
-          ntohl (ch->dest->ccn.channel_of_client));
-  }
-  LOG2 (level,
-        "CHN  Message IDs recv: %d (%LLX), send: %d\n",
-        ntohl (ch->mid_recv.mid),
-        (unsigned long long) ch->mid_futures,
-        ntohl (ch->mid_send.mid));
-}
-
-
-
-/* end of gnunet-service-cadet-new_channel.c */
diff --git a/src/cadet/gnunet-service-cadet-new_channel.h b/src/cadet/gnunet-service-cadet-new_channel.h
deleted file mode 100644
index e572b7633..000000000
--- a/src/cadet/gnunet-service-cadet-new_channel.h
+++ /dev/null
@@ -1,249 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_channel.h
- * @brief GNUnet CADET service with encryption
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_CHANNEL_H
-#define GNUNET_SERVICE_CADET_CHANNEL_H
-
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "cadet_protocol.h"
-
-
-/**
- * A channel is a bidirectional connection between two CADET
- * clients.  Communiation can be reliable, unreliable, in-order
- * or out-of-order.  One client is the "local" client, this
- * one initiated the connection.   The other client is the
- * "incoming" client, this one listened on a port to accept
- * the connection from the "local" client.
- */
-struct CadetChannel;
-
-
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch);
-
-
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch,
-            enum GNUNET_ErrorType level);
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch);
-
-
-/**
- * Create a new channel.
- *
- * @param owner local client owning the channel
- * @param owner_id local chid of this channel at the @a owner
- * @param destination peer to which we should build the channel
- * @param port desired port at @a destination
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_local_new (struct CadetClient *owner,
-                        struct GNUNET_CADET_ClientChannelNumber owner_id,
-                        struct CadetPeer *destination,
-                        const struct GNUNET_HashCode *port,
-                        uint32_t options);
-
-
-/**
- * A client is bound to the port that we have a channel
- * open to.  Send the acknowledgement for the connection
- * request and establish the link with the client.
- *
- * @param ch open incoming channel
- * @param c client listening on the respective port
- */
-void
-GCCH_bind (struct CadetChannel *ch,
-           struct CadetClient *c);
-
-
-/**
- * Destroy locally created channel.  Called by the
- * local client, so no need to tell the client.
- *
- * @param ch channel to destroy
- * @param c client that caused the destruction
- * @param ccn client number of the client @a c
- */
-void
-GCCH_channel_local_destroy (struct CadetChannel *ch,
-                            struct CadetClient *c,
-                            struct GNUNET_CADET_ClientChannelNumber ccn);
-
-
-/**
- * Function called once and only once after a channel was bound
- * to its tunnel via #GCT_add_channel() is ready for transmission.
- * Note that this is only the case for channels that this peer
- * initiates, as for incoming channels we assume that they are
- * ready for transmission immediately upon receiving the open
- * message.  Used to bootstrap the #GCT_send() process.
- *
- * @param ch the channel for which the tunnel is now ready
- */
-void
-GCCH_tunnel_up (struct CadetChannel *ch);
-
-
-/**
- * Create a new channel based on a request coming in over the network.
- *
- * @param t tunnel to the remote peer
- * @param chid identifier of this channel in the tunnel
- * @param origin peer to who initiated the channel
- * @param port desired local port
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_incoming_new (struct CadetTunnel *t,
-                           struct GNUNET_CADET_ChannelTunnelNumber chid,
-                           const struct GNUNET_HashCode *port,
-                           uint32_t options);
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
- * this channel.  If the binding was successful, (re)transmit the
- * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
- *
- * @param ch channel that got the duplicate open
- */
-void
-GCCH_handle_duplicate_open (struct CadetChannel *ch);
-
-
-/**
- * We got payload data for a channel.  Pass it on to the client.
- *
- * @param ch channel that got data
- * @param msg message that was received
- */
-void
-GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
-                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg);
-
-
-/**
- * We got an acknowledgement for payload data for a channel.
- * Possibly resume transmissions.
- *
- * @param ch channel that got the ack
- * @param ack details about what was received
- */
-void
-GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
-                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack);
-
-
-/**
- * We got an acknowledgement for the creation of the channel
- * (the port is open on the other side). Begin transmissions.
- *
- * @param ch channel to destroy
- */
-void
-GCCH_handle_channel_open_ack (struct CadetChannel *ch);
-
-
-/**
- * Destroy channel, based on the other peer closing the
- * connection.  Also needs to remove this channel from
- * the tunnel.
- *
- * FIXME: need to make it possible to defer destruction until we have
- * received all messages up to the destroy, and right now the destroy
- * message (and this API) fails to give is the information we need!
- *
- * FIXME: also need to know if the other peer got a destroy from
- * us before!
- *
- * @param ch channel to destroy
- */
-void
-GCCH_handle_remote_destroy (struct CadetChannel *ch);
-
-
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if
- * channel is reliable and send an ACK to the client if there is still
- * buffer space in the tunnel.
- *
- * @param ch Channel.
- * @param sender_ccn ccn of the sender
- * @param buf payload to transmit.
- * @param buf_len number of bytes in @a buf
- * @return #GNUNET_OK if everything goes well,
- *         #GNUNET_SYSERR in case of an error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
-                        const char *buf,
-                        size_t buf_len);
-
-
-/**
- * Handle ACK from client on local channel.
- *
- * @param ch channel to destroy
- * @param client_ccn ccn of the client sending the ack
- */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch,
-                       struct GNUNET_CADET_ClientChannelNumber client_ccn);
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet-new_connection.c b/src/cadet/gnunet-service-cadet-new_connection.c
deleted file mode 100644
index 60389008c..000000000
--- a/src/cadet/gnunet-service-cadet-new_connection.c
+++ /dev/null
@@ -1,709 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_connection.c
- * @brief management of CORE-level end-to-end connections; establishes
- *        end-to-end routes and transmits messages along the route
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - Optimization: keepalive messages / timeout (timeout to be done @ peer level!)
- * - Optimization: keep performance metrics (?)
- */
-#include "platform.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet_cadet_service.h"
-#include "cadet_protocol.h"
-
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-con",__VA_ARGS__)
-
-
-/**
- * All the states a connection can be in.
- */
-enum CadetConnectionState
-{
-  /**
-   * Uninitialized status, we have not yet even gotten the message queue.
-   */
-  CADET_CONNECTION_NEW,
-
-  /**
-   * Connection create message in queue, awaiting transmission by CORE.
-   */
-  CADET_CONNECTION_SENDING_CREATE,
-
-  /**
-   * Connection create message sent, waiting for ACK.
-   */
-  CADET_CONNECTION_SENT,
-
-  /**
-   * We are an inbound connection, and received a CREATE. Need to
-   * send an CREATE_ACK back.
-   */
-  CADET_CONNECTION_CREATE_RECEIVED,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CONNECTION_READY
-
-};
-
-
-/**
- * Low-level connection to a destination.
- */
-struct CadetConnection
-{
-
-  /**
-   * ID of the connection.
-   */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-
-  /**
-   * To which peer does this connection go?
-   */
-  struct CadetPeer *destination;
-
-  /**
-   * Which tunnel is using this connection?
-   */
-  struct CadetTConnection *ct;
-
-  /**
-   * Path we are using to our destination.
-   */
-  struct CadetPeerPath *path;
-
-  /**
-   * Pending message, NULL if we are ready to transmit.
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-  /**
-   * Handle for calling #GCP_request_mq_cancel() once we are finished.
-   */
-  struct GCP_MessageQueueManager *mq_man;
-
-  /**
-   * Task for connection maintenance.
-   */
-  struct GNUNET_SCHEDULER_Task *task;
-
-  /**
-   * Function to call once we are ready to transmit.
-   */
-  GCC_ReadyCallback ready_cb;
-
-  /**
-   * Closure for @e ready_cb.
-   */
-  void *ready_cb_cls;
-
-  /**
-   * How long do we wait before we try again with a CREATE message?
-   */
-  struct GNUNET_TIME_Relative retry_delay;
-
-  /**
-   * State of the connection.
-   */
-  enum CadetConnectionState state;
-
-  /**
-   * Offset of our @e destination in @e path.
-   */
-  unsigned int off;
-
-  /**
-   * Are we ready to transmit via @e mq_man right now?
-   */
-  int mqm_ready;
-
-};
-
-
-/**
- * Destroy a connection.
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy (struct CadetConnection *cc)
-{
-  struct GNUNET_MQ_Envelope *env = NULL;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying %s\n",
-       GCC_2s (cc));
-  if (CADET_CONNECTION_SENDING_CREATE != cc->state)
-  {
-    struct GNUNET_CADET_ConnectionDestroyMessage *destroy_msg;
-
-    /* Need to notify next hop that we are down. */
-    env = GNUNET_MQ_msg (destroy_msg,
-                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
-    destroy_msg->cid = cc->cid;
-  }
-  if (NULL != cc->mq_man)
-  {
-    GCP_request_mq_cancel (cc->mq_man,
-                           env);
-    cc->mq_man = NULL;
-  }
-  if (NULL != cc->task)
-  {
-    GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = NULL;
-  }
-  GCPP_del_connection (cc->path,
-                       cc->off,
-                       cc);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                        &GCC_get_id (cc)->connection_of_tunnel,
-                                                        cc));
-  GNUNET_free (cc);
-}
-
-
-/**
- * Return the tunnel associated with this connection.
- *
- * @param cc connection to query
- * @return corresponding entry in the tunnel's connection list
- */
-struct CadetTConnection *
-GCC_get_ct (struct CadetConnection *cc)
-{
-  return cc->ct;
-}
-
-
-/**
- * A CADET_CONNECTION_ACK was received for this connection, implying
- * that the end-to-end connection is up.  Process it.
- *
- * @param cc the connection that got the ACK.
- */
-void
-GCC_handle_connection_create_ack (struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received CADET_CONNECTION_CREATE_ACK for %s in state %d (%s)\n",
-       GCC_2s (cc),
-       cc->state,
-       (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
-  if (NULL != cc->task)
-  {
-    GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = NULL;
-  }
-#if FIXME_KEEPALIVE
-  cc->task = GNUNET_SCHEDULER_add_delayed (cc->keepalive_period,
-                                           &send_keepalive,
-                                           cc);
-#endif
-  cc->state = CADET_CONNECTION_READY;
-  if (GNUNET_YES == cc->mqm_ready)
-    cc->ready_cb (cc->ready_cb_cls,
-                  GNUNET_YES);
-}
-
-
-/**
- * Handle KX message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx (struct CadetConnection *cc,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-  if (CADET_CONNECTION_SENT == cc->state)
-  {
-    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
-       clearly something is working, so pretend we got an ACK. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
-         GCC_2s (cc));
-    GCC_handle_connection_create_ack (cc);
-  }
-  GCT_handle_kx (cc->ct,
-                 msg);
-}
-
-
-/**
- * Handle encrypted message.
- *
- * @param cc connection that received encrypted message
- * @param msg the encrypted message to decrypt
- */
-void
-GCC_handle_encrypted (struct CadetConnection *cc,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  if (CADET_CONNECTION_SENT == cc->state)
-  {
-    /* We didn't get the CREATE_ACK, but instead got payload. That's fine,
-       clearly something is working, so pretend we got an ACK. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Faking connection ACK for %s due to ENCRYPTED payload\n",
-         GCC_2s (cc));
-    GCC_handle_connection_create_ack (cc);
-  }
-  GCT_handle_encrypted (cc->ct,
-                        msg);
-}
-
-
-/**
- * Send a CREATE message to the first hop.
- *
- * @param cls the `struct CadetConnection` to initiate
- */
-static void
-send_create (void *cls)
-{
-  struct CadetConnection *cc = cls;
-  struct GNUNET_CADET_ConnectionCreateMessage *create_msg;
-  struct GNUNET_PeerIdentity *pids;
-  struct GNUNET_MQ_Envelope *env;
-  unsigned int path_length;
-
-  cc->task = NULL;
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  path_length = GCPP_get_length (cc->path);
-  env = GNUNET_MQ_msg_extra (create_msg,
-                             (1 + path_length) * sizeof (struct GNUNET_PeerIdentity),
-                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
-  create_msg->cid = cc->cid;
-  pids = (struct GNUNET_PeerIdentity *) &create_msg[1];
-  pids[0] = my_full_id;
-  for (unsigned int i=0;i<path_length;i++)
-    pids[i + 1] = *GCP_get_id (GCPP_get_peer_at_offset (cc->path,
-                                                        i));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CADET_CONNECTION_CREATE message for %s\n",
-       GCC_2s (cc));
-  cc->env = env;
-  cc->mqm_ready = GNUNET_NO;
-  cc->state = CADET_CONNECTION_SENT;
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * Send a CREATE_ACK message towards the origin.
- *
- * @param cls the `struct CadetConnection` to initiate
- */
-static void
-send_create_ack (void *cls)
-{
-  struct CadetConnection *cc = cls;
-  struct GNUNET_CADET_ConnectionCreateAckMessage *ack_msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  cc->task = NULL;
-  GNUNET_assert (CADET_CONNECTION_CREATE_RECEIVED == cc->state);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CONNECTION_CREATE_ACK message for %s\n",
-       GCC_2s (cc));
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  env = GNUNET_MQ_msg (ack_msg,
-                       GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK);
-  ack_msg->cid = cc->cid;
-  cc->env = env;
-  cc->mqm_ready = GNUNET_NO;
-  cc->state = CADET_CONNECTION_READY;
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
- * connection that we already have.  Either our ACK got lost
- * or something is fishy.  Consider retransmitting the ACK.
- *
- * @param cc connection that got the duplicate CREATE
- */
-void
-GCC_handle_duplicate_create (struct CadetConnection *cc)
-{
-  if (GNUNET_YES == cc->mqm_ready)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got duplicate CREATE for %s, scheduling another ACK (%s)\n",
-         GCC_2s (cc),
-         (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
-    /* Tell tunnel that we are not ready for transmission anymore
-       (until CREATE_ACK is done) */
-    cc->ready_cb (cc->ready_cb_cls,
-                  GNUNET_NO);
-    /* Revert back to the state of having only received the 'CREATE',
-       and immediately proceed to send the CREATE_ACK. */
-    cc->state = CADET_CONNECTION_CREATE_RECEIVED;
-    if (NULL != cc->task)
-      GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
-                                         cc);
-  }
-  else
-  {
-    /* We are currently sending something else back, which
-       can only be an ACK or payload, either of which would
-       do. So actually no need to do anything. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got duplicate CREATE for %s. MQ is busy, not queueing another ACK\n",
-         GCC_2s (cc));
-  }
-}
-
-
-/**
- * There has been a change in the message queue existence for our
- * peer at the first hop.  Adjust accordingly.
- *
- * @param cls the `struct CadetConnection`
- * @param available #GNUNET_YES if sending is now possible,
- *                  #GNUNET_NO if sending is no longer possible
- *                  #GNUNET_SYSERR if sending is no longer possible
- *                                 and the last envelope was discarded
- */
-static void
-manage_first_hop_mq (void *cls,
-                     int available)
-{
-  struct CadetConnection *cc = cls;
-
-  if (GNUNET_YES != available)
-  {
-    /* Connection is down, for now... */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Core MQ for %s went down\n",
-         GCC_2s (cc));
-    cc->mqm_ready = GNUNET_NO;
-    cc->state = CADET_CONNECTION_NEW;
-    cc->retry_delay = GNUNET_TIME_UNIT_ZERO;
-    if (NULL != cc->task)
-    {
-      GNUNET_SCHEDULER_cancel (cc->task);
-      cc->task = NULL;
-    }
-    cc->ready_cb (cc->ready_cb_cls,
-                  GNUNET_NO);
-    return;
-  }
-
-  cc->mqm_ready = GNUNET_YES;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Core MQ for %s became available in state %d\n",
-       GCC_2s (cc),
-       cc->state);
-  switch (cc->state)
-  {
-  case CADET_CONNECTION_NEW:
-    /* Transmit immediately */
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create,
-                                         cc);
-    break;
-  case CADET_CONNECTION_SENDING_CREATE:
-    /* Should not be possible to be called in this state. */
-    GNUNET_assert (0);
-    break;
-  case CADET_CONNECTION_SENT:
-    /* Retry a bit later... */
-    cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
-    cc->task = GNUNET_SCHEDULER_add_delayed (cc->retry_delay,
-                                             &send_create,
-                                             cc);
-    break;
-  case CADET_CONNECTION_CREATE_RECEIVED:
-    /* We got the 'CREATE' (incoming connection), should send the CREATE_ACK */
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
-                                         cc);
-    break;
-  case CADET_CONNECTION_READY:
-    cc->ready_cb (cc->ready_cb_cls,
-                  GNUNET_YES);
-    break;
-  }
-}
-
-
-/**
- * Create a connection to @a destination via @a path and notify @a cb
- * whenever we are ready for more data.  Shared logic independent of
- * who is initiating the connection.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param ct which tunnel uses this connection
- * @param init_state initial state for the connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-static struct CadetConnection *
-connection_create (struct CadetPeer *destination,
-                   struct CadetPeerPath *path,
-                   struct CadetTConnection *ct,
-                   const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                   enum CadetConnectionState init_state,
-                   GCC_ReadyCallback ready_cb,
-                   void *ready_cb_cls)
-{
-  struct CadetConnection *cc;
-  struct CadetPeer *first_hop;
-  unsigned int off;
-
-  off = GCPP_find_peer (path,
-                        destination);
-  GNUNET_assert (UINT_MAX > off);
-  cc = GNUNET_new (struct CadetConnection);
-  cc->state = init_state;
-  cc->ct = ct;
-  cc->cid = *cid;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multishortmap_put (connections,
-                                                     &GCC_get_id (cc)->connection_of_tunnel,
-                                                     cc,
-                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  cc->ready_cb = ready_cb;
-  cc->ready_cb_cls = ready_cb_cls;
-  cc->path = path;
-  cc->off = off;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating %s using path %s\n",
-       GCC_2s (cc),
-       GCPP_2s (path));
-  GCPP_add_connection (path,
-                       off,
-                       cc);
-  for (unsigned int i=0;i<off;i++)
-    GCP_add_connection (GCPP_get_peer_at_offset (path,
-                                                 off),
-                        cc);
-
-  first_hop = GCPP_get_peer_at_offset (path,
-                                       0);
-  cc->mq_man = GCP_request_mq (first_hop,
-                               &manage_first_hop_mq,
-                               cc);
-  return cc;
-}
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.  This
- * is an inbound tunnel, so we must use the existing @a cid
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create_inbound (struct CadetPeer *destination,
-                    struct CadetPeerPath *path,
-                    struct CadetTConnection *ct,
-                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                    GCC_ReadyCallback ready_cb,
-                    void *ready_cb_cls)
-{
-  return connection_create (destination,
-                            path,
-                            ct,
-                            cid,
-                            CADET_CONNECTION_CREATE_RECEIVED,
-                            ready_cb,
-                            ready_cb_cls);
-}
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param ct tunnel that uses the connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create (struct CadetPeer *destination,
-            struct CadetPeerPath *path,
-            struct CadetTConnection *ct,
-            GCC_ReadyCallback ready_cb,
-            void *ready_cb_cls)
-{
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE,
-                              &cid,
-                              sizeof (cid));
-  return connection_create (destination,
-                            path,
-                            ct,
-                            &cid,
-                            CADET_CONNECTION_NEW,
-                            ready_cb,
-                            ready_cb_cls);
-}
-
-
-/**
- * Transmit message @a msg via connection @a cc.  Must only be called
- * (once) after the connection has signalled that it is ready via the
- * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
- * connection is right now ready for transmission.
- *
- * @param cc connection identification
- * @param env envelope with message to transmit; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCC_transmit (struct CadetConnection *cc,
-              struct GNUNET_MQ_Envelope *env)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Scheduling message for transmission on %s\n",
-       GCC_2s (cc));
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  GNUNET_assert (CADET_CONNECTION_READY == cc->state);
-  cc->mqm_ready = GNUNET_NO;
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * Obtain the path used by this connection.
- *
- * @param cc connection
- * @return path to @a cc
- */
-struct CadetPeerPath *
-GCC_get_path (struct CadetConnection *cc)
-{
-  return cc->path;
-}
-
-
-/**
- * Obtain unique ID for the connection.
- *
- * @param cc connection.
- * @return unique number of the connection
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (struct CadetConnection *cc)
-{
-  return &cc->cid;
-}
-
-
-/**
- * Get a (static) string for a connection.
- *
- * @param cc Connection.
- */
-const char *
-GCC_2s (const struct CadetConnection *cc)
-{
-  static char buf[128];
-
-  if (NULL == cc)
-    return "Connection(NULL)";
-
-  if (NULL != cc->ct)
-  {
-    GNUNET_snprintf (buf,
-                     sizeof (buf),
-                     "Connection %s (%s)",
-                     GNUNET_sh2s (&cc->cid.connection_of_tunnel),
-                     GCT_2s (cc->ct->t));
-    return buf;
-  }
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Connection %s",
-                   GNUNET_sh2s (&cc->cid.connection_of_tunnel));
-  return buf;
-}
-
-
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
-
-
-/**
- * Log connection info.
- *
- * @param cc connection
- * @param level Debug level to use.
- */
-void
-GCC_debug (struct CadetConnection *cc,
-           enum GNUNET_ErrorType level)
-{
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-con",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-  if (NULL == cc)
-  {
-    LOG2 (level,
-          "Connection (NULL)\n");
-    return;
-  }
-  LOG2 (level,
-        "%s to %s via path %s in state %d is %s\n",
-        GCC_2s (cc),
-        GCP_2s (cc->destination),
-        GCPP_2s (cc->path),
-        cc->state,
-        (GNUNET_YES == cc->mqm_ready) ? "ready" : "busy");
-}
-
-/* end of gnunet-service-cadet-new_connection.c */
diff --git a/src/cadet/gnunet-service-cadet-new_connection.h b/src/cadet/gnunet-service-cadet-new_connection.h
deleted file mode 100644
index efdf9929a..000000000
--- a/src/cadet/gnunet-service-cadet-new_connection.h
+++ /dev/null
@@ -1,207 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_connection.h
- * @brief
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_CONNECTION_H
-#define GNUNET_SERVICE_CADET_CONNECTION_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "cadet_protocol.h"
-
-
-/**
- * Function called to notify tunnel about change in our readyness.
- *
- * @param cls closure
- * @param is_ready #GNUNET_YES if the connection is now ready for transmission,
- *                 #GNUNET_NO if the connection is no longer ready for transmission
- */
-typedef void
-(*GCC_ReadyCallback)(void *cls,
-                     int is_ready);
-
-
-/**
- * Destroy a connection.
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy (struct CadetConnection *cc);
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create (struct CadetPeer *destination,
-            struct CadetPeerPath *path,
-            struct CadetTConnection *ct,
-            GCC_ReadyCallback ready_cb,
-            void *ready_cb_cls);
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.  This
- * is an inbound tunnel, so we must use the existing @a cid
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create_inbound (struct CadetPeer *destination,
-                    struct CadetPeerPath *path,
-                    struct CadetTConnection *ct,
-                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                    GCC_ReadyCallback ready_cb,
-                    void *ready_cb_cls);
-
-
-/**
- * Transmit message @a msg via connection @a cc.  Must only be called
- * (once) after the connection has signalled that it is ready via the
- * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
- * connection is right now ready for transmission.
- *
- * @param cc connection identification
- * @param env envelope with message to transmit;
- *            the #GNUNET_MQ_notify_send() must not have yet been used
- *            for the envelope.  Also, the message better match the
- *            connection identifier of this connection...
- */
-void
-GCC_transmit (struct CadetConnection *cc,
-              struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * A CREATE_ACK was received for this connection, process it.
- *
- * @param cc the connection that got the ACK.
- */
-void
-GCC_handle_connection_create_ack (struct CadetConnection *cc);
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
- * connection that we already have.  Either our ACK got lost
- * or something is fishy.  Consider retransmitting the ACK.
- *
- * @param cc connection that got the duplicate CREATE
- */
-void
-GCC_handle_duplicate_create (struct CadetConnection *cc);
-
-
-/**
- * Handle KX message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx (struct CadetConnection *cc,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
-
-
-/**
- * Handle encrypted message.
- *
- * @param cc connection that received encrypted message
- * @param msg the encrypted message to decrypt
- */
-void
-GCC_handle_encrypted (struct CadetConnection *cc,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
-
-
-/**
- * Return the tunnel associated with this connection.
- *
- * @param cc connection to query
- * @return corresponding entry in the tunnel's connection list
- */
-struct CadetTConnection *
-GCC_get_ct (struct CadetConnection *cc);
-
-
-/**
- * Obtain the path used by this connection.
- *
- * @param cc connection
- * @return path to @a cc
- */
-struct CadetPeerPath *
-GCC_get_path (struct CadetConnection *cc);
-
-
-/**
- * Obtain unique ID for the connection.
- *
- * @param cc connection.
- * @return unique number of the connection
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (struct CadetConnection *cc);
-
-
-/**
- * Get a (static) string for a connection.
- *
- * @param cc Connection.
- */
-const char *
-GCC_2s (const struct CadetConnection *cc);
-
-
-/**
- * Log connection info.
- *
- * @param cc connection
- * @param level Debug level to use.
- */
-void
-GCC_debug (struct CadetConnection *cc,
-           enum GNUNET_ErrorType level);
-
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet-new_dht.c b/src/cadet/gnunet-service-cadet-new_dht.c
deleted file mode 100644
index 849562f23..000000000
--- a/src/cadet/gnunet-service-cadet-new_dht.c
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_dht.c
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-/**
- * How long do we wait before first announcing our presence to the DHT.
- * Used to wait for our HELLO to be available.  Note that we also get
- * notifications when our HELLO is ready, so this is just the maximum
- * we wait for the first notification.
- */
-#define STARTUP_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 500)
-
-/**
- * How long do we wait after we get an updated HELLO before publishing?
- * Allows for the HELLO to be updated again quickly, for example in
- * case multiple addresses changed and we got a partial update.
- */
-#define CHANGE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100)
-
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
-
-
-/**
- * Handle for DHT searches.
- */
-struct GCD_search_handle
-{
-  /**
-   * DHT_GET handle.
-   */
-  struct GNUNET_DHT_GetHandle *dhtget;
-
-};
-
-
-/**
- * Handle to use DHT.
- */
-static struct GNUNET_DHT_Handle *dht_handle;
-
-/**
- * How often to PUT own ID in the DHT.
- */
-static struct GNUNET_TIME_Relative id_announce_time;
-
-/**
- * DHT replication level, see DHT API: #GNUNET_DHT_get_start(), #GNUNET_DHT_put().
- */
-static unsigned long long dht_replication_level;
-
-/**
- * Task to periodically announce itself in the network.
- */
-static struct GNUNET_SCHEDULER_Task *announce_id_task;
-
-/**
- * Delay for the next ID announce.
- */
-static struct GNUNET_TIME_Relative announce_delay;
-
-
-/**
- * Function to process paths received for a new peer addition. The recorded
- * paths form the initial tunnel, which can be optimized later.
- * Called on each result obtained for the DHT search.
- *
- * @param cls closure
- * @param exp when will this value expire
- * @param key key of the result
- * @param get_path path of the get request
- * @param get_path_length lenght of @a get_path
- * @param put_path path of the put request
- * @param put_path_length length of the @a put_path
- * @param type type of the result
- * @param size number of bytes in data
- * @param data pointer to the result data
- */
-static void
-dht_get_id_handler (void *cls, struct GNUNET_TIME_Absolute exp,
-                    const struct GNUNET_HashCode *key,
-                    const struct GNUNET_PeerIdentity *get_path,
-                    unsigned int get_path_length,
-                    const struct GNUNET_PeerIdentity *put_path,
-                    unsigned int put_path_length,
-                    enum GNUNET_BLOCK_Type type,
-                    size_t size,
-                    const void *data)
-{
-  const struct GNUNET_HELLO_Message *hello = data;
-  struct CadetPeer *peer;
-
-  GCPP_try_path_from_dht (get_path,
-                          get_path_length,
-                          put_path,
-                          put_path_length);
-  if ( (size >= sizeof (struct GNUNET_HELLO_Message)) &&
-       (ntohs (hello->header.size) == size) &&
-       (size == GNUNET_HELLO_size (hello)) )
-  {
-    peer = GCP_get (&put_path[0],
-                    GNUNET_YES);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got HELLO for %s\n",
-         GCP_2s (peer));
-    GCP_set_hello (peer,
-                   hello);
-  }
-}
-
-
-/**
- * Periodically announce self id in the DHT
- *
- * @param cls closure
- */
-static void
-announce_id (void *cls)
-{
-  struct GNUNET_HashCode phash;
-  const struct GNUNET_HELLO_Message *hello;
-  size_t size;
-  struct GNUNET_TIME_Absolute expiration;
-  struct GNUNET_TIME_Relative next_put;
-
-  hello = GCH_get_mine ();
-  size = (NULL != hello) ? GNUNET_HELLO_size (hello) : 0;
-  if (0 == size)
-  {
-    expiration = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
-                                           announce_delay);
-    announce_delay = GNUNET_TIME_STD_BACKOFF (announce_delay);
-  }
-  else
-  {
-    expiration = GNUNET_HELLO_get_last_expiration (hello);
-    announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  }
-
-  /* Call again in id_announce_time, unless HELLO expires first,
-   * but wait at least 1s. */
-  next_put
-    = GNUNET_TIME_absolute_get_remaining (expiration);
-  next_put
-    = GNUNET_TIME_relative_min (next_put,
-                                id_announce_time);
-  next_put
-    = GNUNET_TIME_relative_max (next_put,
-                                GNUNET_TIME_UNIT_SECONDS);
-  announce_id_task
-    = GNUNET_SCHEDULER_add_delayed (next_put,
-                                    &announce_id,
-                                    cls);
-  GNUNET_STATISTICS_update (stats,
-                            "# DHT announce",
-                            1,
-                            GNUNET_NO);
-  memset (&phash,
-          0,
-          sizeof (phash));
-  GNUNET_memcpy (&phash,
-                 &my_full_id,
-                 sizeof (my_full_id));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Announcing my HELLO (%u bytes) in the DHT\n",
-       size);
-  GNUNET_DHT_put (dht_handle,   /* DHT handle */
-                  &phash,       /* Key to use */
-                  dht_replication_level,     /* Replication level */
-                  GNUNET_DHT_RO_RECORD_ROUTE
-                  | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
-                  GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
-                  size,  /* Size of the data */
-                  (const char *) hello, /* Data itself */
-                  expiration,  /* Data expiration */
-                  NULL,         /* Continuation */
-                  NULL);        /* Continuation closure */
-}
-
-
-/**
- * Function called by the HELLO subsystem whenever OUR hello
- * changes. Re-triggers the DHT PUT immediately.
- */
-void
-GCD_hello_update ()
-{
-  if (NULL == announce_id_task)
-    return; /* too early */
-  GNUNET_SCHEDULER_cancel (announce_id_task);
-  announce_id_task
-    = GNUNET_SCHEDULER_add_delayed (CHANGE_DELAY,
-                                    &announce_id,
-                                    NULL);
-}
-
-
-/**
- * Initialize the DHT subsystem.
- *
- * @param c Configuration.
- */
-void
-GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "DHT_REPLICATION_LEVEL",
-                                             &dht_replication_level))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "DHT_REPLICATION_LEVEL",
-                               "USING DEFAULT");
-    dht_replication_level = 3;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "ID_ANNOUNCE_TIME",
-                                           &id_announce_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET",
-                               "ID_ANNOUNCE_TIME",
-                               "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  dht_handle = GNUNET_DHT_connect (c,
-                                   64);
-  GNUNET_break (NULL != dht_handle);
-  announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  announce_id_task = GNUNET_SCHEDULER_add_delayed (STARTUP_DELAY,
-                                                   &announce_id,
-                                                   NULL);
-}
-
-
-/**
- * Shut down the DHT subsystem.
- */
-void
-GCD_shutdown (void)
-{
-  if (NULL != dht_handle)
-  {
-    GNUNET_DHT_disconnect (dht_handle);
-    dht_handle = NULL;
-  }
-  if (NULL != announce_id_task)
-  {
-    GNUNET_SCHEDULER_cancel (announce_id_task);
-    announce_id_task = NULL;
-  }
-}
-
-
-/**
- * Search DHT for paths to @a peeR_id
- *
- * @param peer_id peer to search for
- * @return handle to abort search
- */
-struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id)
-{
-  struct GNUNET_HashCode phash;
-  struct GCD_search_handle *h;
-
-  GNUNET_STATISTICS_update (stats,
-                            "# DHT search",
-                            1,
-                            GNUNET_NO);
-  memset (&phash,
-          0,
-          sizeof (phash));
-  GNUNET_memcpy (&phash,
-                 peer_id,
-                 sizeof (*peer_id));
-
-  h = GNUNET_new (struct GCD_search_handle);
-  h->dhtget = GNUNET_DHT_get_start (dht_handle,    /* handle */
-                                    GNUNET_BLOCK_TYPE_DHT_HELLO, /* type */
-                                    &phash,     /* key to search */
-                                    dht_replication_level, /* replication level */
-                                    GNUNET_DHT_RO_RECORD_ROUTE |
-                                    GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
-                                    NULL,       /* xquery */
-                                    0,     /* xquery bits */
-                                    &dht_get_id_handler,
-                                    h);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Starting DHT GET for peer %s (%p)\n",
-       GNUNET_i2s (peer_id),
-       h);
-  return h;
-}
-
-
-/**
- * Stop DHT search started with #GCD_search().
- *
- * @param h handle to search to stop
- */
-void
-GCD_search_stop (struct GCD_search_handle *h)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Stopping DHT GET %p\n",
-       h);
-  GNUNET_DHT_get_stop (h->dhtget);
-  GNUNET_free (h);
-}
-
-/* end of gnunet-service-cadet_dht.c */
diff --git a/src/cadet/gnunet-service-cadet-new_dht.h b/src/cadet/gnunet-service-cadet-new_dht.h
deleted file mode 100644
index 5d7ab29a0..000000000
--- a/src/cadet/gnunet-service-cadet-new_dht.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_dht.h
- * @brief cadet service; dealing with DHT requests and results
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * All functions in this file should use the prefix GCD (Gnunet Cadet Dht)
- */
-#ifndef GNUNET_SERVICE_CADET_DHT_H
-#define GNUNET_SERVICE_CADET_DHT_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * Handle for DHT search operation.
- */
-struct GCD_search_handle;
-
-
-/**
- * Initialize the DHT subsystem.
- *
- * @param c Configuration.
- */
-void
-GCD_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-
-/**
- * Shut down the DHT subsystem.
- */
-void
-GCD_shutdown (void);
-
-
-/**
- * Function called by the HELLO subsystem whenever OUR hello
- * changes. Re-triggers the DHT PUT immediately.
- */
-void
-GCD_hello_update (void);
-
-/**
- * Search DHT for paths to @a peeR_id
- *
- * @param peer_id peer to search for
- * @return handle to abort search
- */
-struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id);
-
-
-/**
- * Stop DHT search started with #GCD_search().
- *
- * @param h handle to search to stop
- */
-void
-GCD_search_stop (struct GCD_search_handle *h);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_DHT_H */
-#endif
-/* end of gnunet-service-cadet_dht.h */
diff --git a/src/cadet/gnunet-service-cadet-new_hello.c b/src/cadet/gnunet-service-cadet-new_hello.c
deleted file mode 100644
index a24325ada..000000000
--- a/src/cadet/gnunet-service-cadet-new_hello.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2014, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_hello.c
- * @brief spread knowledge about how to contact other peers from PEERINFO
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - is most of this necessary/helpful?
- * - should we not simply restrict this to OUR hello?
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#include "gnunet_statistics_service.h"
-#include "gnunet_peerinfo_service.h"
-#include "cadet_protocol.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-hll",__VA_ARGS__)
-
-/**
- * Hello message of local peer.
- */
-static struct GNUNET_HELLO_Message *mine;
-
-/**
- * Handle to peerinfo service.
- */
-static struct GNUNET_PEERINFO_Handle *peerinfo;
-
-/**
- * Iterator context.
- */
-static struct GNUNET_PEERINFO_NotifyContext *nc;
-
-
-/**
- * Process each hello message received from peerinfo.
- *
- * @param cls Closure (unused).
- * @param peer Identity of the peer.
- * @param hello Hello of the peer.
- * @param err_msg Error message.
- */
-static void
-got_hello (void *cls,
-           const struct GNUNET_PeerIdentity *id,
-           const struct GNUNET_HELLO_Message *hello,
-           const char *err_msg)
-{
-  struct CadetPeer *peer;
-
-  if ( (NULL == id) ||
-       (NULL == hello) )
-    return;
-  if (0 == memcmp (id,
-                   &my_full_id,
-                   sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_free_non_null (mine);
-    mine = (struct GNUNET_HELLO_Message *) GNUNET_copy_message (&hello->header);
-    GCD_hello_update ();
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Hello for %s (%d bytes), expires on %s\n",
-       GNUNET_i2s (id),
-       GNUNET_HELLO_size (hello),
-       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration (hello)));
-  peer = GCP_get (id,
-                  GNUNET_YES);
-  GCP_set_hello (peer,
-                 hello);
-}
-
-
-/**
- * Initialize the hello subsystem.
- *
- * @param c Configuration.
- */
-void
-GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  GNUNET_assert (NULL == nc);
-  peerinfo = GNUNET_PEERINFO_connect (c);
-  nc = GNUNET_PEERINFO_notify (c,
-                               GNUNET_NO,
-                               &got_hello,
-                               NULL);
-}
-
-
-/**
- * Shut down the hello subsystem.
- */
-void
-GCH_shutdown ()
-{
-  if (NULL != nc)
-  {
-    GNUNET_PEERINFO_notify_cancel (nc);
-    nc = NULL;
-  }
-  if (NULL != peerinfo)
-  {
-    GNUNET_PEERINFO_disconnect (peerinfo);
-    peerinfo = NULL;
-  }
-  if (NULL != mine)
-  {
-    GNUNET_free (mine);
-    mine = NULL;
-  }
-}
-
-
-/**
- * Get own hello message.
- *
- * @return Own hello message.
- */
-const struct GNUNET_HELLO_Message *
-GCH_get_mine (void)
-{
-  return mine;
-}
-
-/* end of gnunet-service-cadet-new_hello.c */
diff --git a/src/cadet/gnunet-service-cadet-new_hello.h b/src/cadet/gnunet-service-cadet-new_hello.h
deleted file mode 100644
index 4291ae985..000000000
--- a/src/cadet/gnunet-service-cadet-new_hello.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2014, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_hello.h
- * @brief cadet service; dealing with hello messages
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * All functions in this file should use the prefix GCH (Gnunet Cadet Hello)
- */
-
-#ifndef GNUNET_SERVICE_CADET_HELLO_H
-#define GNUNET_SERVICE_CADET_HELLO_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_hello_lib.h"
-
-
-/**
- * Initialize the hello subsystem.
- *
- * @param c Configuration.
- */
-void
-GCH_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-
-/**
- * Shut down the hello subsystem.
- */
-void
-GCH_shutdown (void);
-
-
-/**
- * Get own hello message.
- *
- * @return Own hello message.
- */
-const struct GNUNET_HELLO_Message *
-GCH_get_mine (void);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_HELLO_H */
-#endif
-/* end of gnunet-cadet-service_hello.h */
diff --git a/src/cadet/gnunet-service-cadet-new_peer.c b/src/cadet/gnunet-service-cadet-new_peer.c
deleted file mode 100644
index fe40d76b6..000000000
--- a/src/cadet/gnunet-service-cadet-new_peer.c
+++ /dev/null
@@ -1,1282 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_peer.c
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - optimize stopping/restarting DHT search to situations
- *   where we actually need it (i.e. not if we have a direct connection,
- *   or if we already have plenty of good short ones, or maybe even
- *   to take a break if we have some connections and have searched a lot (?))
- * - optimize MQM ready scans (O(n) -> O(1))
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_signatures.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_ats_service.h"
-#include "gnunet_core_service.h"
-#include "gnunet_statistics_service.h"
-#include "cadet_protocol.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-per",__VA_ARGS__)
-
-
-/**
- * How long do we wait until tearing down an idle peer?
- */
-#define IDLE_PEER_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
-
-/**
- * How long do we keep paths around if we no longer care about the peer?
- */
-#define IDLE_PATH_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 2)
-
-
-
-
-/**
- * Data structure used to track whom we have to notify about changes
- * to our message queue.
- */
-struct GCP_MessageQueueManager
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GCP_MessageQueueManager *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GCP_MessageQueueManager *prev;
-
-  /**
-   * Function to call with updated message queue object.
-   */
-  GCP_MessageQueueNotificationCallback cb;
-
-  /**
-   * Closure for @e cb.
-   */
-  void *cb_cls;
-
-  /**
-   * The peer this is for.
-   */
-  struct CadetPeer *cp;
-
-  /**
-   * Envelope this manager would like to transmit once it is its turn.
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-};
-
-
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetPeer
-{
-  /**
-   * ID of the peer
-   */
-  struct GNUNET_PeerIdentity pid;
-
-  /**
-   * Last time we heard from this peer
-   */
-  struct GNUNET_TIME_Absolute last_contact;
-
-  /**
-   * Array of DLLs of paths traversing the peer, organized by the
-   * offset of the peer on the larger path.
-   */
-  struct CadetPeerPathEntry **path_heads;
-
-  /**
-   * Array of DLL of paths traversing the peer, organized by the
-   * offset of the peer on the larger path.
-   */
-  struct CadetPeerPathEntry **path_tails;
-
-  /**
-   * Notifications to call when @e core_mq changes.
-   */
-  struct GCP_MessageQueueManager *mqm_head;
-
-  /**
-   * Notifications to call when @e core_mq changes.
-   */
-  struct GCP_MessageQueueManager *mqm_tail;
-
-  /**
-   * MIN-heap of paths owned by this peer (they also end at this
-   * peer).  Ordered by desirability.
-   */
-  struct GNUNET_CONTAINER_Heap *path_heap;
-
-  /**
-   * Handle to stop the DHT search for paths to this peer
-   */
-  struct GCD_search_handle *search_h;
-
-  /**
-   * Task to stop the DHT search for paths to this peer
-   */
-  struct GNUNET_SCHEDULER_Task *search_delayedXXX;
-
-  /**
-   * Task to destroy this entry.
-   */
-  struct GNUNET_SCHEDULER_Task *destroy_task;
-
-  /**
-   * Tunnel to this peer, if any.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Connections that go through this peer; indexed by tid.
-   */
-  struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-  /**
-   * Handle for core transmissions.
-   */
-  struct GNUNET_MQ_Handle *core_mq;
-
-  /**
-   * Hello message of the peer.
-   */
-  struct GNUNET_HELLO_Message *hello;
-
-  /**
-   * Handle to us offering the HELLO to the transport.
-   */
-  struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
-
-  /**
-   * Handle to our ATS request asking ATS to suggest an address
-   * to TRANSPORT for this peer (to establish a direct link).
-   */
-  struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
-
-  /**
-   * How many messages are in the queue to this peer.
-   */
-  unsigned int queue_n;
-
-  /**
-   * How many paths do we have to this peer (in all @e path_heads DLLs combined).
-   */
-  unsigned int num_paths;
-
-  /**
-   * Number of message queue managers of this peer that have a message in waiting.
-   *
-   * Used to quickly see if we need to bother scanning the @e msm_head DLL.
-   * TODO: could be replaced by another DLL that would then allow us to avoid
-   * the O(n)-scan of the DLL for ready entries!
-   */
-  unsigned int mqm_ready_counter;
-
-  /**
-   * Current length of the @e path_heads and @path_tails arrays.
-   * The arrays should be grown as needed.
-   */
-  unsigned int path_dll_length;
-
-};
-
-
-/**
- * Get the static string for a peer ID.
- *
- * @param cp Peer.
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *cp)
-{
-  static char buf[32];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "P(%s)",
-                   GNUNET_i2s (&cp->pid));
-  return buf;
-}
-
-
-/**
- * This peer is no longer be needed, clean it up now.
- *
- * @param cls peer to clean up
- */
-static void
-destroy_peer (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying state about peer %s\n",
-       GCP_2s (cp));
-  cp->destroy_task = NULL;
-  GNUNET_assert (NULL == cp->t);
-  GNUNET_assert (NULL == cp->core_mq);
-  for (unsigned int i=0;i<cp->path_dll_length;i++)
-    GNUNET_assert (NULL == cp->path_heads[i]);
-  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_remove (peers,
-                                                       &cp->pid,
-                                                       cp));
-  GNUNET_free_non_null (cp->path_heads);
-  GNUNET_free_non_null (cp->path_tails);
-  cp->path_dll_length = 0;
-  if (NULL != cp->search_h)
-  {
-    GCD_search_stop (cp->search_h);
-    cp->search_h = NULL;
-  }
-  /* FIXME: clean up search_delayedXXX! */
-
-  if (NULL != cp->hello_offer)
-  {
-    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
-    cp->hello_offer = NULL;
-  }
-  if (NULL != cp->connectivity_suggestion)
-  {
-    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-    cp->connectivity_suggestion = NULL;
-  }
-  GNUNET_CONTAINER_multishortmap_destroy (cp->connections);
-  if (NULL != cp->path_heap)
-  {
-    GNUNET_CONTAINER_heap_destroy (cp->path_heap);
-    cp->path_heap = NULL;
-  }
-  GNUNET_free_non_null (cp->hello);
-  /* Peer should not be freed if paths exist; if there are no paths,
-     there ought to be no connections, and without connections, no
-     notifications. Thus we can assert that mqm_head is empty at this
-     point. */
-  GNUNET_assert (NULL == cp->mqm_head);
-  GNUNET_free (cp);
-}
-
-
-/**
- * This peer is now on more "active" duty, activate processes related to it.
- *
- * @param cp the more-active peer
- */
-static void
-consider_peer_activate (struct CadetPeer *cp)
-{
-  uint32_t strength;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Updating peer %s activation state (%u connections)%s%s\n",
-       GCP_2s (cp),
-       GNUNET_CONTAINER_multishortmap_size (cp->connections),
-       (NULL == cp->t) ? "" : " with tunnel",
-       (NULL == cp->core_mq) ? "" : " with CORE link");
-  if (NULL != cp->destroy_task)
-  {
-    /* It's active, do not destory! */
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  if ( (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections)) &&
-       (NULL == cp->t) )
-  {
-    /* We're just on a path or directly connected; don't bother too much */
-    if (NULL != cp->connectivity_suggestion)
-    {
-      GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-      cp->connectivity_suggestion = NULL;
-    }
-    if (NULL != cp->search_h)
-    {
-      GCD_search_stop (cp->search_h);
-      cp->search_h = NULL;
-    }
-    return;
-  }
-  if (NULL == cp->core_mq)
-  {
-    /* Lacks direct connection, try to create one by querying the DHT */
-    if ( (NULL == cp->search_h) &&
-         (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
-      cp->search_h
-        = GCD_search (&cp->pid);
-  }
-  else
-  {
-    /* Have direct connection, stop DHT search if active */
-    if (NULL != cp->search_h)
-    {
-      GCD_search_stop (cp->search_h);
-      cp->search_h = NULL;
-    }
-  }
-
-  /* If we have a tunnel, our urge for connections is much bigger */
-  strength = (NULL != cp->t) ? 32 : 1;
-  if (NULL != cp->connectivity_suggestion)
-    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-  cp->connectivity_suggestion
-    = GNUNET_ATS_connectivity_suggest (ats_ch,
-                                       &cp->pid,
-                                       strength);
-}
-
-
-/**
- * This peer may no longer be needed, consider cleaning it up.
- *
- * @param cp peer to clean up
- */
-static void
-consider_peer_destroy (struct CadetPeer *cp);
-
-
-/**
- * We really no longere care about a peer, stop hogging memory with paths to it.
- * Afterwards, see if there is more to be cleaned up about this peer.
- *
- * @param cls a `struct CadetPeer`.
- */
-static void
-drop_paths (void *cls)
-{
-  struct CadetPeer *cp = cls;
-  struct CadetPeerPath *path;
-
-  cp->destroy_task = NULL;
-  while (NULL != (path = GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
-    GCPP_release (path);
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * This peer may no longer be needed, consider cleaning it up.
- *
- * @param cp peer to clean up
- */
-static void
-consider_peer_destroy (struct CadetPeer *cp)
-{
-  struct GNUNET_TIME_Relative exp;
-
-  if (NULL != cp->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  if (NULL != cp->t)
-    return; /* still relevant! */
-  if (NULL != cp->core_mq)
-    return; /* still relevant! */
-  if (0 != GNUNET_CONTAINER_multishortmap_size (cp->connections))
-    return; /* still relevant! */
-  if (0 < GNUNET_CONTAINER_heap_get_size (cp->path_heap))
-  {
-    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PATH_TIMEOUT,
-                                                     &drop_paths,
-                                                     cp);
-    return;
-  }
-  for (unsigned int i=0;i<cp->path_dll_length;i++)
-    if (NULL != cp->path_heads[i])
-      return; /* still relevant! */
-  if (NULL != cp->hello)
-  {
-    /* relevant only until HELLO expires */
-    exp = GNUNET_TIME_absolute_get_remaining (GNUNET_HELLO_get_last_expiration (cp->hello));
-    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (exp,
-                                                     &destroy_peer,
-                                                     cp);
-    return;
-  }
-  cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PEER_TIMEOUT,
-                                                   &destroy_peer,
-                                                   cp);
-}
-
-
-/**
- * Set the message queue to @a mq for peer @a cp and notify watchers.
- *
- * @param cp peer to modify
- * @param mq message queue to set (can be NULL)
- */
-void
-GCP_set_mq (struct CadetPeer *cp,
-            struct GNUNET_MQ_Handle *mq)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Message queue for peer %s is now %p\n",
-       GCP_2s (cp),
-       mq);
-  cp->core_mq = mq;
-  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head;
-       NULL != mqm;
-       mqm = mqm->next)
-  {
-    if (NULL == mq)
-    {
-      if (NULL != mqm->env)
-      {
-        GNUNET_MQ_discard (mqm->env);
-        mqm->env = NULL;
-        mqm->cb (mqm->cb_cls,
-                 GNUNET_SYSERR);
-      }
-      else
-      {
-        mqm->cb (mqm->cb_cls,
-                 GNUNET_NO);
-      }
-    }
-    else
-    {
-      GNUNET_assert (NULL == mqm->env);
-      mqm->cb (mqm->cb_cls,
-               GNUNET_YES);
-    }
-  }
-  if ( (NULL != mq) ||
-       (NULL != cp->t) )
-    consider_peer_activate (cp);
-  else
-    consider_peer_destroy (cp);
-
-  if ( (NULL != mq) &&
-       (NULL != cp->t) )
-  {
-    /* have a new, direct path to the target, notify tunnel */
-    struct CadetPeerPath *path;
-
-    path = GCPP_get_path_from_route (1,
-                                     &cp->pid);
-    GCT_consider_path (cp->t,
-                       path,
-                       0);
-  }
-}
-
-
-/**
- * Transmit current envelope from this @a mqm.
- *
- * @param mqm mqm to transmit message for now
- */
-static void
-mqm_execute (struct GCP_MessageQueueManager *mqm)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending to peer %s from MQM %p\n",
-       GCP_2s (cp),
-       mqm);
-  /* Move entry to the end of the DLL, to be fair. */
-  if (mqm != cp->mqm_tail)
-  {
-    GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
-                                 cp->mqm_tail,
-                                 mqm);
-    GNUNET_CONTAINER_DLL_insert_tail (cp->mqm_head,
-                                      cp->mqm_tail,
-                                      mqm);
-  }
-  GNUNET_MQ_send (cp->core_mq,
-                  mqm->env);
-  mqm->env = NULL;
-  cp->mqm_ready_counter--;
-  mqm->cb (mqm->cb_cls,
-           GNUNET_YES);
-}
-
-
-/**
- * Function called when CORE took one of the messages from
- * a message queue manager and transmitted it.
- *
- * @param cls the `struct CadetPeeer` where we made progress
- */
-static void
-mqm_send_done (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending to peer %s completed\n",
-       GCP_2s (cp));
-  if (0 == cp->mqm_ready_counter)
-    return; /* nothing to do */
-  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head;
-       NULL != mqm;
-       mqm = mqm->next)
-  {
-    if (NULL == mqm->env)
-      continue;
-    mqm_execute (mqm);
-    return;
-  }
-}
-
-
-/**
- * Send the message in @a env to @a cp.
- *
- * @param mqm the message queue manager to use for transmission
- * @param env envelope with the message to send; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCP_send (struct GCP_MessageQueueManager *mqm,
-          struct GNUNET_MQ_Envelope *env)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Queueing message to peer %s in MQM %p\n",
-       GCP_2s (cp),
-       mqm);
-  GNUNET_assert (NULL != cp->core_mq);
-  GNUNET_assert (NULL == mqm->env);
-  GNUNET_MQ_notify_sent (env,
-                         &mqm_send_done,
-                         cp);
-  mqm->env = env;
-  cp->mqm_ready_counter++;
-  if (0 != GNUNET_MQ_get_length (cp->core_mq))
-    return;
-  mqm_execute (mqm);
-}
-
-
-/**
- * Function called to destroy a peer now.
- *
- * @param cls NULL
- * @param pid identity of the peer (unused)
- * @param value the `struct CadetPeer` to clean up
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-destroy_iterator_cb (void *cls,
-                     const struct GNUNET_PeerIdentity *pid,
-                     void *value)
-{
-  struct CadetPeer *cp = value;
-
-  if (NULL != cp->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  destroy_peer (cp);
-  return GNUNET_OK;
-}
-
-
-/**
- * Clean up all entries about all peers.
- * Must only be called after all tunnels, CORE-connections and
- * connections are down.
- */
-void
-GCP_destroy_all_peers ()
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying all peers now\n");
-  GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                         &destroy_iterator_cb,
-                                         NULL);
-}
-
-
-/**
- * Drop all paths owned by this peer, and do not
- * allow new ones to be added: We are shutting down.
- *
- * @param cp peer to drop paths to
- */
-void
-GCP_drop_owned_paths (struct CadetPeer *cp)
-{
-  struct CadetPeerPath *path;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying all paths to %s\n",
-       GCP_2s (cp));
-  while (NULL != (path =
-                  GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
-    GCPP_release (path);
-  GNUNET_CONTAINER_heap_destroy (cp->path_heap);
-  cp->path_heap = NULL;
-}
-
-
-/**
- * Add an entry to the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_add (struct CadetPeer *cp,
-                    struct CadetPeerPathEntry *entry,
-                    unsigned int off)
-{
-  GNUNET_assert (cp == GCPP_get_peer_at_offset (entry->path,
-                                                off));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Discovered that peer %s is on path %s at offset %u\n",
-       GCP_2s (cp),
-       GCPP_2s (entry->path),
-       off);
-  if (off >= cp->path_dll_length)
-  {
-    unsigned int len = cp->path_dll_length;
-
-    GNUNET_array_grow (cp->path_heads,
-                       len,
-                       off + 4);
-    GNUNET_array_grow (cp->path_tails,
-                       cp->path_dll_length,
-                       off + 4);
-  }
-  GNUNET_CONTAINER_DLL_insert (cp->path_heads[off],
-                               cp->path_tails[off],
-                               entry);
-  cp->num_paths++;
-
-  /* If we have a tunnel to this peer, tell the tunnel that there is a
-     new path available. */
-  if (NULL != cp->t)
-    GCT_consider_path (cp->t,
-                       entry->path,
-                       off);
-
-  if ( (NULL != cp->search_h) &&
-       (DESIRED_CONNECTIONS_PER_TUNNEL <= cp->num_paths) )
-  {
-    /* Now I have enough paths, stop search */
-    GCD_search_stop (cp->search_h);
-    cp->search_h = NULL;
-  }
-}
-
-
-/**
- * Remove an entry from the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_remove (struct CadetPeer *cp,
-                       struct CadetPeerPathEntry *entry,
-                       unsigned int off)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing knowledge about peer %s beging on path %s at offset %u\n",
-       GCP_2s (cp),
-       GCPP_2s (entry->path),
-       off);
-  GNUNET_CONTAINER_DLL_remove (cp->path_heads[off],
-                               cp->path_tails[off],
-                               entry);
-  GNUNET_assert (0 < cp->num_paths);
-  cp->num_paths--;
-  if ( (NULL == cp->core_mq) &&
-       (NULL != cp->t) &&
-       (NULL == cp->search_h) &&
-       (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
-    cp->search_h
-      = GCD_search (&cp->pid);
-}
-
-
-/**
- * Try adding a @a path to this @a peer.  If the peer already
- * has plenty of paths, return NULL.
- *
- * @param cp peer to which the @a path leads to
- * @param path a path looking for an owner; may not be fully initialized yet!
- * @param off offset of @a cp in @a path
- * @param force force attaching the path
- * @return NULL if this peer does not care to become a new owner,
- *         otherwise the node in the peer's path heap for the @a path.
- */
-struct GNUNET_CONTAINER_HeapNode *
-GCP_attach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 unsigned int off,
-                 int force)
-{
-  GNUNET_CONTAINER_HeapCostType desirability;
-  struct CadetPeerPath *root;
-  GNUNET_CONTAINER_HeapCostType root_desirability;
-  struct GNUNET_CONTAINER_HeapNode *hn;
-
-  GNUNET_assert (cp == GCPP_get_peer_at_offset (path,
-                                                off));
-  if (NULL == cp->path_heap)
-  {
-    /* #GCP_drop_owned_paths() was already called, we cannot take new ones! */
-    GNUNET_assert (GNUNET_NO == force);
-    return NULL;
-  }
-  desirability = GCPP_get_desirability (path);
-  if (GNUNET_NO == force)
-  {
-    /* FIXME: desirability is not yet initialized; tricky! */
-    if (GNUNET_NO ==
-        GNUNET_CONTAINER_heap_peek2 (cp->path_heap,
-                                     (void **) &root,
-                                     &root_desirability))
-    {
-      root = NULL;
-      root_desirability = 0;
-    }
-
-    if ( (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) &&
-         (desirability < root_desirability) )
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Decided to not attach path %p to peer %s due to undesirability\n",
-           GCPP_2s (path),
-           GCP_2s (cp));
-      return NULL;
-    }
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Attaching path %s to peer %s (%s)\n",
-       GCPP_2s (path),
-       GCP_2s (cp),
-       (GNUNET_NO == force) ? "desirable" : "forced");
-
-  /* Yes, we'd like to add this path, add to our heap */
-  hn = GNUNET_CONTAINER_heap_insert (cp->path_heap,
-                                     path,
-                                     desirability);
-
-  /* Consider maybe dropping other paths because of the new one */
-  if (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
-      2 * DESIRED_CONNECTIONS_PER_TUNNEL)
-  {
-    /* Now we have way too many, drop least desirable UNLESS it is in use!
-       (Note that this intentionally keeps highly desireable, but currently
-       unused paths around in the hope that we might be able to switch, even
-       if the number of paths exceeds the threshold.) */
-    root = GNUNET_CONTAINER_heap_peek (cp->path_heap);
-    if ( (path != root) &&
-         (NULL ==
-          GCPP_get_connection (root,
-                               cp,
-                               GCPP_get_length (root) - 1)) )
-    {
-      /* Got plenty of paths to this destination, and this is a low-quality
-         one that we don't care, allow it to die. */
-      GNUNET_assert (root ==
-                     GNUNET_CONTAINER_heap_remove_root (cp->path_heap));
-      GCPP_release (root);
-    }
-  }
-  return hn;
-}
-
-
-/**
- * This peer can no longer own @a path as the path
- * has been extended and a peer further down the line
- * is now the new owner.
- *
- * @param cp old owner of the @a path
- * @param path path where the ownership is lost
- * @param hn note in @a cp's path heap that must be deleted
- */
-void
-GCP_detach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 struct GNUNET_CONTAINER_HeapNode *hn)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Detatching path %s from peer %s\n",
-       GCPP_2s (path),
-       GCP_2s (cp));
-  GNUNET_assert (path ==
-                 GNUNET_CONTAINER_heap_remove_node (hn));
-}
-
-
-/**
- * Add a @a connection to this @a cp.
- *
- * @param cp peer via which the @a connection goes
- * @param cc the connection to add
- */
-void
-GCP_add_connection (struct CadetPeer *cp,
-                    struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding connection %s to peer %s\n",
-       GCC_2s (cc),
-       GCP_2s (cp));
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multishortmap_put (cp->connections,
-                                                     &GCC_get_id (cc)->connection_of_tunnel,
-                                                     cc,
-                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-}
-
-
-/**
- * Remove a @a connection that went via this @a cp.
- *
- * @param cp peer via which the @a connection went
- * @param cc the connection to remove
- */
-void
-GCP_remove_connection (struct CadetPeer *cp,
-                       struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing connection %s from peer %s\n",
-       GCC_2s (cc),
-       GCP_2s (cp));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_remove (cp->connections,
-                                                        &GCC_get_id (cc)->connection_of_tunnel,
-                                                        cc));
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO to return NULL if peer is unknown.
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id,
-         int create)
-{
-  struct CadetPeer *cp;
-
-  cp = GNUNET_CONTAINER_multipeermap_get (peers,
-                                          peer_id);
-  if (NULL != cp)
-    return cp;
-  if (GNUNET_NO == create)
-    return NULL;
-  cp = GNUNET_new (struct CadetPeer);
-  cp->pid = *peer_id;
-  cp->connections = GNUNET_CONTAINER_multishortmap_create (32,
-                                                           GNUNET_YES);
-  cp->path_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_put (peers,
-                                                    &cp->pid,
-                                                    cp,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating peer %s\n",
-       GCP_2s (cp));
-  return cp;
-}
-
-
-/**
- * Obtain the peer identity for a `struct CadetPeer`.
- *
- * @param cp our peer handle
- * @return the peer identity
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (struct CadetPeer *cp)
-{
-  return &cp->pid;
-}
-
-
-/**
- * Iterate over all known peers.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls)
-{
-  GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                         iter,
-                                         cls);
-}
-
-
-/**
- * Count the number of known paths toward the peer.
- *
- * @param cp Peer to get path info.
- * @return Number of known paths.
- */
-unsigned int
-GCP_count_paths (const struct CadetPeer *cp)
-{
-  return cp->num_paths;
-}
-
-
-/**
- * Iterate over the paths to a peer.
- *
- * @param cp Peer to get path info.
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *cp,
-                   GCP_PathIterator callback,
-                   void *callback_cls)
-{
-  unsigned int ret = 0;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Iterating over paths to peer %s%s\n",
-       GCP_2s (cp),
-       (NULL == cp->core_mq) ? "" : " including direct link");
-  if (NULL != cp->core_mq)
-  {
-    struct CadetPeerPath *path;
-
-    path = GCPP_get_path_from_route (1,
-                                     &cp->pid);
-    ret++;
-    if (GNUNET_NO ==
-        callback (callback_cls,
-                  path,
-                  1))
-      return ret;
-  }
-  for (unsigned int i=0;i<cp->path_dll_length;i++)
-  {
-    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
-         NULL != pe;
-         pe = pe->next)
-    {
-      ret++;
-      if (GNUNET_NO ==
-          callback (callback_cls,
-                    pe->path,
-                    i))
-        return ret;
-    }
-  }
-  return ret;
-}
-
-
-/**
- * Iterate over the paths to @a cp where
- * @a cp is at distance @a dist from us.
- *
- * @param cp Peer to get path info.
- * @param dist desired distance of @a cp to us on the path
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths_at (struct CadetPeer *cp,
-                      unsigned int dist,
-                      GCP_PathIterator callback,
-                      void *callback_cls)
-{
-  unsigned int ret = 0;
-
-  if (dist >= cp->path_dll_length)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Asked to look for paths at distance %u, but maximum for me is < %u\n",
-         dist,
-         cp->path_dll_length);
-    return 0;
-  }
-  for (struct CadetPeerPathEntry *pe = cp->path_heads[dist];
-       NULL != pe;
-       pe = pe->next)
-  {
-    if (GNUNET_NO ==
-        callback (callback_cls,
-                  pe->path,
-                  dist))
-      return ret;
-    ret++;
-  }
-  return ret;
-}
-
-
-/**
- * Get the tunnel towards a peer.
- *
- * @param cp Peer to get from.
- * @param create #GNUNET_YES to create a tunnel if we do not have one
- * @return Tunnel towards peer.
- */
-struct CadetTunnel *
-GCP_get_tunnel (struct CadetPeer *cp,
-                int create)
-{
-  if (NULL == cp)
-    return NULL;
-  if ( (NULL != cp->t) ||
-       (GNUNET_NO == create) )
-    return cp->t;
-  cp->t = GCT_create_tunnel (cp);
-  consider_peer_activate (cp);
-  return cp->t;
-}
-
-
-/**
- * Hello offer was passed to the transport service. Mark it
- * as done.
- *
- * @param cls the `struct CadetPeer` where the offer completed
- */
-static void
-hello_offer_done (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  cp->hello_offer = NULL;
-}
-
-
-/**
- * We got a HELLO for a @a peer, remember it, and possibly
- * trigger adequate actions (like trying to connect).
- *
- * @param cp the peer we got a HELLO for
- * @param hello the HELLO to remember
- */
-void
-GCP_set_hello (struct CadetPeer *cp,
-               const struct GNUNET_HELLO_Message *hello)
-{
-  struct GNUNET_HELLO_Message *mrg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got %u byte HELLO for peer %s\n",
-       (unsigned int) GNUNET_HELLO_size (hello),
-       GCP_2s (cp));
-  if (NULL != cp->hello_offer)
-  {
-    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
-    cp->hello_offer = NULL;
-  }
-  if (NULL != cp->hello)
-  {
-    mrg = GNUNET_HELLO_merge (hello,
-                              cp->hello);
-    GNUNET_free (cp->hello);
-    cp->hello = mrg;
-  }
-  else
-  {
-    cp->hello = GNUNET_memdup (hello,
-                               GNUNET_HELLO_size (hello));
-  }
-  cp->hello_offer
-    = GNUNET_TRANSPORT_offer_hello (cfg,
-                                    GNUNET_HELLO_get_header (cp->hello) ,
-                                    &hello_offer_done,
-                                    cp);
-  /* New HELLO means cp's destruction time may change... */
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * The tunnel to the given peer no longer exists, remove it from our
- * data structures, and possibly clean up the peer itself.
- *
- * @param cp the peer affected
- * @param t the dead tunnel
- */
-void
-GCP_drop_tunnel (struct CadetPeer *cp,
-                 struct CadetTunnel *t)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Dropping tunnel %s to peer %s\n",
-       GCT_2s (t),
-       GCP_2s (cp));
-  GNUNET_assert (cp->t == t);
-  cp->t = NULL;
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * Test if @a cp has a core-level connection
- *
- * @param cp peer to test
- * @return #GNUNET_YES if @a cp has a core-level connection
- */
-int
-GCP_has_core_connection (struct CadetPeer *cp)
-{
-  return (NULL != cp->core_mq) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Start message queue change notifications.
- *
- * @param cp peer to notify for
- * @param cb function to call if mq becomes available or unavailable
- * @param cb_cls closure for @a cb
- * @return handle to cancel request
- */
-struct GCP_MessageQueueManager *
-GCP_request_mq (struct CadetPeer *cp,
-                GCP_MessageQueueNotificationCallback cb,
-                void *cb_cls)
-{
-  struct GCP_MessageQueueManager *mqm;
-
-  mqm = GNUNET_new (struct GCP_MessageQueueManager);
-  mqm->cb = cb;
-  mqm->cb_cls = cb_cls;
-  mqm->cp = cp;
-  GNUNET_CONTAINER_DLL_insert (cp->mqm_head,
-                               cp->mqm_tail,
-                               mqm);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating MQM %p for peer %s\n",
-       mqm,
-       GCP_2s (cp));
-  if (NULL != cp->core_mq)
-    cb (cb_cls,
-        GNUNET_YES);
-  return mqm;
-}
-
-
-/**
- * Stops message queue change notifications.
- *
- * @param mqm handle matching request to cancel
- * @param last_env final message to transmit, or NULL
- */
-void
-GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
-                       struct GNUNET_MQ_Envelope *last_env)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying MQM %p for peer %s%s\n",
-       mqm,
-       GCP_2s (cp),
-       (NULL == last_env) ? "" : " with last ditch transmission");
-  if (NULL != mqm->env)
-    GNUNET_MQ_discard (mqm->env);
-  if (NULL != last_env)
-  {
-    if (NULL != cp->core_mq)
-      GNUNET_MQ_send (cp->core_mq,
-                      last_env);
-    else
-      GNUNET_MQ_discard (last_env);
-  }
-  GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
-                               cp->mqm_tail,
-                               mqm);
-  GNUNET_free (mqm);
-}
-
-
-/**
- * Send the message in @a env to @a cp, overriding queueing logic.
- * This function should only be used to send error messages outside
- * of flow and congestion control, similar to ICMP.  Note that
- * the envelope may be silently discarded as well.
- *
- * @param cp peer to send the message to
- * @param env envelope with the message to send
- */
-void
-GCP_send_ooo (struct CadetPeer *cp,
-              struct GNUNET_MQ_Envelope *env)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message to %s out of management\n",
-       GCP_2s (cp));
-  if (NULL == cp->core_mq)
-  {
-    GNUNET_MQ_discard (env);
-    return;
-  }
-  GNUNET_MQ_send (cp->core_mq,
-                  env);
-}
-
-
-
-
-/* end of gnunet-service-cadet-new_peer.c */
diff --git a/src/cadet/gnunet-service-cadet-new_peer.h b/src/cadet/gnunet-service-cadet-new_peer.h
deleted file mode 100644
index aaaef15b8..000000000
--- a/src/cadet/gnunet-service-cadet-new_peer.h
+++ /dev/null
@@ -1,380 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_peer.h
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_PEER_H
-#define GNUNET_SERVICE_CADET_PEER_H
-
-#include "gnunet-service-cadet-new.h"
-#include "gnunet_hello_lib.h"
-
-
-/**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
- *
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *peer);
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO to return NULL if peer is unknown.
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id,
-         int create);
-
-
-/**
- * Obtain the peer identity for a `struct CadetPeer`.
- *
- * @param cp our peer handle
- * @return the peer identity
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (struct CadetPeer *cp);
-
-
-/**
- * Iterate over all known peers.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls);
-
-
-/**
- * Count the number of known paths toward the peer.
- *
- * @param cp Peer to get path info.
- * @return Number of known paths.
- */
-unsigned int
-GCP_count_paths (const struct CadetPeer *cp);
-
-
-/**
- * Drop all paths owned by this peer, and do not
- * allow new ones to be added: We are shutting down.
- *
- * @param cp peer to drop paths to
- */
-void
-GCP_drop_owned_paths (struct CadetPeer *cp);
-
-
-/**
- * Peer path iterator.
- *
- * @param cls Closure.
- * @param path Path itself
- * @param off offset of the target peer in @a path
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-typedef int
-(*GCP_PathIterator) (void *cls,
-                     struct CadetPeerPath *path,
-                     unsigned int off);
-
-
-/**
- * Iterate over the paths to a peer.
- *
- * @param cp Peer to get path info.
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *cp,
-                   GCP_PathIterator callback,
-                   void *callback_cls);
-
-
-/**
- * Iterate over the paths to @a peer where
- * @a peer is at distance @a dist from us.
- *
- * @param cp Peer to get path info.
- * @param dist desired distance of @a peer to us on the path
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths_at (struct CadetPeer *cp,
-                      unsigned int dist,
-                      GCP_PathIterator callback,
-                      void *callback_cls);
-
-
-/**
- * Remove an entry from the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_remove (struct CadetPeer *cp,
-                       struct CadetPeerPathEntry *entry,
-                       unsigned int off);
-
-
-/**
- * Add an entry to the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_add (struct CadetPeer *cp,
-                    struct CadetPeerPathEntry *entry,
-                    unsigned int off);
-
-
-/**
- * Get the tunnel towards a peer.
- *
- * @param cp Peer to get from.
- * @param create #GNUNET_YES to create a tunnel if we do not have one
- * @return Tunnel towards peer.
- */
-struct CadetTunnel *
-GCP_get_tunnel (struct CadetPeer *cp,
-                int create);
-
-
-/**
- * The tunnel to the given peer no longer exists, remove it from our
- * data structures, and possibly clean up the peer itself.
- *
- * @param cp the peer affected
- * @param t the dead tunnel
- */
-void
-GCP_drop_tunnel (struct CadetPeer *cp,
-                 struct CadetTunnel *t);
-
-
-/**
- * Try adding a @a path to this @a cp.  If the peer already
- * has plenty of paths, return NULL.
- *
- * @param cp peer to which the @a path leads to
- * @param path a path looking for an owner; may not be fully initialized yet!
- * @param off offset of @a cp in @a path
- * @param force for attaching the path
- * @return NULL if this peer does not care to become a new owner,
- *         otherwise the node in the peer's path heap for the @a path.
- */
-struct GNUNET_CONTAINER_HeapNode *
-GCP_attach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 unsigned int off,
-                 int force);
-
-
-/**
- * This peer can no longer own @a path as the path
- * has been extended and a peer further down the line
- * is now the new owner.
- *
- * @param cp old owner of the @a path
- * @param path path where the ownership is lost
- * @param hn note in @a cp's path heap that must be deleted
- */
-void
-GCP_detach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 struct GNUNET_CONTAINER_HeapNode *hn);
-
-
-/**
- * Add a @a connection to this @a cp.
- *
- * @param cp peer via which the @a connection goes
- * @param cc the connection to add
- */
-void
-GCP_add_connection (struct CadetPeer *cp,
-                    struct CadetConnection *cc);
-
-
-/**
- * Remove a @a connection that went via this @a cp.
- *
- * @param cp peer via which the @a connection went
- * @param cc the connection to remove
- */
-void
-GCP_remove_connection (struct CadetPeer *cp,
-                       struct CadetConnection *cc);
-
-
-/**
- * We got a HELLO for a @a cp, remember it, and possibly
- * trigger adequate actions (like trying to connect).
- *
- * @param cp the peer we got a HELLO for
- * @param hello the HELLO to remember
- */
-void
-GCP_set_hello (struct CadetPeer *cp,
-               const struct GNUNET_HELLO_Message *hello);
-
-
-/**
- * Clean up all entries about all peers.
- * Must only be called after all tunnels, CORE-connections and
- * connections are down.
- */
-void
-GCP_destroy_all_peers (void);
-
-
-/**
- * Data structure used to track whom we have to notify about changes
- * in our ability to transmit to a given peer.
- *
- * All queue managers will be given equal chance for sending messages
- * to @a cp.  This construct this guarantees fairness for access to @a
- * cp among the different message queues.  Each connection or route
- * will have its respective message queue managers for each direction.
- */
-struct GCP_MessageQueueManager;
-
-
-/**
- * Function to call with updated message queue object.
- *
- * @param cls closure
- * @param available #GNUNET_YES if sending is now possible,
- *                  #GNUNET_NO if sending is no longer possible
- *                  #GNUNET_SYSERR if sending is no longer possible
- *                                 and the last envelope was discarded
- */
-typedef void
-(*GCP_MessageQueueNotificationCallback)(void *cls,
-                                        int available);
-
-
-/**
- * Start message queue change notifications.  Will create a new slot
- * to manage the message queue to the given @a cp.
- *
- * @param cp peer to notify for
- * @param cb function to call if mq becomes available or unavailable
- * @param cb_cls closure for @a cb
- * @return handle to cancel request
- */
-struct GCP_MessageQueueManager *
-GCP_request_mq (struct CadetPeer *cp,
-                GCP_MessageQueueNotificationCallback cb,
-                void *cb_cls);
-
-
-/**
- * Test if @a cp has a core-level connection
- *
- * @param cp peer to test
- * @return #GNUNET_YES if @a cp has a core-level connection
- */
-int
-GCP_has_core_connection (struct CadetPeer *cp);
-
-
-/**
- * Send the message in @a env via a @a mqm.  Must only be called at
- * most once after the respective
- * #GCP_MessageQueueNotificationCallback was called with `available`
- * set to #GNUNET_YES, and not after the callback was called with
- * `available` set to #GNUNET_NO or #GNUNET_SYSERR.
- *
- * @param mqm message queue manager for the transmission
- * @param env envelope with the message to send; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCP_send (struct GCP_MessageQueueManager *mqm,
-          struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * Send the message in @a env to @a cp, overriding queueing logic.
- * This function should only be used to send error messages outside
- * of flow and congestion control, similar to ICMP.  Note that
- * the envelope may be silently discarded as well.
- *
- * @param cp peer to send the message to
- * @param env envelope with the message to send
- */
-void
-GCP_send_ooo (struct CadetPeer *cp,
-              struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * Stops message queue change notifications and sends a last message.
- * In practice, this is implemented by sending that @a last_env
- * message immediately (if any), ignoring queue order.
- *
- * @param mqm handle matching request to cancel
- * @param last_env final message to transmit, or NULL
- */
-void
-GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
-                       struct GNUNET_MQ_Envelope *last_env);
-
-
-/**
- * Set the message queue to @a mq for peer @a cp and notify watchers.
- *
- * @param cp peer to modify
- * @param mq message queue to set (can be NULL)
- */
-void
-GCP_set_mq (struct CadetPeer *cp,
-            struct GNUNET_MQ_Handle *mq);
-
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet.c b/src/cadet/gnunet-service-cadet.c
index 3a07f0ee5..af4cebfaa 100644
--- a/src/cadet/gnunet-service-cadet.c
+++ b/src/cadet/gnunet-service-cadet.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001-2013 GNUnet e.V.
+     Copyright (C) 2001-2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,37 +22,82 @@
  * @file cadet/gnunet-service-cadet.c
  * @brief GNUnet CADET service with encryption
  * @author Bartlomiej Polot
- *
- *  FIXME in progress:
- * - rekey - reliability interaction
- * - channel retransmit timing
- *
- * TODO:
- * - relay corking down to core
- * - set ttl relative to path length
- * TODO END
+ * @author Christian Grothoff
  *
  * Dictionary:
  * - peer: other cadet instance. If there is direct connection it's a neighbor.
- * - tunnel: encrypted connection to a peer, neighbor or not.
- * - channel: connection between two clients, on the same or different peers.
- *            have properties like reliability.
  * - path: series of directly connected peer from one peer to another.
  * - connection: path which is being used in a tunnel.
+ * - tunnel: encrypted connection to a peer, neighbor or not.
+ * - channel: logical link between two clients, on the same or different peers.
+ *            have properties like reliability.
  */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "cadet.h"
 #include "gnunet_statistics_service.h"
-
-#include "gnunet-service-cadet_local.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_channel.h"
 #include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_tunnel.h"
+#include "gnunet-service-cadet_core.h"
 #include "gnunet-service-cadet_dht.h"
-#include "gnunet-service-cadet_peer.h"
 #include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+
+#define LOG(level, ...) GNUNET_log (level,__VA_ARGS__)
+
+
+/**
+ * Struct containing information about a client of the service
+ */
+struct CadetClient
+{
+  /**
+   * Linked list next
+   */
+  struct CadetClient *next;
+
+  /**
+   * Linked list prev
+   */
+  struct CadetClient *prev;
+
+  /**
+   * Tunnels that belong to this client, indexed by local id,
+   * value is a `struct CadetChannel`.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
+
+  /**
+   * Handle to communicate with the client
+   */
+  struct GNUNET_MQ_Handle *mq;
+  
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Ports that this client has declared interest in.
+   * Indexed by port, contains `struct OpenPort`
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *ports;
+
+  /**
+   * Channel ID to use for the next incoming channel for this client.
+   * Wraps around (in theory).
+   */
+  struct GNUNET_CADET_ClientChannelNumber next_ccn;
+
+  /**
+   * ID of the client, mainly for debug messages. Purely internal to this file.
+   */
+  unsigned int id;
+};
 
 
 /******************************************************************************/
@@ -62,37 +107,318 @@
 /****************************** Global variables ******************************/
 
 /**
+ * Handle to our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
  * Handle to the statistics service.
  */
 struct GNUNET_STATISTICS_Handle *stats;
 
 /**
- * Local peer own ID (memory efficient handle).
+ * Handle to communicate with ATS.
  */
-GNUNET_PEER_Id myid;
+struct GNUNET_ATS_ConnectivityHandle *ats_ch;
 
 /**
- * Local peer own ID (full value).
+ * Local peer own ID.
  */
 struct GNUNET_PeerIdentity my_full_id;
 
+/**
+ * Own private key.
+ */
+struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 
 /**
- * Signal that shutdown is happening: prevent recover measures.
+ * Signal that shutdown is happening: prevent recovery measures.
  */
 int shutting_down;
 
-/*************************** Static global variables **************************/
+/**
+ * DLL with all the clients, head.
+ */
+static struct CadetClient *clients_head;
 
 /**
- * Own private key.
+ * DLL with all the clients, tail.
  */
-static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
+static struct CadetClient *clients_tail;
 
+/**
+ * Next ID to assign to a client.
+ */
+static unsigned int next_client_id;
+
+/**
+ * All ports clients of this peer have opened.  Maps from
+ * a hashed port to a `struct OpenPort`.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *open_ports;
+
+/**
+ * Map from ports to channels where the ports were closed at the
+ * time we got the inbound connection.
+ * Indexed by h_port, contains `struct CadetChannel`.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
+
+/**
+ * Map from PIDs to `struct CadetPeer` entries.
+ */
+struct GNUNET_CONTAINER_MultiPeerMap *peers;
+
+/**
+ * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
+ * hash codes to `struct CadetConnection` objects.
+ */
+struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+/**
+ * How many messages are needed to trigger an AXOLOTL ratchet advance.
+ */
+unsigned long long ratchet_messages;
+
+/**
+ * How long until we trigger a ratched advance due to time.
+ */
+struct GNUNET_TIME_Relative ratchet_time;
+
+/**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+struct GNUNET_TIME_Relative keepalive_period;
+
+/**
+ * Set to non-zero values to create random drops to test retransmissions.
+ */
+unsigned long long drop_percent;
+
+
+/**
+ * Send a message to a client.
+ *
+ * @param c client to get the message
+ * @param env envelope with the message
+ */
+void
+GSC_send_to_client (struct CadetClient *c,
+                    struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_MQ_send (c->mq,
+                  env);
+}
+
+
+/**
+ * Return identifier for a client as a string.
+ *
+ * @param c client to identify
+ * @return string for debugging
+ */
+const char *
+GSC_2s (struct CadetClient *c)
+{
+  static char buf[32];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Client(%u)",
+                   c->id);
+  return buf;
+}
+
+
+/**
+ * Lookup channel of client @a c by @a ccn.
+ *
+ * @param c client to look in
+ * @param ccn channel ID to look up
+ * @return NULL if no such channel exists
+ */
+static struct CadetChannel *
+lookup_channel (struct CadetClient *c,
+                struct GNUNET_CADET_ClientChannelNumber ccn)
+{
+  return GNUNET_CONTAINER_multihashmap32_get (c->channels,
+                                              ntohl (ccn.channel_of_client));
+}
+
+
+/**
+ * Obtain the next LID to use for incoming connections to
+ * the given client.
+ *
+ * @param c client handle
+ */
+static struct GNUNET_CADET_ClientChannelNumber
+client_get_next_ccn (struct CadetClient *c)
+{
+  struct GNUNET_CADET_ClientChannelNumber ccn = c->next_ccn;
+
+  /* increment until we have a free one... */
+  while (NULL !=
+         lookup_channel (c,
+                         ccn))
+  {
+    ccn.channel_of_client
+      = htonl (1 + (ntohl (ccn.channel_of_client)));
+    if (ntohl (ccn.channel_of_client) >=
+        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+      ccn.channel_of_client = htonl (0);
+  }
+  c->next_ccn.channel_of_client
+    = htonl (1 + (ntohl (ccn.channel_of_client)));
+  return ccn;
+}
+
+
+/**
+ * Bind incoming channel to this client, and notify client about
+ * incoming connection.  Caller is responsible for notifying the other
+ * peer about our acceptance of the channel.
+ *
+ * @param c client to bind to
+ * @param ch channel to be bound
+ * @param dest peer that establishes the connection
+ * @param port port number
+ * @param options options
+ * @return local channel number assigned to the new client
+ */
+struct GNUNET_CADET_ClientChannelNumber
+GSC_bind (struct CadetClient *c,
+          struct CadetChannel *ch,
+          struct CadetPeer *dest,
+          const struct GNUNET_HashCode *port,
+          uint32_t options)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelCreateMessage *cm;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  ccn = client_get_next_ccn (c);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Accepting incoming %s from %s on open port %s (%u), assigning ccn %X\n",
+       GCCH_2s (ch),
+       GCP_2s (dest),
+       GNUNET_h2s (port),
+       (uint32_t) ntohl (options),
+       (uint32_t) ntohl (ccn.channel_of_client));
+  /* notify local client about incoming connection! */
+  env = GNUNET_MQ_msg (cm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  cm->ccn = ccn;
+  cm->port = *port;
+  cm->opt = htonl (options);
+  cm->peer = *GCP_get_id (dest);
+  GSC_send_to_client (c,
+                      env);
+  return ccn;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_tunnels_now (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
+{
+  struct CadetPeer *cp = value;
+  struct CadetTunnel *t = GCP_get_tunnel (cp,
+                                          GNUNET_NO);
+
+  if (NULL != t)
+    GCT_destroy_tunnel_now (t);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_paths_now (void *cls,
+                   const struct GNUNET_PeerIdentity *pid,
+                   void *value)
+{
+  struct CadetPeer *cp = value;
+
+  GCP_drop_owned_paths (cp);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Shutdown everything once the clients have disconnected.
+ */
+static void
+shutdown_rest ()
+{
+  if (NULL != stats)
+  {
+    GNUNET_STATISTICS_destroy (stats,
+                               GNUNET_NO);
+    stats = NULL;
+  }
+  if (NULL != open_ports)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (open_ports);
+    open_ports = NULL;
+  }
+  if (NULL != loose_channels)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (loose_channels);
+    loose_channels = NULL;
+  }
+  /* Destroy tunnels.  Note that all channels must be destroyed first! */
+  GCP_iterate_all (&destroy_tunnels_now,
+                   NULL);
+  /* All tunnels, channels, connections and CORE must be down before this point. */
+  GCP_iterate_all (&destroy_paths_now,
+                   NULL);
+  /* All paths, tunnels, channels, connections and CORE must be down before this point. */
+  GCP_destroy_all_peers ();
+  if (NULL != peers)
+  {
+    GNUNET_CONTAINER_multipeermap_destroy (peers);
+    peers = NULL;
+  }
+  if (NULL != connections)
+  {
+    GNUNET_CONTAINER_multishortmap_destroy (connections);
+    connections = NULL;
+  }
+  if (NULL != ats_ch)
+  {
+    GNUNET_ATS_connectivity_done (ats_ch);
+    ats_ch = NULL;
+  }
+  GCD_shutdown ();
+  GCH_shutdown ();
+  GNUNET_free_non_null (my_private_key);
+  my_private_key = NULL;
+}
 
-/******************************************************************************/
-/************************      MAIN FUNCTIONS      ****************************/
-/******************************************************************************/
 
 /**
  * Task run during shutdown.
@@ -102,83 +428,1087 @@ static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 static void
 shutdown_task (void *cls)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "shutting down\n");
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Shutting down\n");
   shutting_down = GNUNET_YES;
+  GCO_shutdown ();
+  if (NULL == clients_head)
+    shutdown_rest ();
+}
 
-  GML_shutdown ();
-  GCH_shutdown ();
-  GCC_shutdown ();
-  GCT_shutdown ();
-  GCD_shutdown ();
-  GCP_shutdown ();
 
-  GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
-  stats = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "shut down\n");
+/**
+ * We had a remote connection @a value to port @a h_port before
+ * client @a cls opened port @a port.  Bind them now.
+ *
+ * @param cls the `struct CadetClient`
+ * @param h_port the hashed port
+ * @param value the `struct CadetChannel`
+ * @return #GNUNET_YES (iterate over all such channels)
+ */
+static int
+bind_loose_channel (void *cls,
+                    const struct GNUNET_HashCode *port,
+                    void *value)
+{
+  struct OpenPort *op = cls;
+  struct CadetChannel *ch = value;
+
+  GCCH_bind (ch,
+             op->c,
+             &op->port);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       &op->h_port,
+                                                       ch));
+  return GNUNET_YES;
 }
 
 
 /**
- * Process cadet requests.
+ * Handle port open request.  Creates a mapping from the
+ * port to the respective client and checks whether we have
+ * loose channels trying to bind to the port.  If so, those
+ * are bound.
  *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+handle_port_open (void *cls,
+                  const struct GNUNET_CADET_PortMessage *pmsg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "starting to run\n");
+  struct CadetClient *c = cls;
+  struct OpenPort *op;
 
-  stats = GNUNET_STATISTICS_create ("cadet", c);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Open port %s requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  if (NULL == c->ports)
+    c->ports = GNUNET_CONTAINER_multihashmap_create (4,
+                                                     GNUNET_NO);
+  op = GNUNET_new (struct OpenPort);
+  op->c = c;
+  op->port = pmsg->port;
+  GCCH_hash_port (&op->h_port,
+                  &pmsg->port,
+                  &my_full_id);
+  if (GNUNET_OK !=
+      GNUNET_CONTAINER_multihashmap_put (c->ports,
+                                         &op->port,
+                                         op,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  (void) GNUNET_CONTAINER_multihashmap_put (open_ports,
+                                            &op->h_port,
+                                            op,
+                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  GNUNET_CONTAINER_multihashmap_get_multiple (loose_channels,
+                                              &op->h_port,
+                                              &bind_loose_channel,
+                                              op);
+  GNUNET_SERVICE_client_continue (c->client);
+}
 
-  /* Scheduled the task to clean up when shutdown is called */
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "reading key\n");
-  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
-  GNUNET_assert (NULL != my_private_key);
-  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key, &my_full_id.public_key);
-  myid = GNUNET_PEER_intern (&my_full_id);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "STARTING SERVICE (cadet) for peer [%s]\n",
-              GNUNET_i2s (&my_full_id));
 
-  GML_init (server);    /* Local clients */
-  GCH_init (c);         /* Hellos */
-  GCC_init (c);         /* Connections */
-  GCP_init (c);         /* Peers */
-  GCD_init (c);         /* DHT */
-  GCT_init (c, my_private_key); /* Tunnels */
+/**
+ * Handler for port close requests.  Marks this port as closed
+ * (unless of course we have another client with the same port
+ * open).  Note that existing channels accepted on the port are
+ * not affected.
+ *
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
+ */
+static void
+handle_port_close (void *cls,
+                   const struct GNUNET_CADET_PortMessage *pmsg)
+{
+  struct CadetClient *c = cls;
+  struct OpenPort *op;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Cadet service running\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s as requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  op = GNUNET_CONTAINER_multihashmap_get (c->ports,
+                                          &pmsg->port);
+  if (NULL == op)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                                       &op->port,
+                                                       op));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       &op->h_port,
+                                                       op));
+  GNUNET_free (op);
+  GNUNET_SERVICE_client_continue (c->client);
 }
 
 
 /**
- * The main function for the cadet service.
+ * Handler for requests for us creating a new channel to another peer and port.
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls Identification of the client.
+ * @param tcm The actual message.
  */
-int
-main (int argc, char *const *argv)
+static void
+handle_channel_create (void *cls,
+                       const struct GNUNET_CADET_LocalChannelCreateMessage *tcm)
 {
-  int r;
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
 
-  shutting_down = GNUNET_NO;
-  r = GNUNET_SERVICE_run (argc, argv, "cadet", GNUNET_SERVICE_OPTION_NONE, &run,
-                          NULL);
-  GNUNET_free (my_private_key);
+  if (ntohl (tcm->ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    /* Channel ID not in allowed range. */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  ch = lookup_channel (c,
+                       tcm->ccn);
+  if (NULL != ch)
+  {
+    /* Channel ID already in use. Not allowed. */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "New channel to %s at port %s requested by %s\n",
+       GNUNET_i2s (&tcm->peer),
+       GNUNET_h2s (&tcm->port),
+       GSC_2s (c));
 
-  if (GNUNET_OK != r)
+  /* Create channel */
+  ch = GCCH_channel_local_new (c,
+                               tcm->ccn,
+                               GCP_get (&tcm->peer,
+                                        GNUNET_YES),
+                               &tcm->port,
+                               ntohl (tcm->opt));
+  if (NULL == ch)
   {
-    FPRINTF (stderr, "GNUNET_SERVICE_run for CADET has failed!\n");
-    return 1;
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
   }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (tcm->ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
-  return 0;
+  GNUNET_SERVICE_client_continue (c->client);
 }
+
+
+/**
+ * Handler for requests of destroying an existing channel.
+ *
+ * @param cls client identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_channel_destroy (void *cls,
+                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Client attempted to destroy unknown channel.
+       Can happen if the other side went down at the same time.*/
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s tried to destroy unknown channel %X\n",
+         GSC_2s(c),
+         (uint32_t) ntohl (msg->ccn.channel_of_client));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is destroying %s\n",
+       GSC_2s(c),
+       GCCH_2s (ch));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (msg->ccn.channel_of_client),
+                                                         ch));
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Check for client traffic data message is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is OK, #GNUNET_SYSERR if not
+ */
+static int
+check_local_data (void *cls,
+                  const struct GNUNET_CADET_LocalData *msg)
+{
+  size_t payload_size;
+  size_t payload_claimed_size;
+  const char *buf;
+  struct GNUNET_MessageHeader pa;
+
+  /* FIXME: what is the format we shall allow for @a msg?
+     ONE payload item or multiple? Seems current cadet_api
+     at least in theory allows more than one. Next-gen
+     cadet_api will likely no more, so we could then
+     simplify this mess again. */
+  /* Sanity check for message size */
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  buf = (const char *) &msg[1];
+  while (payload_size >= sizeof (struct GNUNET_MessageHeader))
+  {
+    /* need to memcpy() for alignment */
+    GNUNET_memcpy (&pa,
+                   buf,
+                   sizeof (pa));
+    payload_claimed_size = ntohs (pa.size);
+    if ( (payload_size < payload_claimed_size) ||
+         (payload_claimed_size < sizeof (struct GNUNET_MessageHeader)) ||
+         (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size) )
+    {
+      GNUNET_break (0);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Local data of %u total size had sub-message %u at %u with %u bytes\n",
+           ntohs (msg->header.size),
+           ntohs (pa.type),
+           (unsigned int) (buf - (const char *) &msg[1]),
+           (unsigned int) payload_claimed_size);
+      return GNUNET_SYSERR;
+    }
+    payload_size -= payload_claimed_size;
+    buf += payload_claimed_size;
+  }
+  if (0 != payload_size)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handler for client payload traffic to be send on a channel to
+ * another peer.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_local_data (void *cls,
+                   const struct GNUNET_CADET_LocalData *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+  size_t payload_size;
+  const char *buf;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Dropping payload for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  GNUNET_STATISTICS_update (stats,
+                            "# payload received from clients",
+                            payload_size,
+                            GNUNET_NO);
+  buf = (const char *) &msg[1];
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received %u bytes payload from %s for %s\n",
+       (unsigned int) payload_size,
+       GSC_2s (c),
+       GCCH_2s (ch));
+  if (GNUNET_OK !=
+      GCCH_handle_local_data (ch,
+                              msg->ccn,
+                              buf,
+                              payload_size))
+  {
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for client's ACKs for payload traffic.
+ *
+ * @param cls identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_local_ack (void *cls,
+                  const struct GNUNET_CADET_LocalAck *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Ignoring local ACK for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got a local ACK from %s for %s\n",
+       GSC_2s(c),
+       GCCH_2s (ch));
+  GCCH_handle_local_ack (ch,
+                         msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all peers to send a monitoring client info about each peer.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value Peer info.
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_peers_iterator (void *cls,
+                        const struct GNUNET_PeerIdentity *peer,
+                        void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoPeer *msg;
+
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  msg->destination = *peer;
+  msg->paths = htons (GCP_count_paths (p));
+  msg->tunnel = htons (NULL != GCP_get_tunnel (p,
+                                               GNUNET_NO));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's INFO PEERS request.
+ *
+ * @param cls Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_get_peers (void *cls,
+                  const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_peers_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all paths of a peer to build an InfoPeer message.
+ * Message contains blocks of peers, first not included.
+ *
+ * @param cls message queue for transmission
+ * @param path Path itself
+ * @param off offset of the peer on @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
+ */
+static int
+path_info_iterator (void *cls,
+                    struct CadetPeerPath *path,
+                    unsigned int off)
+{
+  struct GNUNET_MQ_Handle *mq = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *resp;
+  struct GNUNET_PeerIdentity *id;
+  uint16_t path_size;
+  unsigned int i;
+  unsigned int path_length;
+
+  path_length = GCPP_get_length (path);
+  path_size = sizeof (struct GNUNET_PeerIdentity) * (path_length - 1);
+  if (sizeof (*resp) + path_size > UINT16_MAX)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Path of %u entries is too long for info message\n",
+         path_length);
+    return GNUNET_YES;
+  }
+  env = GNUNET_MQ_msg_extra (resp,
+                             path_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
+  id = (struct GNUNET_PeerIdentity *) &resp[1];
+
+  /* Don't copy first peer.  First peer is always the local one.  Last
+   * peer is always the destination (leave as 0, EOL).
+   */
+  for (i = 0; i < off; i++)
+    id[i] = *GCP_get_id (GCPP_get_peer_at_offset (path,
+                                                  i + 1));
+  GNUNET_MQ_send (mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's SHOW_PEER request.
+ *
+ * @param cls Identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_show_peer (void *cls,
+                  const struct GNUNET_CADET_LocalInfo *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *resp;
+
+  p = GCP_get (&msg->peer,
+               GNUNET_NO);
+  if (NULL != p)
+    GCP_iterate_paths (p,
+                       &path_info_iterator,
+                       c->mq);
+  /* Send message with 0/0 to indicate the end */
+  env = GNUNET_MQ_msg (resp,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER_END);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all tunnels to send a monitoring client info about each tunnel.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value a `struct CadetPeer`
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_tunnels_iterator (void *cls,
+                          const struct GNUNET_PeerIdentity *peer,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoTunnel *msg;
+  struct CadetTunnel *t;
+
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL == t)
+    return GNUNET_YES;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  msg->destination = *peer;
+  msg->channels = htonl (GCT_count_channels (t));
+  msg->connections = htonl (GCT_count_any_connections (t));
+  msg->cstate = htons (0);
+  msg->estate = htons ((uint16_t) GCT_get_estate (t));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS request.
+ *
+ * @param cls client Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_info_tunnels (void *cls,
+                     const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_tunnels_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Update the message with information about the connection.
+ *
+ * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
+ * @param ct a connection about which we should store information in @a cls
+ */
+static void
+iter_connection (void *cls,
+                 struct CadetTConnection *ct)
+{
+  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
+  struct CadetConnection *cc = ct->cc;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
+
+  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
+  h[msg->connections++] = *(GCC_get_id (cc));
+}
+
+
+/**
+ * Update the message with information about the channel.
+ *
+ * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
+ * @param ch a channel about which we should store information in @a cls
+ */
+static void
+iter_channel (void *cls,
+              struct CadetChannel *ch)
+{
+  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
+  struct GNUNET_CADET_ChannelTunnelNumber *chn
+    = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
+
+  chn[msg->channels++] = GCCH_get_id (ch);
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL request.
+ *
+ * @param cls Identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_info_tunnel (void *cls,
+                    const struct GNUNET_CADET_LocalInfo *msg)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoTunnel *resp;
+  struct CadetTunnel *t;
+  struct CadetPeer *p;
+  unsigned int ch_n;
+  unsigned int c_n;
+
+  p = GCP_get (&msg->peer,
+               GNUNET_NO);
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL == t)
+  {
+    /* We don't know the tunnel */
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_LocalInfoTunnel *warn;
+
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Tunnel to %s unknown\n",
+         GNUNET_i2s_full (&msg->peer));
+    env = GNUNET_MQ_msg (warn,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+    warn->destination = msg->peer;
+    GNUNET_MQ_send (c->mq,
+                    env);
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+
+  /* Initialize context */
+  ch_n = GCT_count_channels (t);
+  c_n = GCT_count_any_connections (t);
+  env = GNUNET_MQ_msg_extra (resp,
+                             c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier) +
+                             ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber),
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+  resp->destination = msg->peer;
+  /* Do not reorder! #iter_channel needs counters in HBO! */
+  GCT_iterate_connections (t,
+                           &iter_connection,
+                           resp);
+  GCT_iterate_channels (t,
+                        &iter_channel,
+                        resp);
+  resp->connections = htonl (resp->connections);
+  resp->channels = htonl (resp->channels);
+  resp->cstate = htons (0);
+  resp->estate = htons (GCT_get_estate (t));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all peers to dump info for each peer.
+ *
+ * @param cls Closure (unused).
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value Peer info.
+ *
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+show_peer_iterator (void *cls,
+                    const struct GNUNET_PeerIdentity *peer,
+                    void *value)
+{
+  struct CadetPeer *p = value;
+  struct CadetTunnel *t;
+
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL != t)
+    GCT_debug (t,
+               GNUNET_ERROR_TYPE_ERROR);
+  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's INFO_DUMP request.
+ *
+ * @param cls Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_info_dump (void *cls,
+                  const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received dump info request from client %u\n",
+       c->id);
+
+  LOG (GNUNET_ERROR_TYPE_ERROR,
+       "*************************** DUMP START ***************************\n");
+  for (struct CadetClient *ci = clients_head;
+       NULL != ci;
+       ci = ci->next)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         "Client %u (%p), handle: %p, ports: %u, channels: %u\n",
+         ci->id,
+         ci,
+         ci->client,
+         (NULL != c->ports)
+         ? GNUNET_CONTAINER_multihashmap_size (ci->ports)
+         : 0,
+         GNUNET_CONTAINER_multihashmap32_size (ci->channels));
+  }
+  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
+  GCP_iterate_all (&show_peer_iterator,
+                   NULL);
+
+  LOG (GNUNET_ERROR_TYPE_ERROR,
+       "**************************** DUMP END ****************************\n");
+
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+
+/**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param client the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *client,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct CadetClient *c;
+
+  c = GNUNET_new (struct CadetClient);
+  c->client = client;
+  c->mq = mq;
+  c->id = next_client_id++; /* overflow not important: just for debug */
+  c->channels
+    = GNUNET_CONTAINER_multihashmap32_create (32);
+  GNUNET_CONTAINER_DLL_insert (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            +1,
+                            GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s connected\n",
+       GSC_2s (c));
+  return c;
+}
+
+
+/**
+ * A channel was destroyed by the other peer. Tell our client.
+ *
+ * @param c client that lost a channel
+ * @param ccn channel identification number for the client
+ * @param ch the channel object
+ */
+void
+GSC_handle_remote_channel_destroy (struct CadetClient *c,
+                                   struct GNUNET_CADET_ClientChannelNumber ccn,
+                                   struct CadetChannel *ch)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelDestroyMessage *tdm;
+
+  env = GNUNET_MQ_msg (tdm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+  tdm->ccn = ccn;
+  GSC_send_to_client (c,
+                      env);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (ccn.channel_of_client),
+                                                         ch));
+}
+
+
+/**
+ * A client that created a loose channel that was not bound to a port
+ * disconnected, drop it from the #loose_channels list.
+ *
+ * @param h_port the hashed port the channel was trying to bind to
+ * @param ch the channel that was lost
+ */
+void
+GSC_drop_loose_channel (const struct GNUNET_HashCode *h_port,
+                        struct CadetChannel *ch)
+{
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       h_port,
+                                                       ch));
+}
+
+
+/**
+ * Iterator for deleting each channel whose client endpoint disconnected.
+ *
+ * @param cls Closure (client that has disconnected).
+ * @param key The local channel id in host byte order
+ * @param value The value stored at the key (channel to destroy).
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+channel_destroy_iterator (void *cls,
+                          uint32_t key,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+  struct CadetChannel *ch = value;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying %s, due to %s disconnecting.\n",
+       GCCH_2s (ch),
+       GSC_2s (c));
+  ccn.channel_of_client = htonl (key);
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              ccn);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         key,
+                                                         ch));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Remove client's ports from the global hashmap on disconnect.
+ *
+ * @param cls the `struct CadetClient`
+ * @param port the port.
+ * @param value the `struct OpenPort` to remove
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+client_release_ports (void *cls,
+                      const struct GNUNET_HashCode *port,
+                      void *value)
+{
+  struct CadetClient *c = cls;
+  struct OpenPort *op = value;
+
+  GNUNET_assert (c == op->c);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s due to %s disconnect.\n",
+       GNUNET_h2s (port),
+       GSC_2s (c));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       &op->h_port,
+                                                       op));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                                       port,
+                                                       op));
+  GNUNET_free (op);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param client the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
+{
+  struct CadetClient *c = internal_cls;
+
+  GNUNET_assert (c->client == client);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is disconnecting.\n",
+       GSC_2s (c));
+  if (NULL != c->channels)
+  {
+    GNUNET_CONTAINER_multihashmap32_iterate (c->channels,
+                                             &channel_destroy_iterator,
+                                             c);
+    GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (c->channels));
+    GNUNET_CONTAINER_multihashmap32_destroy (c->channels);
+  }
+  if (NULL != c->ports)
+  {
+    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
+                                           &client_release_ports,
+                                           c);
+    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
+  }
+  GNUNET_CONTAINER_DLL_remove (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            -1,
+                            GNUNET_NO);
+  GNUNET_free (c);
+  if ( (NULL == clients_head) &&
+       (GNUNET_YES == shutting_down) )
+    shutdown_rest ();
+}
+
+
+/**
+ * Setup CADET internals.
+ *
+ * @param cls closure
+ * @param server the initialized server
+ * @param c configuration to use
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
+{
+  cfg = c;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "RATCHET_MESSAGES",
+                                             &ratchet_messages))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_MESSAGES",
+                               "needs to be a number");
+    ratchet_messages = 64;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "RATCHET_TIME",
+                                           &ratchet_time))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_TIME",
+                               "need delay value");
+    ratchet_time = GNUNET_TIME_UNIT_HOURS;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "REFRESH_CONNECTION_TIME",
+                                           &keepalive_period))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "REFRESH_CONNECTION_TIME",
+                               "need delay value");
+    keepalive_period = GNUNET_TIME_UNIT_MINUTES;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "DROP_PERCENT",
+                                             &drop_percent))
+  {
+    drop_percent = 0;
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+  }
+  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
+  if (NULL == my_private_key)
+  {
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key,
+                                      &my_full_id.public_key);
+  stats = GNUNET_STATISTICS_create ("cadet",
+                                    c);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  ats_ch = GNUNET_ATS_connectivity_init (c);
+  /* FIXME: optimize code to allow GNUNET_YES here! */
+  open_ports = GNUNET_CONTAINER_multihashmap_create (16,
+                                                     GNUNET_NO);
+  loose_channels = GNUNET_CONTAINER_multihashmap_create (16,
+                                                         GNUNET_NO);
+  peers = GNUNET_CONTAINER_multipeermap_create (16,
+                                                GNUNET_YES);
+  connections = GNUNET_CONTAINER_multishortmap_create (256,
+                                                       GNUNET_YES);
+  GCH_init (c);
+  GCD_init (c);
+  GCO_init (c);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "CADET started for peer %s\n",
+              GNUNET_i2s (&my_full_id));
+
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("cadet",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (port_open,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
+                          struct GNUNET_CADET_PortMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (port_close,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
+                          struct GNUNET_CADET_PortMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (channel_create,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
+                          struct GNUNET_CADET_LocalChannelCreateMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (channel_destroy,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
+                          struct GNUNET_CADET_LocalChannelDestroyMessage,
+                          NULL),
+ GNUNET_MQ_hd_var_size (local_data,
+                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                        struct GNUNET_CADET_LocalData,
+                        NULL),
+ GNUNET_MQ_hd_fixed_size (local_ack,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
+                          struct GNUNET_CADET_LocalAck,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (get_peers,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (show_peer,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
+                          struct GNUNET_CADET_LocalInfo,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_tunnels,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_tunnel,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
+                          struct GNUNET_CADET_LocalInfo,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_dump,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_handler_end ());
+
+/* end of gnunet-service-cadet-new.c */
diff --git a/src/cadet/gnunet-service-cadet-new.h b/src/cadet/gnunet-service-cadet.h
index 4a76c578b..162867823 100644
--- a/src/cadet/gnunet-service-cadet-new.h
+++ b/src/cadet/gnunet-service-cadet.h
@@ -20,7 +20,7 @@
 */
 
 /**
- * @file cadet/gnunet-service-cadet-new.h
+ * @file cadet/gnunet-service-cadet.h
  * @brief Information we track per peer.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
@@ -29,7 +29,6 @@
 #define GNUNET_SERVICE_CADET_H
 
 #include "gnunet_util_lib.h"
-#define NEW_CADET 1
 #include "cadet_protocol.h"
 
 /**
@@ -146,6 +145,30 @@ struct CadetTConnection
 
 
 /**
+ * Port opened by a client.
+ */
+struct OpenPort
+{
+
+  /**
+   * Client that opened the port.
+   */
+  struct CadetClient *c;
+
+  /**
+   * Port number.
+   */
+  struct GNUNET_HashCode port;
+
+  /**
+   * Port hashed with our PID (matches incoming OPEN messages).
+   */
+  struct GNUNET_HashCode h_port;
+  
+};
+
+
+/**
  * Active path through the network (used by a tunnel).  There may
  * be at most one connection per path.
  */
@@ -193,7 +216,8 @@ extern struct GNUNET_PeerIdentity my_full_id;
 extern struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 
 /**
- * All ports clients of this peer have opened.
+ * All ports clients of this peer have opened.  Maps from
+ * a hashed port to a `struct OpenPort`.
  */
 extern struct GNUNET_CONTAINER_MultiHashMap *open_ports;
 
@@ -206,7 +230,7 @@ extern struct GNUNET_CONTAINER_MultiShortmap *connections;
 /**
  * Map from ports to channels where the ports were closed at the
  * time we got the inbound connection.
- * Indexed by port, contains `struct CadetChannel`.
+ * Indexed by h_port, contains `struct CadetChannel`.
  */
 extern struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
 
@@ -226,10 +250,20 @@ extern unsigned long long ratchet_messages;
 extern struct GNUNET_TIME_Relative ratchet_time;
 
 /**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+extern struct GNUNET_TIME_Relative keepalive_period;
+
+/**
  * Signal that shutdown is happening: prevent recovery measures.
  */
 extern int shutting_down;
 
+/**
+ * Set to non-zero values to create random drops to test retransmissions.
+ */
+extern unsigned long long drop_percent;
+
 
 /**
  * Send a message to a client.
@@ -254,6 +288,17 @@ GSC_handle_remote_channel_destroy (struct CadetClient *c,
                                    struct GNUNET_CADET_ClientChannelNumber ccn,
                                    struct CadetChannel *ch);
 
+/**
+ * A client that created a loose channel that was not bound to a port
+ * disconnected, drop it from the #loose_channels list.
+ *
+ * @param h_port the hashed port the channel was trying to bind to
+ * @param ch the channel that was lost
+ */
+void
+GSC_drop_loose_channel (const struct GNUNET_HashCode *h_port,
+                        struct CadetChannel *ch);
+
 
 /**
  * Bind incoming channel to this client, and notify client
diff --git a/src/cadet/gnunet-service-cadet_channel.c b/src/cadet/gnunet-service-cadet_channel.c
index 7b7c6e57c..739b68228 100644
--- a/src/cadet/gnunet-service-cadet_channel.c
+++ b/src/cadet/gnunet-service-cadet_channel.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,31 +17,63 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
-
+/**
+ * @file cadet/gnunet-service-cadet_channel.c
+ * @brief logical links between CADET clients
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - Congestion/flow control:
+ *   + estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+ *     (and figure out how/where to use this!)
+ *   + figure out flow control without ACKs (unreliable traffic!)
+ * - revisit handling of 'unbuffered' traffic!
+ *   (need to push down through tunnel into connection selection)
+ * - revisit handling of 'buffered' traffic: 4 is a rather small buffer; maybe
+ *   reserve more bits in 'options' to allow for buffer size control?
+ */
 #include "platform.h"
-#include "gnunet_util_lib.h"
-
+#include "cadet.h"
 #include "gnunet_statistics_service.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_paths.h"
 
-#include "cadet.h"
-#include "cadet_protocol.h"
+#define LOG(level,...) GNUNET_log_from (level,"cadet-chn",__VA_ARGS__)
 
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_local.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_peer.h"
+/**
+ * How long do we initially wait before retransmitting?
+ */
+#define CADET_INITIAL_RETRANSMIT_TIME GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
 
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-chn",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
+/**
+ * How long do we wait before dropping state about incoming
+ * connection to closed port?
+ */
+#define TIMEOUT_CLOSED_PORT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
 
-#define CADET_RETRANSMIT_TIME    GNUNET_TIME_relative_multiply(\
-                                    GNUNET_TIME_UNIT_MILLISECONDS, 250)
-#define CADET_RETRANSMIT_MARGIN  4
+/**
+ * How long do we wait at least before retransmitting ever?
+ */
+#define MIN_RTT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 75)
+
+/**
+ * Maximum message ID into the future we accept for out-of-order messages.
+ * If the message is more than this into the future, we drop it.  This is
+ * important both to detect values that are actually in the past, as well
+ * as to limit adversarially triggerable memory consumption.
+ *
+ * Note that right now we have "max_pending_messages = 4" hard-coded in
+ * the logic below, so a value of 4 would suffice here. But we plan to
+ * allow larger windows in the future...
+ */
+#define MAX_OUT_OF_ORDER_DISTANCE 1024
 
 
 /**
- * All the states a connection can be in.
+ * All the states a channel can be in.
  */
 enum CadetChannelState
 {
@@ -51,9 +83,15 @@ enum CadetChannelState
   CADET_CHANNEL_NEW,
 
   /**
-   * Connection create message sent, waiting for ACK.
+   * Channel is to a port that is not open, we're waiting for the
+   * port to be opened.
+   */
+  CADET_CHANNEL_LOOSE,
+
+  /**
+   * CHANNEL_OPEN message sent, waiting for CHANNEL_OPEN_ACK.
    */
-  CADET_CHANNEL_SENT,
+  CADET_CHANNEL_OPEN_SENT,
 
   /**
    * Connection confirmed, ready to carry traffic.
@@ -63,138 +101,144 @@ enum CadetChannelState
 
 
 /**
- * Info holder for channel messages in queues.
+ * Info needed to retry a message in case it gets lost.
+ * Note that we DO use this structure also for unreliable
+ * messages.
  */
-struct CadetChannelQueue
+struct CadetReliableMessage
 {
   /**
-   * Tunnel Queue.
+   * Double linked list, FIFO style
    */
-  struct CadetTunnelQueue *tq;
+  struct CadetReliableMessage *next;
 
   /**
-   * Message type (DATA/DATA_ACK)
+   * Double linked list, FIFO style
    */
-  uint16_t type;
+  struct CadetReliableMessage *prev;
 
   /**
-   * Message copy (for DATAs, to start retransmission timer)
+   * Which channel is this message in?
    */
-  struct CadetReliableMessage *copy;
+  struct CadetChannel *ch;
 
   /**
-   * Reliability (for DATA_ACKs, to access rel->ack_q)
+   * Entry in the tunnels queue for this message, NULL if it has left
+   * the tunnel.  Used to cancel transmission in case we receive an
+   * ACK in time.
    */
-  struct CadetChannelReliability *rel;
-};
-
+  struct CadetTunnelQueueEntry *qe;
 
-/**
- * Info needed to retry a message in case it gets lost.
- */
-struct CadetReliableMessage
-{
   /**
-   * Double linked list, FIFO style
+   * Data message we are trying to send.
    */
-  struct CadetReliableMessage   *next;
-  struct CadetReliableMessage   *prev;
+  struct GNUNET_CADET_ChannelAppDataMessage *data_message;
 
   /**
-   * Type of message (payload, channel management).
+   * How soon should we retry if we fail to get an ACK?
+   * Messages in the queue are sorted by this value.
    */
-  int16_t                       type;
+  struct GNUNET_TIME_Absolute next_retry;
 
   /**
-   * Tunnel Reliability queue this message is in.
+   * How long do we wait for an ACK after transmission?
+   * Use for the back-off calculation.
    */
-  struct CadetChannelReliability *rel;
+  struct GNUNET_TIME_Relative retry_delay;
 
   /**
-   * ID of the message (ACK needed to free)
+   * Time when we first successfully transmitted the message
+   * (that is, set @e num_transmissions to 1).
    */
-  uint32_t                      mid;
+  struct GNUNET_TIME_Absolute first_transmission_time;
 
   /**
-   * Tunnel Queue.
+   * Identifier of the connection that this message took when it
+   * was first transmitted.  Only useful if @e num_transmissions is 1.
    */
-  struct CadetChannelQueue      *chq;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier connection_taken;
 
   /**
-   * When was this message issued (to calculate ACK delay)
+   * How often was this message transmitted?  #GNUNET_SYSERR if there
+   * was an error transmitting the message, #GNUNET_NO if it was not
+   * yet transmitted ever, otherwise the number of (re) transmissions.
    */
-  struct GNUNET_TIME_Absolute   timestamp;
+  int num_transmissions;
 
-  /* struct GNUNET_CADET_ChannelAppDataMessage with payload */
 };
 
 
 /**
- * Info about the traffic state for a client in a channel.
+ * List of received out-of-order data messages.
  */
-struct CadetChannelReliability
+struct CadetOutOfOrderMessage
 {
   /**
-   * Channel this is about.
+   * Double linked list, FIFO style
    */
-  struct CadetChannel *ch;
+  struct CadetOutOfOrderMessage *next;
 
   /**
-   * DLL of messages sent and not yet ACK'd.
+   * Double linked list, FIFO style
    */
-  struct CadetReliableMessage        *head_sent;
-  struct CadetReliableMessage        *tail_sent;
+  struct CadetOutOfOrderMessage *prev;
 
   /**
-   * DLL of messages received out of order.
+   * ID of the message (messages up to this point needed
+   * before we give this one to the client).
    */
-  struct CadetReliableMessage        *head_recv;
-  struct CadetReliableMessage        *tail_recv;
+  struct ChannelMessageIdentifier mid;
 
   /**
-   * Messages received.
+   * The envelope with the payload of the out-of-order message
    */
-  unsigned int                      n_recv;
+  struct GNUNET_MQ_Envelope *env;
 
-  /**
-   * Next MID to use for outgoing traffic.
-   */
-  uint32_t                          mid_send;
+};
 
-  /**
-   * Next MID expected for incoming traffic.
-   */
-  uint32_t                          mid_recv;
 
+/**
+ * Client endpoint of a `struct CadetChannel`.  A channel may be a
+ * loopback channel, in which case it has two of these endpoints.
+ * Note that flow control also is required in both directions.
+ */
+struct CadetChannelClient
+{
   /**
-   * Handle for queued unique data CREATE, DATA_ACK.
+   * Client handle.  Not by itself sufficient to designate
+   * the client endpoint, as the same client handle may
+   * be used for both the owner and the destination, and
+   * we thus also need the channel ID to identify the client.
    */
-  struct CadetChannelQueue           *uniq;
+  struct CadetClient *c;
 
   /**
-   * Can we send data to the client?
+   * Head of DLL of messages received out of order or while client was unready.
    */
-  int                               client_ready;
+  struct CadetOutOfOrderMessage *head_recv;
 
   /**
-   * Can the client send data to us?
+   * Tail DLL of messages received out of order or while client was unready.
    */
-  int                               client_allowed;
+  struct CadetOutOfOrderMessage *tail_recv;
 
   /**
-   * Task to resend/poll in case no ACK is received.
+   * Local tunnel number for this client.
+   * (if owner >= #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI,
+   *  otherwise < #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
    */
-  struct GNUNET_SCHEDULER_Task *   retry_task;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
   /**
-   * Counter for exponential backoff.
+   * Number of entries currently in @a head_recv DLL.
    */
-  struct GNUNET_TIME_Relative       retry_timer;
+  unsigned int num_recv;
 
   /**
-   * How long does it usually take to get an ACK.
+   * Can we send data to the client?
    */
-  struct GNUNET_TIME_Relative       expected_delay;
+  int client_ready;
+
 };
 
 
@@ -209,41 +253,44 @@ struct CadetChannel
   struct CadetTunnel *t;
 
   /**
-   * Destination port of the channel.
+   * Client owner of the tunnel, if any.
+   * (Used if this channel represends the initiating end of the tunnel.)
    */
-  struct GNUNET_HashCode port;
+  struct CadetChannelClient *owner;
 
   /**
-   * Global channel number ( < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+   * Client destination of the tunnel, if any.
+   * (Used if this channel represents the listening end of the tunnel.)
    */
-  struct GNUNET_CADET_ChannelTunnelNumber gid;
+  struct CadetChannelClient *dest;
 
   /**
-   * Local tunnel number for root (owner) client.
-   * ( >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI or 0 )
+   * Last entry in the tunnel's queue relating to control messages
+   * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN or
+   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK).  Used to cancel
+   * transmission in case we receive updated information.
    */
-  struct GNUNET_CADET_ClientChannelNumber lid_root;
+  struct CadetTunnelQueueEntry *last_control_qe;
 
   /**
-   * Local tunnel number for local destination clients (incoming number)
-   * ( >= GNUNET_CADET_LOCAL_CHANNEL_ID_SERV or 0).
+   * Head of DLL of messages sent and not yet ACK'd.
    */
-  struct GNUNET_CADET_ClientChannelNumber lid_dest;
+  struct CadetReliableMessage *head_sent;
 
   /**
-   * Channel state.
+   * Tail of DLL of messages sent and not yet ACK'd.
    */
-  enum CadetChannelState state;
+  struct CadetReliableMessage *tail_sent;
 
   /**
-   * Is the tunnel bufferless (minimum latency)?
+   * Task to resend/poll in case no ACK is received.
    */
-  int nobuffer;
+  struct GNUNET_SCHEDULER_Task *retry_control_task;
 
   /**
-   * Is the tunnel reliable?
+   * Task to resend/poll in case no ACK is received.
    */
-  int reliable;
+  struct GNUNET_SCHEDULER_Task *retry_data_task;
 
   /**
    * Last time the channel was used
@@ -251,21 +298,34 @@ struct CadetChannel
   struct GNUNET_TIME_Absolute timestamp;
 
   /**
-   * Client owner of the tunnel, if any
+   * Destination port of the channel.
    */
-  struct CadetClient *root;
+  struct GNUNET_HashCode port;
 
   /**
-   * Client destination of the tunnel, if any.
+   * Hash'ed port of the channel with initiator and destination PID.
    */
-  struct CadetClient *dest;
+  struct GNUNET_HashCode h_port;
 
   /**
-   * Flag to signal the destruction of the channel.
-   * If this is set to #GNUNET_YES the channel will be destroyed
-   * when the queue is empty.
+   * Counter for exponential backoff.
    */
-  int destroy;
+  struct GNUNET_TIME_Relative retry_time;
+
+  /**
+   * Bitfield of already-received messages past @e mid_recv.
+   */
+  uint64_t mid_futures;
+
+  /**
+   * Next MID expected for incoming traffic.
+   */
+  struct ChannelMessageIdentifier mid_recv;
+
+  /**
+   * Next MID to use for outgoing traffic.
+   */
+  struct ChannelMessageIdentifier mid_send;
 
   /**
    * Total (reliable) messages pending ACK for this channel.
@@ -273,2290 +333,1763 @@ struct CadetChannel
   unsigned int pending_messages;
 
   /**
-   * Reliability data.
-   * Only present (non-NULL) at the owner of a tunnel.
+   * Maximum (reliable) messages pending ACK for this channel
+   * before we throttle the client.
    */
-  struct CadetChannelReliability *root_rel;
+  unsigned int max_pending_messages;
 
   /**
-   * Reliability data.
-   * Only present (non-NULL) at the destination of a tunnel.
+   * Number identifying this channel in its tunnel.
    */
-  struct CadetChannelReliability *dest_rel;
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
 
-};
+  /**
+   * Channel state.
+   */
+  enum CadetChannelState state;
+
+  /**
+   * Count how many ACKs we skipped, used to prevent long
+   * sequences of ACK skipping.
+   */
+  unsigned int skip_ack_series;
 
+  /**
+   * Is the tunnel bufferless (minimum latency)?
+   */
+  int nobuffer;
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
+  /**
+   * Is the tunnel reliable?
+   */
+  int reliable;
 
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
+  /**
+   * Is the tunnel out-of-order?
+   */
+  int out_of_order;
 
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
+  /**
+   * Is this channel a loopback channel, where the destination is us again?
+   */
+  int is_loopback;
 
+  /**
+   * Flag to signal the destruction of the channel.  If this is set to
+   * #GNUNET_YES the channel will be destroyed once the queue is
+   * empty.
+   */
+  int destroy;
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
+};
 
 
 /**
- * Destroy a reliable message after it has been acknowledged, either by
- * direct mid ACK or bitfield. Updates the appropriate data structures and
- * timers and frees all memory.
+ * Get the static string for identification of the channel.
  *
- * @param copy Message that is no longer needed: remote peer got it.
- * @param update_time Is the timing information relevant?
- *                    If this message is ACK in a batch the timing information
- *                    is skewed by the retransmission, count only for the
- *                    retransmitted message.
+ * @param ch Channel.
  *
- * @return #GNUNET_YES if channel was destroyed as a result of the call,
- *         #GNUNET_NO otherwise.
+ * @return Static string with the channel IDs.
  */
-static int
-rel_message_free (struct CadetReliableMessage *copy, int update_time);
+const char *
+GCCH_2s (const struct CadetChannel *ch)
+{
+  static char buf[128];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Channel %s:%s ctn:%X(%X/%X)",
+                   (GNUNET_YES == ch->is_loopback)
+                   ? "loopback"
+                   : GNUNET_i2s (GCP_get_id (GCT_get_destination (ch->t))),
+                   GNUNET_h2s (&ch->port),
+                   ch->ctn,
+                   (NULL == ch->owner) ? 0 : ntohl (ch->owner->ccn.channel_of_client),
+                   (NULL == ch->dest) ? 0 : ntohl (ch->dest->ccn.channel_of_client));
+  return buf;
+}
 
-/**
- * send a channel create message.
- *
- * @param ch Channel for which to send.
- */
-static void
-send_create (struct CadetChannel *ch);
 
 /**
- * Confirm we got a channel create, FWD ack.
+ * Hash the @a port and @a initiator and @a listener to 
+ * calculate the "challenge" @a h_port we send to the other
+ * peer on #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN.
  *
- * @param ch The channel to confirm.
- * @param fwd Should we send a FWD ACK? (going dest->root)
+ * @param[out] h_port set to the hash of @a port, @a initiator and @a listener
+ * @param port cadet port, as seen by CADET clients
+ * @param listener peer that is listining on @a port
  */
-static void
-send_ack (struct CadetChannel *ch, int fwd);
-
+void
+GCCH_hash_port (struct GNUNET_HashCode *h_port,
+                const struct GNUNET_HashCode *port,
+                const struct GNUNET_PeerIdentity *listener)
+{
+  struct GNUNET_HashContext *hc;
+
+  hc = GNUNET_CRYPTO_hash_context_start ();
+  GNUNET_CRYPTO_hash_context_read (hc,
+                                   port,
+                                   sizeof (*port));
+  GNUNET_CRYPTO_hash_context_read (hc,
+                                   listener,
+                                   sizeof (*listener));
+  GNUNET_CRYPTO_hash_context_finish (hc,
+                                     h_port);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Calculated port hash %s\n",
+       GNUNET_h2s (h_port));
+}
 
 
 /**
- * Test if the channel is loopback: both root and dest are on the local peer.
+ * Get the channel's public ID.
  *
- * @param ch Channel to test.
+ * @param ch Channel.
  *
- * @return #GNUNET_YES if channel is loopback, #GNUNET_NO otherwise.
+ * @return ID used to identify the channel with the remote peer.
  */
-static int
-is_loopback (const struct CadetChannel *ch)
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch)
 {
-  if (NULL != ch->t)
-    return GCT_is_loopback (ch->t);
-
-  return (NULL != ch->root && NULL != ch->dest);
+  return ch->ctn;
 }
 
 
 /**
- * Save a copy of the data message for later retransmission.
+ * Release memory associated with @a ccc
  *
- * @param msg Message to copy.
- * @param mid Message ID.
- * @param rel Reliability data for retransmission.
+ * @param ccc data structure to clean up
  */
-static struct CadetReliableMessage *
-copy_message (const struct GNUNET_CADET_ChannelAppDataMessage *msg, uint32_t mid,
-              struct CadetChannelReliability *rel)
+static void
+free_channel_client (struct CadetChannelClient *ccc)
 {
-  struct CadetReliableMessage *copy;
-  uint16_t size;
-
-  size = ntohs (msg->header.size);
-  copy = GNUNET_malloc (sizeof (*copy) + size);
-  copy->mid = mid;
-  copy->rel = rel;
-  copy->type = GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA;
-  GNUNET_memcpy (&copy[1], msg, size);
+  struct CadetOutOfOrderMessage *com;
 
-  return copy;
+  while (NULL != (com = ccc->head_recv))
+  {
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+  }
+  GNUNET_free (ccc);
 }
 
+
 /**
- * We have received a message out of order, or the client is not ready.
- * Buffer it until we receive an ACK from the client or the missing
- * message from the channel.
+ * Destroy the given channel.
  *
- * @param msg Message to buffer (MUST be of type CADET_DATA).
- * @param rel Reliability data to the corresponding direction.
+ * @param ch channel to destroy
  */
 static void
-add_buffered_data (const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                   struct CadetChannelReliability *rel)
+channel_destroy (struct CadetChannel *ch)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *prev;
-  uint32_t mid;
+  struct CadetReliableMessage *crm;
 
-  mid = ntohl (msg->mid);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add_buffered_data MID %u (%u)\n",
-       mid, rel->n_recv);
-
-  rel->n_recv++;
-
-  // FIXME do something better than O(n), although n < 64...
-  // FIXME start from the end (most messages are the latest ones)
-  for (prev = rel->head_recv; NULL != prev; prev = prev->next)
+  while (NULL != (crm = ch->head_sent))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " prev %u\n", prev->mid);
-    if (prev->mid == mid)
+    GNUNET_assert (ch == crm->ch);
+    if (NULL != crm->qe)
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " already there!\n");
-      rel->n_recv--;
-      return;
-    }
-    else if (GC_is_pid_bigger (prev->mid, mid))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " bingo!\n");
-      copy = copy_message (msg, mid, rel);
-      GNUNET_CONTAINER_DLL_insert_before (rel->head_recv, rel->tail_recv,
-                                          prev, copy);
-      return;
+      GCT_send_cancel (crm->qe);
+      crm->qe = NULL;
     }
+    GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                                 ch->tail_sent,
+                                 crm);
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
   }
-  copy = copy_message (msg, mid, rel);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " insert at tail! (now: %u)\n", rel->n_recv);
-  GNUNET_CONTAINER_DLL_insert_tail (rel->head_recv, rel->tail_recv, copy);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add_buffered_data END\n");
+  if (NULL != ch->owner)
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
+  }
+  if (NULL != ch->dest)
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  if (NULL != ch->last_control_qe)
+  {
+    GCT_send_cancel (ch->last_control_qe);
+    ch->last_control_qe = NULL;
+  }
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if (NULL != ch->retry_control_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  if (GNUNET_NO == ch->is_loopback)
+  {
+    GCT_remove_channel (ch->t,
+                        ch,
+                        ch->ctn);
+    ch->t = NULL;
+  }
+  GNUNET_free (ch);
 }
 
 
 /**
- * Add a destination client to a channel, initializing all data structures
- * in the channel and the client.
+ * Send a channel create message.
  *
- * @param ch Channel to which add the destination.
- * @param c Client which to add to the channel.
+ * @param cls Channel for which to send.
  */
 static void
-add_destination (struct CadetChannel *ch, struct CadetClient *c)
-{
-  if (NULL != ch->dest)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Assign local id as destination */
-  ch->lid_dest = GML_get_next_ccn (c);
-
-  /* Store in client's hashmap */
-  GML_channel_add (c, ch->lid_dest, ch);
-
-  GNUNET_break (NULL == ch->dest_rel);
-  ch->dest_rel = GNUNET_new (struct CadetChannelReliability);
-  ch->dest_rel->ch = ch;
-  ch->dest_rel->expected_delay.rel_value_us = 0;
-  ch->dest_rel->retry_timer = CADET_RETRANSMIT_TIME;
-
-  ch->dest = c;
-}
+send_channel_open (void *cls);
 
 
 /**
- * Set options in a channel, extracted from a bit flag field.
+ * Function called once the tunnel confirms that we sent the
+ * create message.  Delays for a bit until we retry.
  *
- * @param ch Channel to set options to.
- * @param options Bit array in host byte order.
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-channel_set_options (struct CadetChannel *ch, uint32_t options)
+channel_open_sent_cb (void *cls,
+                      const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  ch->nobuffer = (options & GNUNET_CADET_OPTION_NOBUFFER) != 0 ?
-  GNUNET_YES : GNUNET_NO;
-  ch->reliable = (options & GNUNET_CADET_OPTION_RELIABLE) != 0 ?
-  GNUNET_YES : GNUNET_NO;
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
+  ch->retry_time = GNUNET_TIME_STD_BACKOFF (ch->retry_time);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sent CADET_CHANNEL_OPEN on %s, retrying in %s\n",
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (ch->retry_time,
+                                               GNUNET_YES));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_delayed (ch->retry_time,
+                                    &send_channel_open,
+                                    ch);
 }
 
 
 /**
- * Get a bit flag field with the options of a channel.
- *
- * @param ch Channel to get options from.
+ * Send a channel open message.
  *
- * @return Bit array in host byte order.
+ * @param cls Channel for which to send.
  */
-static uint32_t
-channel_get_options (struct CadetChannel *ch)
+static void
+send_channel_open (void *cls)
 {
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelOpenMessage msgcc;
   uint32_t options;
 
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN message for %s\n",
+       GCCH_2s (ch));
   options = 0;
   if (ch->nobuffer)
     options |= GNUNET_CADET_OPTION_NOBUFFER;
   if (ch->reliable)
     options |= GNUNET_CADET_OPTION_RELIABLE;
-
-  return options;
+  if (ch->out_of_order)
+    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
+  msgcc.header.size = htons (sizeof (msgcc));
+  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
+  msgcc.opt = htonl (options);
+  msgcc.h_port = ch->h_port;
+  msgcc.ctn = ch->ctn;
+  ch->state = CADET_CHANNEL_OPEN_SENT;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msgcc.header,
+                                  &channel_open_sent_cb,
+                                  ch);
+  GNUNET_assert (NULL == ch->retry_control_task);
 }
 
 
 /**
- * Notify a client that the channel is no longer valid.
- *
- * @param ch Channel that is destroyed.
- * @param local_only Should we avoid sending it to other peers?
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
  */
-static void
-send_destroy (struct CadetChannel *ch, int local_only)
+void
+GCCH_tunnel_up (struct CadetChannel *ch)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->gid;
-
-  /* If root is not NULL, notify.
-   * If it's NULL, check lid_root. When a local destroy comes in, root
-   * is set to NULL but lid_root is left untouched. In this case, do nothing,
-   * the client is the one who requested the channel to be destroyed.
-   */
-  if (NULL != ch->root)
-    GML_send_channel_destroy (ch->root, ch->lid_root);
-  else if (0 == ch->lid_root.channel_of_client && GNUNET_NO == local_only)
-    GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_NO, NULL);
-
-  if (NULL != ch->dest)
-    GML_send_channel_destroy (ch->dest, ch->lid_dest);
-  else if (0 == ch->lid_dest.channel_of_client && GNUNET_NO == local_only)
-    GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_YES, NULL);
+  GNUNET_assert (NULL == ch->retry_control_task);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Tunnel up, sending CHANNEL_OPEN on %s now\n",
+       GCCH_2s (ch));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_now (&send_channel_open,
+                                ch);
 }
 
 
 /**
- * Notify the destination client that a new incoming channel was created.
+ * Create a new channel.
  *
- * @param ch Channel that was created.
+ * @param owner local client owning the channel
+ * @param ccn local number of this channel at the @a owner
+ * @param destination peer to which we should build the channel
+ * @param port desired port at @a destination
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-static void
-send_client_create (struct CadetChannel *ch)
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber ccn,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options)
 {
-  uint32_t opt;
-
-  if (NULL == ch->dest)
-    return;
-
-  opt = 0;
-  opt |= GNUNET_YES == ch->reliable ? GNUNET_CADET_OPTION_RELIABLE : 0;
-  opt |= GNUNET_YES == ch->nobuffer ? GNUNET_CADET_OPTION_NOBUFFER : 0;
-  GML_send_channel_create (ch->dest,
-                           ch->lid_dest,
-                           &ch->port,
-                           opt,
-                           GCT_get_destination (ch->t));
-
-}
+  struct CadetChannel *ch;
+  struct CadetChannelClient *ccco;
 
+  ccco = GNUNET_new (struct CadetChannelClient);
+  ccco->c = owner;
+  ccco->ccn = ccn;
+  ccco->client_ready = GNUNET_YES;
 
-/**
- * Send data to a client.
- *
- * If the client is ready, send directly, otherwise buffer while listening
- * for a local ACK.
- *
- * @param ch Channel
- * @param msg Message.
- * @param fwd Is this a fwd (root->dest) message?
- */
-static void
-send_client_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd)
-{
-  if (fwd)
-  {
-    if (ch->dest_rel->client_ready)
+  ch = GNUNET_new (struct CadetChannel);
+  ch->mid_recv.mid = htonl (1); /* The OPEN_ACK counts as message 0! */
+  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
+  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
+  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
+  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  ch->owner = ccco;
+  ch->port = *port;
+  GCCH_hash_port (&ch->h_port,
+                  port,
+                  GCP_get_id (destination));
+  if (0 == memcmp (&my_full_id,
+                   GCP_get_id (destination),
+                   sizeof (struct GNUNET_PeerIdentity)))
+  {
+    struct OpenPort *op;
+
+    ch->is_loopback = GNUNET_YES;
+    op = GNUNET_CONTAINER_multihashmap_get (open_ports,
+                                            &ch->h_port);
+    if (NULL == op)
     {
-      GML_send_data (ch->dest, msg, ch->lid_dest);
-      ch->dest_rel->client_ready = GNUNET_NO;
-      ch->dest_rel->mid_recv++;
+      /* port closed, wait for it to possibly open */
+      ch->state = CADET_CHANNEL_LOOSE;
+      (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
+                                                &ch->h_port,
+                                                ch,
+                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Created loose incoming loopback channel to port %s\n",
+           GNUNET_h2s (&ch->port));
     }
     else
-      add_buffered_data (msg, ch->dest_rel);
+    {
+      GCCH_bind (ch,
+                 op->c,
+                 &op->port);
+    }
   }
   else
   {
-    if (ch->root_rel->client_ready)
-    {
-      GML_send_data (ch->root, msg, ch->lid_root);
-      ch->root_rel->client_ready = GNUNET_NO;
-      ch->root_rel->mid_recv++;
-    }
-    else
-      add_buffered_data (msg, ch->root_rel);
+    ch->t = GCP_get_tunnel (destination,
+                            GNUNET_YES);
+    ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+    ch->ctn = GCT_add_channel (ch->t,
+                               ch);
   }
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Created channel to port %s at peer %s for %s using %s\n",
+       GNUNET_h2s (port),
+       GCP_2s (destination),
+       GSC_2s (owner),
+       (GNUNET_YES == ch->is_loopback) ? "loopback" : GCT_2s (ch->t));
+  return ch;
 }
 
 
 /**
- * Send a buffered message to the client, for in order delivery or
- * as result of client ACK.
+ * We had an incoming channel to a port that is closed.
+ * It has not been opened for a while, drop it.
  *
- * @param ch Channel on which to empty the message buffer.
- * @param c Client to send to.
- * @param fwd Is this to send FWD data?.
+ * @param cls the channel to drop
  */
 static void
-send_client_buffered_data (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int fwd)
+timeout_closed_cb (void *cls)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_buffered_data\n");
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  if (GNUNET_NO == rel->client_ready)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "client not ready\n");
-    return;
-  }
+  struct CadetChannel *ch = cls;
 
-  copy = rel->head_recv;
-  /* We never buffer channel management messages */
-  if (NULL != copy)
-  {
-    if (copy->mid == rel->mid_recv || GNUNET_NO == ch->reliable)
-    {
-      struct GNUNET_CADET_ChannelAppDataMessage *msg = (struct GNUNET_CADET_ChannelAppDataMessage *) &copy[1];
-
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " have %u! now expecting %u\n",
-           copy->mid, rel->mid_recv + 1);
-      send_client_data (ch, msg, fwd);
-      rel->n_recv--;
-      GNUNET_CONTAINER_DLL_remove (rel->head_recv, rel->tail_recv, copy);
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " free copy recv MID %u (%p), %u left\n",
-           copy->mid, copy, rel->n_recv);
-      GNUNET_free (copy);
-      GCCH_send_data_ack (ch, fwd);
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " reliable && don't have %u, next is %u\n",
-           rel->mid_recv, copy->mid);
-      if (GNUNET_YES == ch->destroy)
-      {
-        /* We don't have the next data piece and the remote peer has closed the
-         * channel. We won't receive it anymore, so just destroy the channel.
-         * FIXME: wait some time to allow other connections to
-         *        deliver missing messages
-         */
-        send_destroy (ch, GNUNET_YES);
-        GCCH_destroy (ch);
-      }
-    }
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_buffered_data END\n");
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing incoming channel to port %s from peer %s due to timeout\n",
+       GNUNET_h2s (&ch->port),
+       GCP_2s (GCT_get_destination (ch->t)));
+  channel_destroy (ch);
 }
 
 
 /**
- * Allow a client to send more data.
- *
- * In case the client was already allowed to send data, do nothing.
+ * Create a new channel based on a request coming in over the network.
  *
- * @param ch Channel.
- * @param fwd Is this a FWD ACK? (FWD ACKs are sent to root)
+ * @param t tunnel to the remote peer
+ * @param ctn identifier of this channel in the tunnel
+ * @param h_port desired hash of local port
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-static void
-send_client_ack (struct CadetChannel *ch, int fwd)
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber ctn,
+                           const struct GNUNET_HashCode *h_port,
+                           uint32_t options)
 {
-  struct CadetChannelReliability *rel = fwd ? ch->root_rel : ch->dest_rel;
-  struct CadetClient *c = fwd ? ch->root : ch->dest;
-
-  if (NULL == c)
-  {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending %s ack to client on channel %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
+  struct CadetChannel *ch;
+  struct OpenPort *op;
 
-  if (NULL == rel)
-  {
-    GNUNET_break (0);
-    return;
+  ch = GNUNET_new (struct CadetChannel);
+  ch->h_port = *h_port;
+  ch->t = t;
+  ch->ctn = ctn;
+  ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
+  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
+  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
+  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
+
+  op = GNUNET_CONTAINER_multihashmap_get (open_ports,
+                                          h_port);
+  if (NULL == op)
+  {
+    /* port closed, wait for it to possibly open */
+    ch->state = CADET_CHANNEL_LOOSE;
+    (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
+                                              &ch->h_port,
+                                              ch,
+                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+    GNUNET_assert (NULL == ch->retry_control_task);
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_delayed (TIMEOUT_CLOSED_PORT,
+                                      &timeout_closed_cb,
+                                      ch);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Created loose incoming channel to port %s from peer %s\n",
+         GNUNET_h2s (&ch->port),
+         GCP_2s (GCT_get_destination (ch->t)));
   }
-
-  if (GNUNET_YES == rel->client_allowed)
+  else
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  already allowed\n");
-    return;
+    GCCH_bind (ch,
+               op->c,
+               &op->port);
   }
-  rel->client_allowed = GNUNET_YES;
-
-  GML_send_ack (c, fwd ? ch->lid_root : ch->lid_dest);
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
+  return ch;
 }
 
 
 /**
- * Notify the root that the destination rejected the channel.
+ * Function called once the tunnel confirms that we sent the
+ * ACK message.  Just remembers it was sent, we do not expect
+ * ACKs for ACKs ;-).
  *
- * @param ch Rejected channel.
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-send_client_nack (struct CadetChannel *ch)
+send_ack_cb (void *cls,
+             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  if (NULL == ch->root)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  GML_send_channel_nack (ch->root, ch->lid_root);
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
 }
 
 
 /**
- * We haven't received an ACK after a certain time: restransmit the message.
+ * Compute and send the current #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK to the other peer.
  *
- * @param cls Closure (CadetChannelReliability with the message to restransmit)
+ * @param ch channel to send the #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK for
  */
 static void
-channel_retransmit_message (void *cls)
+send_channel_data_ack (struct CadetChannel *ch)
 {
-  struct CadetChannelReliability *rel = cls;
-  struct CadetReliableMessage *copy;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ChannelAppDataMessage *payload;
-  int fwd;
-
-  rel->retry_task = NULL;
-  ch = rel->ch;
-  copy = rel->head_sent;
-  if (NULL == copy)
-  {
-    GNUNET_break (0); // FIXME tripped in rps testcase
-    return;
-  }
-
-  payload = (struct GNUNET_CADET_ChannelAppDataMessage *) &copy[1];
-  fwd = (rel == ch->root_rel);
-
-  /* Message not found in the queue that we are going to use. */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "RETRANSMIT MID %u\n", copy->mid);
+  struct GNUNET_CADET_ChannelDataAckMessage msg;
 
-  GCCH_send_prebuilt_message (&payload->header, ch, fwd, copy);
-  GNUNET_STATISTICS_update (stats, "# data retransmitted", 1, GNUNET_NO);
+  if (GNUNET_NO == ch->reliable)
+    return; /* no ACKs */
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.ctn = ch->ctn;
+  msg.mid.mid = htonl (ntohl (ch->mid_recv.mid));
+  msg.futures = GNUNET_htonll (ch->mid_futures);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending DATA_ACK %u:%llX via %s\n",
+       (unsigned int) ntohl (msg.mid.mid),
+       (unsigned long long) ch->mid_futures,
+       GCCH_2s (ch));
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msg.header,
+                                  &send_ack_cb,
+                                  ch);
 }
 
 
 /**
- * We haven't received an Channel ACK after a certain time: resend the CREATE.
+ * Send our initial #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK to the client confirming that the
+ * connection is up.
  *
- * @param cls Closure (CadetChannelReliability of the channel to recreate)
+ * @param cls the `struct CadetChannel`
  */
 static void
-channel_recreate (void *cls)
+send_open_ack (void *cls)
 {
-  struct CadetChannelReliability *rel = cls;
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelOpenAckMessage msg;
 
-  rel->retry_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "RE-CREATE\n");
-  GNUNET_STATISTICS_update (stats,
-                            "# data retransmitted", 1, GNUNET_NO);
-
-  if (rel == rel->ch->root_rel)
-  {
-    send_create (rel->ch);
-  }
-  else if (rel == rel->ch->dest_rel)
-  {
-    send_ack (rel->ch, GNUNET_YES);
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.reserved = htonl (0);
+  msg.ctn = ch->ctn;
+  msg.port = ch->port;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msg.header,
+                                  &send_ack_cb,
+                                  ch);
 }
 
 
 /**
- * Message has been sent: start retransmission timer.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
  *
- * @param cls Closure (queue structure).
- * @param t Tunnel.
- * @param q Queue handler (no longer valid).
- * @param type Type of message.
- * @param size Size of the message.
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
  */
-static void
-ch_message_sent (void *cls,
-                 struct CadetTunnel *t,
-                 struct CadetTunnelQueue *q,
-                 uint16_t type, size_t size)
+void
+GCCH_handle_duplicate_open (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
 {
-  struct CadetChannelQueue *chq = cls;
-  struct CadetReliableMessage *copy = chq->copy;
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "channel_message_sent callback %s\n",
-       GC_m2s (chq->type));
-
-  switch (chq->type)
+  if (NULL == ch->dest)
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "data MID %u sent\n", copy->mid);
-      GNUNET_assert (chq == copy->chq);
-      copy->timestamp = GNUNET_TIME_absolute_get ();
-      rel = copy->rel;
-      if (NULL == rel->retry_task)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  scheduling retry in %d * %s\n",
-             CADET_RETRANSMIT_MARGIN,
-             GNUNET_STRINGS_relative_time_to_string (rel->expected_delay,
-                                                     GNUNET_YES));
-        if (0 != rel->expected_delay.rel_value_us)
-        {
-          rel->retry_timer =
-              GNUNET_TIME_relative_saturating_multiply (rel->expected_delay,
-                                                        CADET_RETRANSMIT_MARGIN);
-        }
-        else
-        {
-          rel->retry_timer = CADET_RETRANSMIT_TIME;
-        }
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  using delay %s\n",
-             GNUNET_STRINGS_relative_time_to_string (rel->retry_timer,
-                                                     GNUNET_NO));
-        rel->retry_task =
-            GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
-                                          &channel_retransmit_message, rel);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "retry running %p\n", rel->retry_task);
-      }
-      copy->chq = NULL;
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "sent %s\n", GC_m2s (chq->type));
-      rel = chq->rel;
-      GNUNET_assert (rel->uniq == chq);
-      rel->uniq = NULL;
-
-      if (CADET_CHANNEL_READY != rel->ch->state
-          && GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK != type
-          && GNUNET_NO == rel->ch->destroy)
-      {
-        GNUNET_assert (NULL == rel->retry_task);
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "STD BACKOFF %s\n",
-             GNUNET_STRINGS_relative_time_to_string (rel->retry_timer,
-                                                     GNUNET_NO));
-        rel->retry_timer = GNUNET_TIME_STD_BACKOFF (rel->retry_timer);
-        rel->retry_task = GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
-                                                        &channel_recreate, rel);
-      }
-      break;
-
-    default:
-      GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: port is closed\n",
+         GCCH_2s (ch));
+    return;
   }
-
-  GNUNET_free (chq);
+  if (NULL != ch->retry_control_task)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: control message is pending\n",
+         GCCH_2s (ch));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retransmitting CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_now (&send_open_ack,
+                                ch);
 }
 
 
 /**
- * send a channel create message.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK to the client to solicit more messages.
  *
- * @param ch Channel for which to send.
+ * @param ch channel the ack is for
+ * @param to_owner #GNUNET_YES to send to owner,
+ *                 #GNUNET_NO to send to dest
  */
 static void
-send_create (struct CadetChannel *ch)
+send_ack_to_client (struct CadetChannel *ch,
+                    int to_owner)
 {
-  struct GNUNET_CADET_ChannelOpenMessage msgcc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalAck *ack;
+  struct CadetChannelClient *ccc;
 
-  msgcc.header.size = htons (sizeof (msgcc));
-  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
-  msgcc.ctn = ch->gid;
-  msgcc.port = ch->port;
-  msgcc.opt = htonl (channel_get_options (ch));
-
-  GCCH_send_prebuilt_message (&msgcc.header, ch, GNUNET_YES, NULL);
+  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
+  if (NULL == ccc)
+  {
+    /* This can happen if we are just getting ACKs after
+       our local client already disconnected. */
+    GNUNET_assert (GNUNET_YES == ch->destroy);
+    return;
+  }
+  env = GNUNET_MQ_msg (ack,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
+  ack->ccn = ccc->ccn;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X (%u/%u pending)\n",
+       GSC_2s (ccc->c),
+       (GNUNET_YES == to_owner) ? "owner" : "dest",
+       ntohl (ack->ccn.channel_of_client),
+       ch->pending_messages,
+       ch->max_pending_messages);
+  GSC_send_to_client (ccc->c,
+                      env);
 }
 
 
 /**
- * Confirm we got a channel create or FWD ack.
+ * A client is bound to the port that we have a channel
+ * open to.  Send the acknowledgement for the connection
+ * request and establish the link with the client.
  *
- * @param ch The channel to confirm.
- * @param fwd Should we send a FWD ACK? (going dest->root)
+ * @param ch open incoming channel
+ * @param c client listening on the respective @a port
+ * @param port the port @a is listening on
  */
-static void
-send_ack (struct CadetChannel *ch, int fwd)
+void
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c,
+           const struct GNUNET_HashCode *port)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
+  uint32_t options;
+  struct CadetChannelClient *cccd;
 
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending channel %s ack for channel %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
-
-  msg.ctn =ch->gid;
-  GCCH_send_prebuilt_message (&msg.header, ch, !fwd, NULL);
+       "Binding %s from %s to port %s of %s\n",
+       GCCH_2s (ch),
+       GCT_2s (ch->t),
+       GNUNET_h2s (&ch->port),
+       GSC_2s (c));
+  if (NULL != ch->retry_control_task)
+  {
+    /* there might be a timeout task here */
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  options = 0;
+  if (ch->nobuffer)
+    options |= GNUNET_CADET_OPTION_NOBUFFER;
+  if (ch->reliable)
+    options |= GNUNET_CADET_OPTION_RELIABLE;
+  if (ch->out_of_order)
+    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
+  cccd = GNUNET_new (struct CadetChannelClient);
+  GNUNET_assert (NULL == ch->dest);
+  ch->dest = cccd;
+  ch->port = *port;
+  cccd->c = c;
+  cccd->client_ready = GNUNET_YES;
+  cccd->ccn = GSC_bind (c,
+                        ch,
+                        (GNUNET_YES == ch->is_loopback)
+                        ? GCP_get (&my_full_id,
+                                   GNUNET_YES)
+                        : GCT_get_destination (ch->t),
+                        port,
+                        options);
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  ch->mid_recv.mid = htonl (1); /* The OPEN counts as message 0! */
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    ch->state = CADET_CHANNEL_OPEN_SENT;
+    GCCH_handle_channel_open_ack (ch,
+                                  NULL,
+                                  port);
+  }
+  else
+  {
+    /* notify other peer that we accepted the connection */
+    ch->state = CADET_CHANNEL_READY;
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_now (&send_open_ack,
+                                  ch);
+  }
+  /* give client it's initial supply of ACKs */
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  for (unsigned int i=0;i<ch->max_pending_messages;i++)
+    send_ack_to_client (ch,
+                        GNUNET_NO);
 }
 
 
 /**
- * Send a message and don't keep any info about it: we won't need to cancel it
- * or resend it.
+ * One of our clients has disconnected, tell the other one that we
+ * are finished. Done asynchronously to avoid concurrent modification
+ * issues if this is the same client.
  *
- * @param msg Header of the message to fire away.
- * @param ch Channel on which the message should go.
- * @param force Is this a forced (undroppable) message?
+ * @param cls the `struct CadetChannel` where one of the ends is now dead
  */
 static void
-fire_and_forget (const struct GNUNET_MessageHeader *msg,
-                 struct CadetChannel *ch,
-                 int force)
+signal_remote_destroy_cb (void *cls)
 {
-  GNUNET_break (NULL ==
-                GCT_send_prebuilt_message (msg, ch->t, NULL,
-                                           force, NULL, NULL));
+  struct CadetChannel *ch = cls;
+  struct CadetChannelClient *ccc;
+
+  /* Find which end is left... */
+  ch->retry_control_task = NULL;
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  GSC_handle_remote_channel_destroy (ccc->c,
+                                     ccc->ccn,
+                                     ch);
+  channel_destroy (ch);
 }
 
 
 /**
- * Notify that a channel create didn't succeed.
+ * Destroy locally created channel.  Called by the local client, so no
+ * need to tell the client.
  *
- * @param ch The channel to reject.
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
  */
-static void
-send_nack (struct CadetChannel *ch)
+void
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending channel NACK for channel %s\n",
+       "%s asks for destruction of %s\n",
+       GSC_2s (c),
        GCCH_2s (ch));
+  GNUNET_assert (NULL != c);
+  if ( (NULL != ch->owner) &&
+       (c == ch->owner->c) &&
+       (ccn.channel_of_client == ch->owner->ccn.channel_of_client) )
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
+  }
+  else if ( (NULL != ch->dest) &&
+            (c == ch->dest->c) &&
+            (ccn.channel_of_client == ch->dest->ccn.channel_of_client) )
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
 
-  msg.ctn = ch->gid;
-  GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_NO, NULL);
-}
-
-
-/**
- * Destroy all reliable messages queued for a channel,
- * during a channel destruction.
- * Frees the reliability structure itself.
- *
- * @param rel Reliability data for a channel.
- */
-static void
-channel_rel_free_all (struct CadetChannelReliability *rel)
-{
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-
-  if (NULL == rel)
-    return;
-
-  for (copy = rel->head_recv; NULL != copy; copy = next)
+  if (GNUNET_YES == ch->destroy)
   {
-    next = copy->next;
-    GNUNET_CONTAINER_DLL_remove (rel->head_recv, rel->tail_recv, copy);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " COPYFREE ALL RECV %p\n", copy);
-    GNUNET_break (NULL == copy->chq);
-    GNUNET_free (copy);
+    /* other end already destroyed, with the local client gone, no need
+       to finish transmissions, just destroy immediately. */
+    channel_destroy (ch);
+    return;
   }
-  for (copy = rel->head_sent; NULL != copy; copy = next)
+  if ( (NULL != ch->head_sent) &&
+       ( (NULL != ch->owner) ||
+         (NULL != ch->dest) ) )
   {
-    next = copy->next;
-    GNUNET_CONTAINER_DLL_remove (rel->head_sent, rel->tail_sent, copy);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " COPYFREE ALL SEND %p\n", copy);
-    if (NULL != copy->chq)
-    {
-      if (NULL != copy->chq->tq)
-      {
-        GCT_cancel (copy->chq->tq);
-        /* ch_message_sent will free copy->q */
-      }
-      else
-      {
-        GNUNET_free (copy->chq);
-        GNUNET_break (0);
-      }
-    }
-    GNUNET_free (copy);
+    /* Wait for other end to destroy us as well,
+       and otherwise allow send queue to be transmitted first */
+    ch->destroy = GNUNET_YES;
+    return;
   }
-  if (NULL != rel->uniq && NULL != rel->uniq->tq)
+  if ( (GNUNET_YES == ch->is_loopback) &&
+       ( (NULL != ch->owner) ||
+         (NULL != ch->dest) ) )
   {
-    GCT_cancel (rel->uniq->tq);
-    /* ch_message_sent is called freeing uniq */
+    if (NULL != ch->retry_control_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_now (&signal_remote_destroy_cb,
+                                  ch);
+    return;
   }
-  if (NULL != rel->retry_task)
+  if (GNUNET_NO == ch->is_loopback)
   {
-    GNUNET_SCHEDULER_cancel (rel->retry_task);
-    rel->retry_task = NULL;
+    /* If the we ever sent the CHANNEL_CREATE, we need to send a destroy message. */
+    switch (ch->state)
+    {
+    case CADET_CHANNEL_NEW:
+      /* We gave up on a channel that we created as a client to a remote
+         target, but that never went anywhere. Nothing to do here. */
+      break;
+    case CADET_CHANNEL_LOOSE:
+      GSC_drop_loose_channel (&ch->h_port,
+                              ch);
+      break;
+    default:
+      GCT_send_channel_destroy (ch->t,
+                                ch->ctn);
+    }
   }
-  GNUNET_free (rel);
+  /* Nothing left to do, just finish destruction */
+  channel_destroy (ch);
 }
 
 
 /**
- * Mark future messages as ACK'd.
+ * We got an acknowledgement for the creation of the channel
+ * (the port is open on the other side).  Verify that the
+ * other end really has the right port, and begin transmissions.
  *
- * @param rel Reliability data.
- * @param msg DataACK message with a bitfield of future ACK'd messages.
- *
- * @return How many messages have been freed.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message
+ * @param port port number (needed to verify receiver knows the port)
  */
-static unsigned int
-channel_rel_free_sent (struct CadetChannelReliability *rel,
-                       const struct GNUNET_CADET_ChannelDataAckMessage *msg)
+void
+GCCH_handle_channel_open_ack (struct CadetChannel *ch,
+                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                              const struct GNUNET_HashCode *port)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-  uint64_t bitfield;
-  uint64_t mask;
-  uint32_t mid;
-  uint32_t target;
-  unsigned int i;
-  unsigned int r;
-
-  bitfield = msg->futures;
-  mid = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "free_sent_reliable %u %lX\n", mid, bitfield);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " rel %p, head %p\n", rel, rel->head_sent);
-  for (i = 0, r = 0, copy = rel->head_sent;
-       i < 64 && NULL != copy && 0 != bitfield;
-       i++)
+  switch (ch->state)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " trying bit %u (mid %u)\n", i, mid + i + 1);
-    mask = 0x1LL << i;
-    if (0 == (bitfield & mask))
-     continue;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " set!\n");
-    /* Bit was set, clear the bit from the bitfield */
-    bitfield &= ~mask;
-
-    /* The i-th bit was set. Do we have that copy? */
-    /* Skip copies with mid < target */
-    target = mid + i + 1;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " target %u\n", target);
-    while (NULL != copy && GC_is_pid_bigger (target, copy->mid))
-      copy = copy->next;
-
-    /* Did we run out of copies? (previously freed, it's ok) */
-    if (NULL == copy)
+  case CADET_CHANNEL_NEW:
+    /* this should be impossible */
+    GNUNET_break (0);
+    break;
+  case CADET_CHANNEL_LOOSE:
+    /* This makes no sense. */
+    GNUNET_break_op (0);
+    break;
+  case CADET_CHANNEL_OPEN_SENT:
+    if (NULL == ch->owner)
+    {
+      /* We're not the owner, wrong direction! */
+      GNUNET_break_op (0);
+      return;
+    }
+    if (0 != memcmp (&ch->port,
+                     port,
+                     sizeof (struct GNUNET_HashCode)))
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "run out of copies...\n");
-      return r;
+      /* Other peer failed to provide the right port, 
+         refuse connection. */
+      GNUNET_break_op (0);
+      return;
     }
-
-    /* Did we overshoot the target? (previously freed, it's ok) */
-    if (GC_is_pid_bigger (copy->mid, target))
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CHANNEL_OPEN_ACK for waiting %s, entering READY state\n",
+         GCCH_2s (ch));
+    if (NULL != ch->retry_control_task) /* can be NULL if ch->is_loopback */
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " next copy %u\n", copy->mid);
-      i += copy->mid - target - 1;   /* MID: 90, t = 85, i += 4 (i++ later) */
-      mask = (0x1LL << (i + 1)) - 1; /* Mask = i-th bit and all before */
-      bitfield &= ~mask;             /* Clear all bits up to MID - 1 */
-      continue;
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+      ch->retry_control_task = NULL;
     }
-
-    /* Now copy->mid == target, free it */
-    next = copy->next;
-    GNUNET_break (GNUNET_YES != rel_message_free (copy, GNUNET_YES));
-    r++;
-    copy = next;
+    ch->state = CADET_CHANNEL_READY;
+    /* On first connect, send client as many ACKs as we allow messages
+       to be buffered! */
+    for (unsigned int i=0;i<ch->max_pending_messages;i++)
+      send_ack_to_client (ch,
+                          GNUNET_YES);
+    break;
+  case CADET_CHANNEL_READY:
+    /* duplicate ACK, maybe we retried the CREATE. Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received duplicate channel OPEN_ACK for %s\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate CREATE_ACKs",
+                              1,
+                              GNUNET_NO);
+    break;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "free_sent_reliable END\n");
-  return r;
 }
 
 
 /**
- * Destroy a reliable message after it has been acknowledged, either by
- * direct mid ACK or bitfield. Updates the appropriate data structures and
- * timers and frees all memory.
- *
- * @param copy Message that is no longer needed: remote peer got it.
- * @param update_time Is the timing information relevant?
- *                    If this message is ACK in a batch the timing information
- *                    is skewed by the retransmission, count only for the
- *                    retransmitted message.
+ * Test if element @a e1 comes before element @a e2.
  *
- * @return #GNUNET_YES if channel was destroyed as a result of the call,
- *         #GNUNET_NO otherwise.
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param m1 a message of to sort
+ * @param m2 another message to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
  */
 static int
-rel_message_free (struct CadetReliableMessage *copy, int update_time)
+is_before (void *cls,
+           struct CadetOutOfOrderMessage *m1,
+           struct CadetOutOfOrderMessage *m2)
 {
-  struct CadetChannelReliability *rel;
-  struct GNUNET_TIME_Relative time;
+  int *duplicate = cls;
+  uint32_t v1 = ntohl (m1->mid.mid);
+  uint32_t v2 = ntohl (m2->mid.mid);
+  uint32_t delta;
 
-  rel = copy->rel;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Freeing %u\n", copy->mid);
-  if (GNUNET_YES == update_time)
+  delta = v2 - v1;
+  if (0 == delta)
+    *duplicate = GNUNET_YES;
+  if (delta > (uint32_t) INT_MAX)
   {
-    time = GNUNET_TIME_absolute_get_duration (copy->timestamp);
-    if (0 == rel->expected_delay.rel_value_us)
-      rel->expected_delay = time;
-    else
-    {
-      rel->expected_delay.rel_value_us *= 7;
-      rel->expected_delay.rel_value_us += time.rel_value_us;
-      rel->expected_delay.rel_value_us /= 8;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  message time %12s\n",
-         GNUNET_STRINGS_relative_time_to_string (time, GNUNET_NO));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  new delay    %12s\n",
-         GNUNET_STRINGS_relative_time_to_string (rel->expected_delay,
-                                                 GNUNET_NO));
-    rel->retry_timer = rel->expected_delay;
+    /* in overflow range, we can safely assume we wrapped around */
+    return GNUNET_NO;
   }
   else
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "batch free, ignoring timing\n");
-  }
-  rel->ch->pending_messages--;
-  if (NULL != copy->chq)
-  {
-    GCT_cancel (copy->chq->tq);
-    /* copy->q is set to NULL by ch_message_sent */
-  }
-  GNUNET_CONTAINER_DLL_remove (rel->head_sent, rel->tail_sent, copy);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " free send copy MID %u at %p\n",
-       copy->mid, copy);
-  GNUNET_free (copy);
-
-  if (GNUNET_NO != rel->ch->destroy && 0 == rel->ch->pending_messages)
-  {
-    GCCH_destroy (rel->ch);
+    /* result is small, thus v2 > v1, thus m1 < m2 */
     return GNUNET_YES;
   }
-  return GNUNET_NO;
 }
 
 
 /**
- * Channel was ACK'd by remote peer, mark as ready and cancel retransmission.
+ * We got payload data for a channel.  Pass it on to the client
+ * and send an ACK to the other end (once flow control allows it!)
  *
- * @param ch Channel to mark as ready.
- * @param fwd Was the ACK message a FWD ACK? (dest->root, SYNACK)
+ * @param ch channel that got data
+ * @param cti identifier of the connection that delivered the message
+ * @param msg message that was received
  */
-static void
-channel_confirm (struct CadetChannel *ch, int fwd)
+void
+GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
+                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg)
 {
-  struct CadetChannelReliability *rel;
-  enum CadetChannelState oldstate;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-  if (NULL == rel)
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalData *ld;
+  struct CadetChannelClient *ccc;
+  size_t payload_size;
+  struct CadetOutOfOrderMessage *com;
+  int duplicate;
+  uint32_t mid_min;
+  uint32_t mid_max;
+  uint32_t mid_msg;
+  uint32_t delta;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  if ( (GNUNET_YES == ch->destroy) &&
+       (NULL == ch->owner) &&
+       (NULL == ch->dest) )
+  {
+    /* This client is gone, but we still have messages to send to
+       the other end (which is why @a ch is not yet dead).  However,
+       we cannot pass messages to our client anymore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping incoming payload on %s as this end is already closed\n",
+         GCCH_2s (ch));
+    /* send back DESTROY notification to stop further retransmissions! */
+    GCT_send_channel_destroy (ch->t,
+                              ch->ctn);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  env = GNUNET_MQ_msg_extra (ld,
+                             payload_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+  ld->ccn = (NULL == ch->dest) ? ch->owner->ccn : ch->dest->ccn;
+  GNUNET_memcpy (&ld[1],
+                 &msg[1],
+                 payload_size);
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if ( (GNUNET_YES == ccc->client_ready) &&
+       ( (GNUNET_YES == ch->out_of_order) ||
+         (msg->mid.mid == ch->mid_recv.mid) ) )
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Giving %u bytes of payload with MID %u from %s to client %s\n",
+         (unsigned int) payload_size,
+         ntohl (msg->mid.mid),
+         GCCH_2s (ch),
+         GSC_2s (ccc->c));
+    ccc->client_ready = GNUNET_NO;
+    GSC_send_to_client (ccc->c,
+                        env);
+    ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
+    ch->mid_futures >>= 1;
+    send_channel_data_ack (ch);
     return;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  channel confirm %s %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
-  oldstate = ch->state;
-  ch->state = CADET_CHANNEL_READY;
 
-  if (CADET_CHANNEL_READY != oldstate || GNUNET_YES == is_loopback (ch))
+  if (GNUNET_YES == ch->reliable)
   {
-    rel->client_ready = GNUNET_YES;
-    rel->expected_delay = rel->retry_timer;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  confirm retry timer %s\n",
-         GNUNET_STRINGS_relative_time_to_string (rel->retry_timer, GNUNET_NO));
-    if (GCT_get_connections_buffer (ch->t) > 0 || GCT_is_loopback (ch->t))
-      send_client_ack (ch, fwd);
-
-    if (NULL != rel->retry_task)
+    /* check if message ought to be dropped because it is ancient/too distant/duplicate */
+    mid_min = ntohl (ch->mid_recv.mid);
+    mid_max = mid_min + ch->max_pending_messages;
+    mid_msg = ntohl (msg->mid.mid);
+    if ( ( (uint32_t) (mid_msg - mid_min) > ch->max_pending_messages) ||
+         ( (uint32_t) (mid_max - mid_msg) > ch->max_pending_messages) )
     {
-      GNUNET_SCHEDULER_cancel (rel->retry_task);
-      rel->retry_task = NULL;
-    }
-    else if (NULL != rel->uniq)
-    {
-      GCT_cancel (rel->uniq->tq);
-      /* ch_message_sent will free and NULL uniq */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "%s at %u drops ancient or far-future message %u\n",
+           GCCH_2s (ch),
+           (unsigned int) mid_min,
+           ntohl (msg->mid.mid));
+
+      GNUNET_STATISTICS_update (stats,
+                                "# duplicate DATA (ancient or future)",
+                                1,
+                                GNUNET_NO);
+      GNUNET_MQ_discard (env);
+      send_channel_data_ack (ch);
+      return;
     }
-    else if (GNUNET_NO == is_loopback (ch))
+    /* mark bit for future ACKs */
+    delta = mid_msg - mid_min - 1; /* overflow/underflow are OK here */
+    if (delta < 64)
     {
-      /* We SHOULD have been trying to retransmit this! */
-      GNUNET_break (0);
+      if (0 != (ch->mid_futures & (1LLU << delta)))
+      {
+        /* Duplicate within the queue, drop also */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+             (unsigned int) payload_size,
+             GCCH_2s (ch),
+             ntohl (msg->mid.mid));
+        GNUNET_STATISTICS_update (stats,
+                                  "# duplicate DATA",
+                                  1,
+                                  GNUNET_NO);
+        GNUNET_MQ_discard (env);
+        send_channel_data_ack (ch);
+        return;
+      }
+      ch->mid_futures |= (1LLU << delta);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Marked bit %llX for mid %u (base: %u); now: %llX\n",
+           (1LLU << delta),
+           mid_msg,
+           mid_min,
+           ch->mid_futures);
     }
   }
-
-  /* In case of a FWD ACK (SYNACK) send a BCK ACK (ACK). */
-  if (GNUNET_YES == fwd)
-    send_ack (ch, GNUNET_NO);
-}
-
-
-/**
- * Save a copy to retransmit in case it gets lost.
- *
- * Initializes all needed callbacks and timers.
- *
- * @param ch Channel this message goes on.
- * @param msg Message to copy.
- * @param fwd Is this fwd traffic?
- */
-static struct CadetReliableMessage *
-channel_save_copy (struct CadetChannel *ch,
-                   const struct GNUNET_MessageHeader *msg,
-                   int fwd)
-{
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  uint32_t mid;
-  uint16_t type;
-  uint16_t size;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-  mid = rel->mid_send - 1;
-  type = ntohs (msg->type);
-  size = ntohs (msg->size);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "save MID %u %s\n", mid, GC_m2s (type));
-  copy = GNUNET_malloc (sizeof (struct CadetReliableMessage) + size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  at %p\n", copy);
-  copy->mid = mid;
-  copy->rel = rel;
-  copy->type = type;
-  GNUNET_memcpy (&copy[1], msg, size);
-  GNUNET_CONTAINER_DLL_insert_tail (rel->head_sent, rel->tail_sent, copy);
-  ch->pending_messages++;
-
-  return copy;
-}
-
-
-/**
- * Create a new channel.
- *
- * @param t Tunnel this channel is in.
- * @param owner Client that owns the channel, NULL for foreign channels.
- * @param lid_root Local ID for root client.
- *
- * @return A new initialized channel. NULL on error.
- */
-static struct CadetChannel *
-channel_new (struct CadetTunnel *t,
-             struct CadetClient *owner,
-             struct GNUNET_CADET_ClientChannelNumber lid_root)
-{
-  struct CadetChannel *ch;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->root = owner;
-  ch->lid_root = lid_root;
-  ch->t = t;
-
-  GNUNET_STATISTICS_update (stats, "# channels", 1, GNUNET_NO);
-
-  if (NULL != owner)
-  {
-    ch->gid = GCT_get_next_ctn (t);
-    GML_channel_add (owner, lid_root, ch);
-  }
-  GCT_add_channel (t, ch);
-
-  return ch;
-}
-
-
-/**
- * Handle a loopback message: call the appropriate handler for the message type.
- *
- * @param ch Channel this message is on.
- * @param msgh Message header.
- * @param fwd Is this FWD traffic?
- */
-void
-handle_loopback (struct CadetChannel *ch,
-                 const struct GNUNET_MessageHeader *msgh,
-                 int fwd)
-{
-  uint16_t type;
-
-  type = ntohs (msgh->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Loopback %s %s message!\n",
-       GC_f2s (fwd), GC_m2s (type));
-
-  switch (type)
+  else /* ! ch->reliable */
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      /* Don't send hop ACK, wait for client to ACK */
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "SEND loopback %u (%u)\n",
-           ntohl (((struct GNUNET_CADET_ChannelAppDataMessage *) msgh)->mid), ntohs (msgh->size));
-      GCCH_handle_data (ch, (struct GNUNET_CADET_ChannelAppDataMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      GCCH_handle_data_ack (ch,
-                            (const struct GNUNET_CADET_ChannelDataAckMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      GCCH_handle_create (ch->t,
-                          (const struct GNUNET_CADET_ChannelOpenMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      GCCH_handle_ack (ch,
-                       (const struct GNUNET_CADET_ChannelManageMessage *) msgh,
-                       fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      GCCH_handle_nack (ch);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      GCCH_handle_destroy (ch,
-                           (const struct GNUNET_CADET_ChannelManageMessage *) msgh,
-                           fwd);
-      break;
+    /* Channel is unreliable, so we do not ACK. But we also cannot
+       allow buffering everything, so check if we have space... */
+    if (ccc->num_recv >= ch->max_pending_messages)
+    {
+      struct CadetOutOfOrderMessage *drop;
 
-    default:
-      GNUNET_break_op (0);
+      /* Yep, need to drop. Drop the oldest message in
+         the buffer. */
       LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "end-to-end message not known (%u)\n",
-           ntohs (msgh->type));
+           "Queue full due slow client on %s, dropping oldest message\n",
+           GCCH_2s (ch));
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to slow client",
+                                1,
+                                GNUNET_NO);
+      drop = ccc->head_recv;
+      GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                   ccc->tail_recv,
+                                   drop);
+      ccc->num_recv--;
+      GNUNET_MQ_discard (drop->env);
+      GNUNET_free (drop);
+    }
   }
-}
-
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Destroy a channel and free all resources.
- *
- * @param ch Channel to destroy.
- */
-void
-GCCH_destroy (struct CadetChannel *ch)
-{
-  struct CadetClient *c;
-  struct CadetTunnel *t;
 
-  if (NULL == ch)
+  /* Insert message into sorted out-of-order queue */
+  com = GNUNET_new (struct CadetOutOfOrderMessage);
+  com->mid = msg->mid;
+  com->env = env;
+  duplicate = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
+                                      is_before,
+                                      &duplicate,
+                                      ccc->head_recv,
+                                      ccc->tail_recv,
+                                      com);
+  ccc->num_recv++;
+  if (GNUNET_YES == duplicate)
+  {
+    /* Duplicate within the queue, drop also (this is not covered by
+       the case above if "delta" >= 64, which could be the case if
+       max_pending_messages is also >= 64 or if our client is unready
+       and we are seeing retransmissions of the message our client is
+       blocked on. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+         (unsigned int) payload_size,
+         GCCH_2s (ch),
+         ntohl (msg->mid.mid));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA",
+                              1,
+                              GNUNET_NO);
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+    send_channel_data_ack (ch);
     return;
-  if (2 == ch->destroy)
-    return; /* recursive call */
-  ch->destroy = 2;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "destroying channel %s:%u\n",
-              GCT_2s (ch->t), ch->gid);
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-
-  c = ch->root;
-  if (NULL != c)
-  {
-    GML_channel_remove (c, ch->lid_root, ch);
   }
-
-  c = ch->dest;
-  if (NULL != c)
-  {
-    GML_channel_remove (c, ch->lid_dest, ch);
-  }
-
-  channel_rel_free_all (ch->root_rel);
-  channel_rel_free_all (ch->dest_rel);
-
-  t = ch->t;
-  GCT_remove_channel (t, ch);
-  GNUNET_STATISTICS_update (stats, "# channels", -1, GNUNET_NO);
-
-  GNUNET_free (ch);
-  GCT_destroy_if_empty (t);
-}
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch)
-{
-  return ch->gid;
-}
-
-
-/**
- * Get the channel tunnel.
- *
- * @param ch Channel to get the tunnel from.
- *
- * @return tunnel of the channel.
- */
-struct CadetTunnel *
-GCCH_get_tunnel (const struct CadetChannel *ch)
-{
-  return ch->t;
-}
-
-
-/**
- * Get free buffer space towards the client on a specific channel.
- *
- * @param ch Channel.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - 64]
- */
-unsigned int
-GCCH_get_buffer (struct CadetChannel *ch, int fwd)
-{
-  struct CadetChannelReliability *rel;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   get buffer, channel %s\n", GCCH_2s (ch));
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-  /* If rel is NULL it means that the end is not yet created,
-   * most probably is a loopback channel at the point of sending
-   * the ChannelCreate to itself.
-   */
-  if (NULL == rel)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  rel is NULL: max\n");
-    return 64;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   n_recv %d\n", rel->n_recv);
-  return (64 - rel->n_recv);
-}
-
-
-/**
- * Get flow control status of end point: is client allow to send?
- *
- * @param ch Channel.
- * @param fwd Is query about FWD traffic? (Request root status).
- *
- * @return #GNUNET_YES if client is allowed to send us data.
- */
-int
-GCCH_get_allowed (struct CadetChannel *ch, int fwd)
-{
-  struct CadetChannelReliability *rel;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-
-  if (NULL == rel)
-  {
-    /* Probably shutting down: root/dest NULL'ed to mark disconnection */
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return 0;
-  }
-
-  return rel->client_allowed;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queued %s payload of %u bytes on %s-%X(%p) (mid %u, need %u first)\n",
+       (GNUNET_YES == ccc->client_ready)
+       ? "out-of-order"
+       : "client-not-ready",
+       (unsigned int) payload_size,
+       GCCH_2s (ch),
+       ntohl (ccc->ccn.channel_of_client),
+       ccc,
+       ntohl (msg->mid.mid),
+       ntohl (ch->mid_recv.mid));
+  /* NOTE: this ACK we _could_ skip, as the packet is out-of-order and
+     the sender may already be transmitting the previous one.  Needs
+     experimental evaluation to see if/when this ACK helps or
+     hurts. (We might even want another option.) */
+  send_channel_data_ack (ch);
 }
 
 
 /**
- * Is the root client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
- *
- * @return #GNUNET_YES in case it is.
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
-int
-GCCH_is_origin (struct CadetChannel *ch, int fwd)
-{
-  struct CadetClient *c;
-
-  c = fwd ? ch->root : ch->dest;
-  return NULL != c;
-}
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
 
 /**
- * Is the destination client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
+ * We need to retry a transmission, the last one took too long to
+ * be acknowledged.
  *
- * @return #GNUNET_YES in case it is.
+ * @param cls the `struct CadetChannel` where we need to retransmit
  */
-int
-GCCH_is_terminal (struct CadetChannel *ch, int fwd)
+static void
+retry_transmission (void *cls)
 {
-  struct CadetClient *c;
+  struct CadetChannel *ch = cls;
+  struct CadetReliableMessage *crm = ch->head_sent;
 
-  c = fwd ? ch->dest : ch->root;
-  return NULL != c;
+  ch->retry_data_task = NULL;
+  GNUNET_assert (NULL == crm->qe);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retrying transmission on %s of message %u\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid));
+  crm->qe = GCT_send (ch->t,
+                      &crm->data_message->header,
+                      &data_sent_cb,
+                      crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
 }
 
 
 /**
- * Send an end-to-end ACK message for the most recent in-sequence payload.
- *
- * If channel is not reliable, do nothing.
+ * We got an PLAINTEXT_DATA_ACK for a message in our queue, remove it from
+ * the queue and tell our client that it can send more.
  *
- * @param ch Channel this is about.
- * @param fwd Is for FWD traffic? (ACK dest->owner)
+ * @param ch the channel that got the PLAINTEXT_DATA_ACK
+ * @param cti identifier of the connection that delivered the message
+ * @param crm the message that got acknowledged
  */
-void
-GCCH_send_data_ack (struct CadetChannel *ch, int fwd)
+static void
+handle_matching_ack (struct CadetChannel *ch,
+                     const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                     struct CadetReliableMessage *crm)
 {
-  struct GNUNET_CADET_ChannelDataAckMessage msg;
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  unsigned int delta;
-  uint64_t mask;
-  uint32_t ack;
-
-  if (GNUNET_NO == ch->reliable)
-    return;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  ack = rel->mid_recv - 1;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->gid;
-  msg.mid = htonl (ack);
-
-  msg.futures = 0LL;
-  for (copy = rel->head_recv; NULL != copy; copy = copy->next)
-  {
-    if (copy->type != GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA)
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                               ch->tail_sent,
+                               crm);
+  ch->pending_messages--;
+  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       ch->pending_messages);
+  if (NULL != crm->qe)
+  {
+    GCT_send_cancel (crm->qe);
+    crm->qe = NULL;
+  }
+  if ( (1 == crm->num_transmissions) &&
+       (NULL != cti) )
+  {
+    GCC_ack_observed (cti);
+    if (0 == memcmp (cti,
+                     &crm->connection_taken,
+                     sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier)))
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " Type %s, expected DATA\n",
-           GC_m2s (copy->type));
-      continue;
+      GCC_latency_observed (cti,
+                            GNUNET_TIME_absolute_get_duration (crm->first_transmission_time));
     }
-    GNUNET_assert (GC_is_pid_bigger(copy->mid, ack));
-    delta = copy->mid - (ack + 1);
-    if (63 < delta)
-      break;
-    mask = 0x1LL << delta;
-    msg.futures |= mask;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         " setting bit for %u (delta %u) (%lX) -> %lX\n",
-         copy->mid, delta, mask, msg.futures);
   }
-
-  GCCH_send_prebuilt_message (&msg.header, ch, !fwd, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_data_ack END\n");
+  GNUNET_free (crm->data_message);
+  GNUNET_free (crm);
+  send_ack_to_client (ch,
+                      (NULL == ch->owner)
+                      ? GNUNET_NO
+                      : GNUNET_YES);
 }
 
 
 /**
- * Allow a client to send us more data, in case it was choked.
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
  *
- * @param ch Channel.
- * @param fwd Is this about FWD traffic? (Root client).
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
  */
 void
-GCCH_allow_client (struct CadetChannel *ch, int fwd)
+GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
+                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack)
 {
-  struct CadetChannelReliability *rel;
-  unsigned int buffer;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMCH allow\n");
+  struct CadetReliableMessage *crm;
+  struct CadetReliableMessage *crmn;
+  int found;
+  uint32_t mid_base;
+  uint64_t mid_mask;
+  unsigned int delta;
 
-  if (CADET_CHANNEL_READY != ch->state)
+  GNUNET_break (GNUNET_NO == ch->is_loopback);
+  if (GNUNET_NO == ch->reliable)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " channel not ready yet!\n");
+    /* not expecting ACKs on unreliable channel, odd */
+    GNUNET_break_op (0);
     return;
   }
-
-  if (GNUNET_YES == ch->reliable)
-  {
-    rel = fwd ? ch->root_rel : ch->dest_rel;
-    if (NULL == rel)
+  /* mid_base is the MID of the next message that the
+     other peer expects (i.e. that is missing!), everything
+     LOWER (but excluding mid_base itself) was received. */
+  mid_base = ntohl (ack->mid.mid);
+  mid_mask = GNUNET_htonll (ack->futures);
+  found = GNUNET_NO;
+  for (crm = ch->head_sent;
+        NULL != crm;
+       crm = crmn)
+  {
+    crmn = crm->next;
+    delta = (unsigned int) (ntohl (crm->data_message->mid.mid) - mid_base);
+    if (delta >= UINT_MAX - ch->max_pending_messages)
     {
-      GNUNET_break (GNUNET_NO != ch->destroy);
-      return;
-    }
-    if (NULL != rel->head_sent)
-    {
-      if (64 <= rel->mid_send - rel->head_sent->mid)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " too big MID gap! Wait for ACK.\n");
-        return;
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " gap ok: %u - %u\n",
-             rel->head_sent->mid, rel->mid_send);
-        struct CadetReliableMessage *aux;
-        for (aux = rel->head_sent; NULL != aux; aux = aux->next)
-        {
-          LOG (GNUNET_ERROR_TYPE_DEBUG, "   - sent mid %u\n", aux->mid);
-        }
-      }
+      /* overflow, means crm was a bit in the past, so this ACK counts for it. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with base %u satisfying past message %u on %s\n",
+           (unsigned int) mid_base,
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch,
+                           cti,
+                           crm);
+      found = GNUNET_YES;
+      continue;
     }
-    else
+    delta--;
+    if (delta >= 64)
+      continue;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Testing bit %llX for mid %u (base: %u)\n",
+         (1LLU << delta),
+         ntohl (crm->data_message->mid.mid),
+         mid_base);
+    if (0 != (mid_mask & (1LLU << delta)))
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " head sent is NULL\n");
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with mask for %u on %s\n",
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch,
+                           cti,
+                           crm);
+      found = GNUNET_YES;
     }
   }
-
-  if (is_loopback (ch))
-    buffer = GCCH_get_buffer (ch, fwd);
-  else
-    buffer = GCT_get_connections_buffer (ch->t);
-
-  if (0 == buffer)
+  if (GNUNET_NO == found)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " no buffer space.\n");
+    /* ACK for message we already dropped, might have been a
+       duplicate ACK? Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate DATA_ACK on %s, ignoring\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA_ACKs",
+                              1,
+                              GNUNET_NO);
     return;
   }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " buffer space %u, allowing\n", buffer);
-  send_client_ack (ch, fwd);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if ( (NULL != ch->head_sent) &&
+       (NULL == ch->head_sent->qe) )
+    ch->retry_data_task
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
 }
 
 
 /**
- * Log channel info.
+ * Destroy channel, based on the other peer closing the
+ * connection.  Also needs to remove this channel from
+ * the tunnel.
  *
- * @param ch Channel.
- * @param level Debug level to use.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *            NULL if we are simulating receiving a destroy due to shutdown
  */
 void
-GCCH_debug (struct CadetChannel *ch, enum GNUNET_ErrorType level)
+GCCH_handle_remote_destroy (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
 {
-  int do_log;
+  struct CadetChannelClient *ccc;
 
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-chn",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  if (NULL == ch)
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received remote channel DESTROY for %s\n",
+       GCCH_2s (ch));
+  if (GNUNET_YES == ch->destroy)
   {
-    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
+    /* Local client already gone, this is instant-death. */
+    channel_destroy (ch);
     return;
   }
-  LOG2 (level, "CHN Channel %s:%X (%p)\n", GCT_2s (ch->t), ch->gid, ch);
-  LOG2 (level, "CHN   root %p/%p\n", ch->root, ch->root_rel);
-  if (NULL != ch->root)
-  {
-    LOG2 (level, "CHN   cli %s\n", GML_2s (ch->root));
-    LOG2 (level, "CHN   ready %s\n", ch->root_rel->client_ready ? "YES" : "NO");
-    LOG2 (level, "CHN   id %X\n", ch->lid_root.channel_of_client);
-    LOG2 (level, "CHN   recv %d\n", ch->root_rel->n_recv);
-    LOG2 (level, "CHN   MID r: %d, s: %d\n",
-          ch->root_rel->mid_recv, ch->root_rel->mid_send);
-  }
-  LOG2 (level, "CHN   dest %p/%p\n",
-              ch->dest, ch->dest_rel);
-  if (NULL != ch->dest)
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if ( (NULL != ccc) &&
+       (NULL != ccc->head_recv) )
   {
-    LOG2 (level, "CHN   cli %s\n", GML_2s (ch->dest));
-    LOG2 (level, "CHN   ready %s\n", ch->dest_rel->client_ready ? "YES" : "NO");
-    LOG2 (level, "CHN   id %X\n", ch->lid_dest);
-    LOG2 (level, "CHN   recv %d\n", ch->dest_rel->n_recv);
-    LOG2 (level, "CHN   MID r: %d, s: %d\n",
-          ch->dest_rel->mid_recv, ch->dest_rel->mid_send);
-
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Lost end of transmission due to remote shutdown on %s\n",
+         GCCH_2s (ch));
+    /* FIXME: change API to notify client about truncated transmission! */
   }
+  ch->destroy = GNUNET_YES;
+  if (NULL != ccc)
+    GSC_handle_remote_channel_destroy (ccc->c,
+                                       ccc->ccn,
+                                       ch);
+  channel_destroy (ch);
 }
 
 
 /**
- * Handle an ACK given by a client.
+ * Test if element @a e1 comes before element @a e2.
  *
- * Mark client as ready and send him any buffered data we could have for him.
- *
- * @param ch Channel.
- * @param fwd Is this a "FWD ACK"? (FWD ACKs are sent by dest and go BCK)
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param crm1 an element of to sort
+ * @param crm2 another element to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
  */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch, int fwd)
+static int
+cmp_crm_by_next_retry (void *cls,
+                       struct CadetReliableMessage *crm1,
+                       struct CadetReliableMessage *crm2)
 {
-  struct CadetChannelReliability *rel;
-  struct CadetClient *c;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  c   = fwd ? ch->dest     : ch->root;
-
-  rel->client_ready = GNUNET_YES;
-  send_client_buffered_data (ch, c, fwd);
-
-  if (GNUNET_YES == ch->destroy && 0 == rel->n_recv)
-  {
-    send_destroy (ch, GNUNET_YES);
-    GCCH_destroy (ch);
-    return;
-  }
-  /* if loopback is marked for destruction, no need to ACK to the other peer,
-   * it requested the destruction and is already gone, therefore, else if.
-   */
-  else if (is_loopback (ch))
-  {
-    unsigned int buffer;
-
-    buffer = GCCH_get_buffer (ch, fwd);
-    if (0 < buffer)
-      GCCH_allow_client (ch, fwd);
-
-    return;
-  }
-  GCT_send_connection_acks (ch->t);
+  if (crm1->next_retry.abs_value_us <
+      crm2->next_retry.abs_value_us)
+    return GNUNET_YES;
+  return GNUNET_NO;
 }
 
 
 /**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if channel
- * is reliable and send an ACK to the client if there is still buffer space
- * in the tunnel.
- *
- * @param ch Channel.
- * @param c Client which sent the data.
- * @param fwd Is this a FWD data?
- * @param message Data message.
- * @param size Size of data.
- *
- * @return #GNUNET_OK if everything goes well, #GNUNET_SYSERR in case of en error.
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct CadetClient *c,
-                        int fwd,
-                        const struct GNUNET_MessageHeader *message,
-                        size_t size)
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct CadetChannelReliability *rel;
-  struct GNUNET_CADET_ChannelAppDataMessage *payload;
-  uint16_t p2p_size = sizeof(struct GNUNET_CADET_ChannelAppDataMessage) + size;
-  unsigned char cbuf[p2p_size];
-  unsigned char buffer;
-
-  /* Is the client in the channel? */
-  if ( !( (fwd &&
-           ch->root == c)
-         ||
-          (!fwd &&
-           ch->dest == c) ) )
+  struct CadetReliableMessage *crm = cls;
+  struct CadetChannel *ch = crm->ch;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  GNUNET_assert (NULL != crm->qe);
+  crm->qe = NULL;
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                               ch->tail_sent,
+                               crm);
+  if (GNUNET_NO == ch->reliable)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
+    ch->pending_messages--;
+    send_ack_to_client (ch,
+                        (NULL == ch->owner)
+                        ? GNUNET_NO
+                        : GNUNET_YES);
+    return;
   }
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-
-  if (GNUNET_NO == rel->client_allowed)
+  if (NULL == cid)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    /* There was an error sending. */
+    crm->num_transmissions = GNUNET_SYSERR;
   }
-
-  rel->client_allowed = GNUNET_NO;
-
-  /* Ok, everything is correct, send the message. */
-  payload = (struct GNUNET_CADET_ChannelAppDataMessage *) cbuf;
-  payload->mid = htonl (rel->mid_send);
-  rel->mid_send++;
-  GNUNET_memcpy (&payload[1], message, size);
-  payload->header.size = htons (p2p_size);
-  payload->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
-  payload->ctn = ch->gid;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on channel...\n");
-  GCCH_send_prebuilt_message (&payload->header, ch, fwd, NULL);
-
-  if (is_loopback (ch))
-    buffer = GCCH_get_buffer (ch, fwd);
-  else
-    buffer = GCT_get_connections_buffer (ch->t);
-
-  if (0 < buffer)
-    GCCH_allow_client (ch, fwd);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a channel destroy requested by a client.
- *
- * TODO: add "reason" field
- *
- * Destroy the channel and the tunnel in case this was the last channel.
- *
- * @param ch Channel.
- * @param c Client that requested the destruction (to avoid notifying him).
- * @param is_root Is the request coming from root?
- */
-void
-GCCH_handle_local_destroy (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int is_root)
-{
-  ch->destroy = GNUNET_YES;
-  /* Cleanup after the tunnel */
-  if (GNUNET_NO == is_root && c == ch->dest)
+  else if (GNUNET_SYSERR != crm->num_transmissions)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " Client %s is destination.\n", GML_2s (c));
-    GML_client_delete_channel (c, ch, ch->lid_dest);
-    ch->dest = NULL;
+    /* Increment transmission counter, and possibly store @a cid
+       if this was the first transmission. */
+    crm->num_transmissions++;
+    if (1 == crm->num_transmissions)
+    {
+      crm->first_transmission_time = GNUNET_TIME_absolute_get ();
+      crm->connection_taken = *cid;
+      GCC_ack_expected (cid);
+    }
   }
-  if (GNUNET_YES == is_root && c == ch->root)
+  if ( (0 == crm->retry_delay.rel_value_us) &&
+       (NULL != cid) )
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " Client %s is owner.\n", GML_2s (c));
-    GML_client_delete_channel (c, ch, ch->lid_root);
-    ch->root = NULL;
-  }
+    struct CadetConnection *cc = GCC_lookup (cid);
 
-  send_destroy (ch, GNUNET_NO);
-  if (0 == ch->pending_messages)
-    GCCH_destroy (ch);
+    if (NULL != cc)
+      crm->retry_delay = GCC_get_metrics (cc)->aged_latency;
+    else
+      crm->retry_delay = ch->retry_time;
+  }
+  crm->retry_delay = GNUNET_TIME_STD_BACKOFF (crm->retry_delay);
+  crm->retry_delay = GNUNET_TIME_relative_max (crm->retry_delay,
+                                               MIN_RTT_DELAY);
+  crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
+
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetReliableMessage,
+                                      cmp_crm_by_next_retry,
+                                      NULL,
+                                      ch->head_sent,
+                                      ch->tail_sent,
+                                      crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message %u sent, next transmission on %s in %s\n",
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (ch->head_sent->next_retry),
+                                               GNUNET_YES));
+  if (NULL == ch->head_sent->qe)
+  {
+    if (NULL != ch->retry_data_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
+  }
 }
 
 
 /**
- * Handle a channel create requested by a client.
- *
- * Create the channel and the tunnel in case this was the first0 channel.
+ * Handle data given by a client.
  *
- * @param c Client that requested the creation (will be the root).
- * @param msg Create Channel message.
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
  *
- * @return #GNUNET_OK if everything went fine, #GNUNET_SYSERR otherwise.
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
  */
 int
-GCCH_handle_local_create (struct CadetClient *c,
-                          struct GNUNET_CADET_LocalChannelCreateMessage *msg)
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len)
 {
-  struct CadetChannel *ch;
-  struct CadetTunnel *t;
-  struct CadetPeer *peer;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  towards %s:%u\n",
-       GNUNET_i2s (&msg->peer), GNUNET_h2s (&msg->port));
-  ccn = msg->ccn;
+  struct CadetReliableMessage *crm;
 
-  /* Sanity check for duplicate channel IDs */
-  if (NULL != GML_channel_get (c, ccn))
+  if (ch->pending_messages > ch->max_pending_messages)
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
-  peer = GCP_get (&msg->peer, GNUNET_YES);
-  GCP_add_tunnel (peer);
-  t = GCP_get_tunnel (peer);
-
-  if (GCP_get_short_id (peer) == myid)
-  {
-    GCT_change_cstate (t, CADET_TUNNEL_READY);
-  }
-  else
+  if (GNUNET_YES == ch->destroy)
   {
-    /* FIXME change to a tunnel API, eliminate ch <-> peer connection */
-    GCP_connect (peer);
+    /* we are going down, drop messages */
+    return GNUNET_OK;
   }
+  ch->pending_messages++;
 
-  /* Create channel */
-  ch = channel_new (t, c, ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  ch->port = msg->port;
-  channel_set_options (ch, ntohl (msg->opt));
-
-  /* In unreliable channels, we'll use the DLL to buffer BCK data */
-  ch->root_rel = GNUNET_new (struct CadetChannelReliability);
-  ch->root_rel->ch = ch;
-  ch->root_rel->retry_timer = CADET_RETRANSMIT_TIME;
-  ch->root_rel->expected_delay.rel_value_us = 0;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "CREATED CHANNEL %s\n", GCCH_2s (ch));
-
-  send_create (ch);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for cadet network payload traffic.
- *
- * @param ch Channel for the message.
- * @param msg Unencryted data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-void
-GCCH_handle_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd)
-{
-  struct CadetChannelReliability *rel;
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader *payload_msg;
-  uint32_t mid;
-  uint16_t payload_type;
-  uint16_t payload_size;
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
-  }
-
-  /*  Initialize FWD/BCK data */
-  c   = fwd ? ch->dest     : ch->root;
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-
-  if (NULL == c)
+  if (GNUNET_YES == ch->is_loopback)
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
-  }
+    struct CadetChannelClient *receiver;
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_LocalData *ld;
+    int ack_to_owner;
 
-  if (CADET_CHANNEL_READY != ch->state)
-  {
-    if (GNUNET_NO == fwd)
+    env = GNUNET_MQ_msg_extra (ld,
+                               buf_len,
+                               GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+    if ( (NULL != ch->owner) &&
+         (sender_ccn.channel_of_client ==
+          ch->owner->ccn.channel_of_client) )
     {
-      /* If we are the root, this means the other peer has sent traffic before
-       * receiving our ACK. Even if the SYNACK goes missing, no traffic should
-       * be sent before the ACK.
-       */
-      GNUNET_break_op (0);
-      return;
+      receiver = ch->dest;
+      ack_to_owner = GNUNET_YES;
     }
-    /* If we are the dest, this means that the SYNACK got to the root but
-     * the ACK went missing. Treat this as an ACK.
-     */
-    channel_confirm (ch, GNUNET_NO);
-  }
-
-  payload_msg = (struct GNUNET_MessageHeader *) &msg[1];
-  payload_type = ntohs (payload_msg->type);
-  payload_size = ntohs (payload_msg->size);
-
-  GNUNET_STATISTICS_update (stats, "# messages received", 1, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats, "# bytes received", payload_size, GNUNET_NO);
-
-  mid = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_INFO, "<== %s (%s %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA), GC_m2s (payload_type), mid,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  if ( (GNUNET_NO == ch->reliable) ||
-       ( (! GC_is_pid_bigger (rel->mid_recv, mid)) &&
-         GC_is_pid_bigger (rel->mid_recv + 64, mid) ) )
-  {
-    if (GNUNET_YES == ch->reliable)
+    else if ( (NULL != ch->dest) &&
+              (sender_ccn.channel_of_client ==
+               ch->dest->ccn.channel_of_client) )
     {
-      /* Is this the exact next expected messasge? */
-      if (mid == rel->mid_recv)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "as expected, sending to client\n");
-        send_client_data (ch, msg, fwd);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "save for later\n");
-        add_buffered_data (msg, rel);
-      }
+      receiver = ch->owner;
+      ack_to_owner = GNUNET_NO;
     }
     else
     {
-      /* Tunnel is unreliable: send to clients directly */
-      /* FIXME: accept Out Of Order traffic */
-      rel->mid_recv = mid + 1;
-      send_client_data (ch, msg, fwd);
+      GNUNET_break (0);
+      return GNUNET_SYSERR;
     }
-  }
-  else
-  {
-    GNUNET_STATISTICS_update (stats, "# duplicate MID", 1, GNUNET_NO);
-    if (GC_is_pid_bigger (rel->mid_recv, mid))
+    GNUNET_assert (NULL != receiver);
+    ld->ccn = receiver->ccn;
+    GNUNET_memcpy (&ld[1],
+                   buf,
+                   buf_len);
+    if (GNUNET_YES == receiver->client_ready)
     {
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "MID %u on channel %s not expected (window: %u - %u). Dropping!\n",
-           mid, GCCH_2s (ch), rel->mid_recv, rel->mid_recv + 63);
+      ch->pending_messages--;
+      GSC_send_to_client (receiver->c,
+                          env);
+      send_ack_to_client (ch,
+                          ack_to_owner);
     }
     else
     {
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           "Duplicate MID %u, channel %s (expecting MID %u). Re-sending ACK!\n",
-           mid, GCCH_2s (ch), rel->mid_recv);
-      if (NULL != rel->uniq)
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "We are trying to send an ACK, but don't seem have the "
-             "bandwidth. Have you set enough [ats] QUOTA in your config?\n");
-      }
-
+      struct CadetOutOfOrderMessage *oom;
+
+      oom = GNUNET_new (struct CadetOutOfOrderMessage);
+      oom->env = env;
+      GNUNET_CONTAINER_DLL_insert_tail (receiver->head_recv,
+                                        receiver->tail_recv,
+                                        oom);
+      receiver->num_recv++;
     }
-  }
-
-  GCCH_send_data_ack (ch, fwd);
+    return GNUNET_OK;
+  }
+
+  /* Everything is correct, send the message. */
+  crm = GNUNET_malloc (sizeof (*crm));
+  crm->ch = ch;
+  crm->data_message = GNUNET_malloc (sizeof (struct GNUNET_CADET_ChannelAppDataMessage)
+                                     + buf_len);
+  crm->data_message->header.size = htons (sizeof (struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
+  crm->data_message->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
+  ch->mid_send.mid = htonl (ntohl (ch->mid_send.mid) + 1);
+  crm->data_message->mid = ch->mid_send;
+  crm->data_message->ctn = ch->ctn;
+  GNUNET_memcpy (&crm->data_message[1],
+                 buf,
+                 buf_len);
+  GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent,
+                                    ch->tail_sent,
+                                    crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message %u from local client to %s with %u bytes\n",
+       ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       buf_len);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  crm->qe = GCT_send (ch->t,
+                      &crm->data_message->header,
+                      &data_sent_cb,
+                      crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
+  return GNUNET_OK;
 }
 
 
 /**
- * Handler for cadet network traffic end-to-end ACKs.
+ * Handle ACK from client on local channel.  Means the client is ready
+ * for more data, see if we have any for it.
  *
- * @param ch Channel on which we got this message.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel to destroy
+ * @param client_ccn ccn of the client sending the ack
  */
 void
-GCCH_handle_data_ack (struct CadetChannel *ch,
-                      const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                      int fwd)
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn)
 {
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-  uint32_t ack;
-  int work;
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    /* Inverted: if message came 'FWD' is a 'BCK ACK'. */
-    fwd = (NULL != ch->dest) ? GNUNET_NO : GNUNET_YES;
-  }
-
-  ack = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (0x%010lX %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK), msg->futures, ack,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  if (GNUNET_YES == fwd)
-    rel = ch->root_rel;
+  struct CadetChannelClient *ccc;
+  struct CadetOutOfOrderMessage *com;
+
+  if ( (NULL != ch->owner) &&
+       (ch->owner->ccn.channel_of_client == client_ccn.channel_of_client) )
+    ccc = ch->owner;
+  else if ( (NULL != ch->dest) &&
+            (ch->dest->ccn.channel_of_client == client_ccn.channel_of_client) )
+    ccc = ch->dest;
   else
-    rel = ch->dest_rel;
-
-  if (NULL == rel)
-  {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
-  }
-
-  /* Free ACK'd copies: no need to retransmit those anymore FIXME refactor */
-  for (work = GNUNET_NO, copy = rel->head_sent; copy != NULL; copy = next)
+    GNUNET_assert (0);
+  ccc->client_ready = GNUNET_YES;
+  com = ccc->head_recv;
+  if (NULL == com)
   {
-    if (GC_is_pid_bigger (copy->mid, ack))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  head %u, out!\n", copy->mid);
-      if (0 < channel_rel_free_sent (rel, msg))
-        work = GNUNET_YES;
-      break;
-    }
-    work = GNUNET_YES;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  id %u\n", copy->mid);
-    next = copy->next;
-    if (GNUNET_YES == rel_message_free (copy, GNUNET_YES))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " channel destoyed\n");
-      return;
-    }
-  }
-
-  /* ACK client if needed and possible */
-  GCCH_allow_client (ch, fwd);
-
-  /* If some message was free'd, update the retransmission delay */
-  if (GNUNET_YES == work)
-  {
-    if (NULL != rel->retry_task)
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data, but none pending on %s-%X(%p)!\n",
+         GSC_2s (ccc->c),
+         ntohl (client_ccn.channel_of_client),
+         GCCH_2s (ch),
+         ntohl (ccc->ccn.channel_of_client),
+         ccc);
+    return; /* none pending */
+  }
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    int to_owner;
+
+    /* Messages are always in-order, just send */
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GSC_send_to_client (ccc->c,
+                        com->env);
+    /* Notify sender that we can receive more */
+    if ( (NULL != ch->owner) &&
+         (ccc->ccn.channel_of_client ==
+          ch->owner->ccn.channel_of_client) )
     {
-      GNUNET_SCHEDULER_cancel (rel->retry_task);
-      rel->retry_task = NULL;
-      if (NULL != rel->head_sent && NULL == rel->head_sent->chq)
-      {
-        struct GNUNET_TIME_Absolute new_target;
-        struct GNUNET_TIME_Relative delay;
-
-        delay = GNUNET_TIME_relative_saturating_multiply (rel->retry_timer,
-                                                          CADET_RETRANSMIT_MARGIN);
-        new_target = GNUNET_TIME_absolute_add (rel->head_sent->timestamp,
-                                               delay);
-        delay = GNUNET_TIME_absolute_get_remaining (new_target);
-        rel->retry_task =
-            GNUNET_SCHEDULER_add_delayed (delay,
-                                          &channel_retransmit_message,
-                                          rel);
-      }
+      to_owner = GNUNET_NO;
     }
     else
     {
-      /* Work was done but no task was pending.
-       * Task was cancelled by a retransmission that is sitting in the queue.
-       */
-      // FIXME add test to make sure this is the case, probably add return
-      // value to GCCH_send_prebuilt_message
+      GNUNET_assert ( (NULL != ch->dest) &&
+                      (ccc->ccn.channel_of_client ==
+                       ch->dest->ccn.channel_of_client) );
+      to_owner = GNUNET_YES;
     }
+    send_ack_to_client (ch,
+                        to_owner);
+    GNUNET_free (com);
+    return;
   }
-}
-
 
-/**
- * Handler for channel create messages.
- *
- * Does not have fwd parameter because it's always 'FWD': channel is incoming.
- *
- * @param t Tunnel this channel will be in.
- * @param msg Channel crate message.
- */
-struct CadetChannel *
-GCCH_handle_create (struct CadetTunnel *t,
-                    const struct GNUNET_CADET_ChannelOpenMessage *msg)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct GNUNET_CADET_ChannelTunnelNumber gid;
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-  int new_channel;
-  const struct GNUNET_HashCode *port;
-
-  gid = msg->ctn;
-  ch = GCT_get_channel (t, gid);
-  if (NULL == ch)
+  if ( (com->mid.mid != ch->mid_recv.mid) &&
+       (GNUNET_NO == ch->out_of_order) &&
+       (GNUNET_YES == ch->reliable) )
   {
-    /* Create channel */
-    ccn.channel_of_client = htonl (0);
-    ch = channel_new (t, NULL, ccn);
-    ch->gid = gid;
-    channel_set_options (ch, ntohl (msg->opt));
-    new_channel = GNUNET_YES;
-  }
-  else
-  {
-    new_channel = GNUNET_NO;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data (but next one is out-of-order %u vs. %u)!\n",
+         GSC_2s (ccc->c),
+         ntohl (ccc->ccn.channel_of_client),
+         ntohl (com->mid.mid),
+         ntohl (ch->mid_recv.mid));
+    return; /* missing next one in-order */
   }
-  port = &msg->port;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN), ccn, port,
-       GCCH_2s (ch), ch, GC_f2s (GNUNET_YES), ntohs (msg->header.size));
-
-  if (GNUNET_YES == new_channel || GCT_is_loopback (t))
-  {
-    /* Find a destination client */
-    ch->port = *port;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   port %s\n", GNUNET_h2s (port));
-    c = GML_client_get_by_port (port);
-    if (NULL == c)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  no client has port registered\n");
-      if (is_loopback (ch))
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  loopback: destroy on handler\n");
-        send_nack (ch);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  not loopback: destroy now\n");
-        send_nack (ch);
-        GCCH_destroy (ch);
-      }
-      return NULL;
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  client %p has port registered\n", c);
-    }
-
-    add_destination (ch, c);
-    if (GNUNET_YES == ch->reliable)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Reliable\n");
-    else
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Not Reliable\n");
 
-    send_client_create (ch);
-    ch->state =  CADET_CHANNEL_SENT;
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  duplicate create channel\n");
-    if (NULL != ch->dest_rel->retry_task)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  clearing retry task\n");
-      /* we were waiting to re-send our 'SYNACK', wait no more! */
-      GNUNET_SCHEDULER_cancel (ch->dest_rel->retry_task);
-      ch->dest_rel->retry_task = NULL;
-    }
-    else if (NULL != ch->dest_rel->uniq)
-    {
-      /* we are waiting to for our 'SYNACK' to leave the queue, all done! */
-      return ch;
-    }
-  }
-  send_ack (ch, GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got LOCAL_ACK, giving payload message %u to %s-%X on %s\n",
+       ntohl (com->mid.mid),
+       GSC_2s (ccc->c),
+       ntohl (ccc->ccn.channel_of_client),
+       GCCH_2s (ch));
 
-  return ch;
+  /* all good, pass next message to client */
+  GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                               ccc->tail_recv,
+                               com);
+  ccc->num_recv--;
+  /* FIXME: if unreliable, this is not aggressive
+     enough, as it would be OK to have lost some! */
+
+  ch->mid_recv.mid = htonl (1 + ntohl (com->mid.mid));
+  ch->mid_futures >>= 1; /* equivalent to division by 2 */
+  ccc->client_ready = GNUNET_NO;
+  GSC_send_to_client (ccc->c,
+                      com->env);
+  GNUNET_free (com);
+  send_channel_data_ack (ch);
+  if (NULL != ccc->head_recv)
+    return;
+  if (GNUNET_NO == ch->destroy)
+    return;
+  GCT_send_channel_destroy (ch->t,
+                            ch->ctn);
+  channel_destroy (ch);
 }
 
 
-/**
- * Handler for channel NACK messages.
- *
- * NACK messages always go dest -> root, no need for 'fwd' or 'msg' parameter.
- *
- * @param ch Channel.
- */
-void
-GCCH_handle_nack (struct CadetChannel *ch)
-{
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED), ch->gid, 0,
-       GCCH_2s (ch), ch, "---", 0);
-
-  send_client_nack (ch);
-  GCCH_destroy (ch);
-}
+#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
 
 
 /**
- * Handler for channel ack messages.
+ * Log channel info.
  *
  * @param ch Channel.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-void
-GCCH_handle_ack (struct CadetChannel *ch,
-                 const struct GNUNET_CADET_ChannelManageMessage *msg,
-                 int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK), ch->gid, 0,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
-  }
-
-  channel_confirm (ch, !fwd);
-}
-
-
-/**
- * Handler for channel destroy messages.
- *
- * @param ch Channel to be destroyed of.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param level Debug level to use.
  */
 void
-GCCH_handle_destroy (struct CadetChannel *ch,
-                     const struct GNUNET_CADET_ChannelManageMessage *msg,
-                     int fwd)
+GCCH_debug (struct CadetChannel *ch,
+            enum GNUNET_ErrorType level)
 {
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY), ch->gid, 0,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
-  }
+  int do_log;
 
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-  if ( (fwd && NULL == ch->dest) || (!fwd && NULL == ch->root) )
-  {
-    /* Not for us (don't destroy twice a half-open loopback channel) */
+  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
+                                       "cadet-chn",
+                                       __FILE__, __FUNCTION__, __LINE__);
+  if (0 == do_log)
     return;
-  }
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  if (0 == rel->n_recv)
-  {
-    send_destroy (ch, GNUNET_YES);
-    GCCH_destroy (ch);
-  }
-  else
-  {
-    ch->destroy = GNUNET_YES;
-  }
-}
-
-
-/**
- * Sends an already built message on a channel.
- *
- * If the channel is on a loopback tunnel, notifies the appropriate destination
- * client locally.
- *
- * On a normal channel passes the message to the tunnel for encryption and
- * sending on a connection.
- *
- * This function DOES NOT save the message for retransmission.
- *
- * @param message Message to send. Function makes a copy of it.
- * @param ch Channel on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param existing_copy This is a retransmission, don't save a copy.
- */
-void
-GCCH_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                            struct CadetChannel *ch, int fwd,
-                            void *existing_copy)
-{
-  struct CadetChannelQueue *chq;
-  uint32_t data_id;
-  uint16_t type;
-  uint16_t size;
-  char info[32];
-
-  type = ntohs (message->type);
-  size = ntohs (message->size);
-
-  data_id = 0;
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-    {
-      struct GNUNET_CADET_ChannelAppDataMessage *data_msg;
-      struct GNUNET_MessageHeader *payload_msg;
-      uint16_t payload_type;
-
-      data_msg = (struct GNUNET_CADET_ChannelAppDataMessage *) message;
-      data_id = ntohl (data_msg->mid);
-      payload_msg = (struct GNUNET_MessageHeader *) &data_msg[1];
-      payload_type = ntohs (payload_msg->type);
-      strncpy (info, GC_m2s (payload_type), 31);
-      info[31] = '\0';
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    {
-      struct GNUNET_CADET_ChannelDataAckMessage *ack_msg;
-      ack_msg = (struct GNUNET_CADET_ChannelDataAckMessage *) message;
-      data_id = ntohl (ack_msg->mid);
-      SPRINTF (info, "0x%010lX",
-               (unsigned long int) ack_msg->futures);
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    {
-      struct GNUNET_CADET_ChannelOpenMessage *cc_msg;
-      cc_msg = (struct GNUNET_CADET_ChannelOpenMessage *) message;
-      SPRINTF (info, "  0x%08X", ntohl (cc_msg->ctn.cn));
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-    {
-      struct GNUNET_CADET_ChannelManageMessage *m_msg;
-      m_msg = (struct GNUNET_CADET_ChannelManageMessage *) message;
-      SPRINTF (info, "  0x%08X", ntohl (m_msg->ctn.cn));
-      break;
-    }
-    default:
-      info[0] = '\0';
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "==> %s (%12s %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (type), info, data_id,
-       GCCH_2s (ch), ch, GC_f2s (fwd), size);
 
-  if (GCT_is_loopback (ch->t))
+  if (NULL == ch)
   {
-    handle_loopback (ch, message, fwd);
+    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
     return;
   }
-
-  switch (type)
+  LOG2 (level,
+        "CHN %s:%X (%p)\n",
+        GCT_2s (ch->t),
+        ch->ctn,
+        ch);
+  if (NULL != ch->owner)
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      if (GNUNET_YES == ch->reliable)
-      {
-        chq = GNUNET_new (struct CadetChannelQueue);
-        chq->type = type;
-        if (NULL == existing_copy)
-          chq->copy = channel_save_copy (ch, message, fwd);
-        else
-        {
-          chq->copy = (struct CadetReliableMessage *) existing_copy;
-          if (NULL != chq->copy->chq)
-          {
-            /* Last retransmission was queued but not yet sent!
-             * This retransmission was scheduled by a ch_message_sent which
-             * followed a very fast RTT, so the tiny delay made the
-             * retransmission function to execute before the previous
-             * retransmitted message even had a chance to leave the peer.
-             * Cancel this message and wait until the pending
-             * retransmission leaves the peer and ch_message_sent starts
-             * the timer for the next one.
-             */
-            GNUNET_free (chq);
-            LOG (GNUNET_ERROR_TYPE_DEBUG,
-                 "  exisitng copy not yet transmitted!\n");
-            return;
-          }
-          LOG (GNUNET_ERROR_TYPE_DEBUG,
-               "  using existing copy: %p {r:%p q:%p t:%u}\n",
-               existing_copy,
-               chq->copy->rel, chq->copy->chq, chq->copy->type);
-        }
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  new chq: %p\n", chq);
-        chq->copy->chq = chq;
-        chq->tq = GCT_send_prebuilt_message (message, ch->t, NULL,
-                                             GNUNET_YES,
-                                             &ch_message_sent, chq);
-        /* q itself is stored in copy */
-        GNUNET_assert (NULL != chq->tq || GNUNET_NO != ch->destroy);
-      }
-      else
-      {
-        fire_and_forget (message, ch, GNUNET_NO);
-      }
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      chq = GNUNET_new (struct CadetChannelQueue);
-      chq->type = type;
-      chq->rel = fwd ? ch->root_rel : ch->dest_rel;
-      if (NULL != chq->rel->uniq)
-      {
-        if (NULL != chq->rel->uniq->tq)
-        {
-          GCT_cancel (chq->rel->uniq->tq);
-          /* ch_message_sent is called, freeing and NULLing uniq */
-          GNUNET_break (NULL == chq->rel->uniq);
-        }
-        else
-        {
-          GNUNET_break (0);
-          GNUNET_free (chq->rel->uniq);
-        }
-      }
-
-      chq->rel->uniq = chq;
-      chq->tq = GCT_send_prebuilt_message (message, ch->t, NULL, GNUNET_YES,
-                                           &ch_message_sent, chq);
-      if (NULL == chq->tq)
-      {
-        GNUNET_break (0);
-        chq->rel->uniq = NULL;
-        GCT_debug (ch->t, GNUNET_ERROR_TYPE_ERROR);
-        GNUNET_free (chq);
-        chq = NULL;
-        return;
-      }
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      fire_and_forget (message, ch, GNUNET_YES);
-      break;
-
-
-    default:
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING, "type %s unknown!\n", GC_m2s (type));
-      fire_and_forget (message, ch, GNUNET_YES);
+    LOG2 (level,
+          "CHN origin %s ready %s local-id: %u\n",
+          GSC_2s (ch->owner->c),
+          ch->owner->client_ready ? "YES" : "NO",
+          ntohl (ch->owner->ccn.channel_of_client));
   }
+  if (NULL != ch->dest)
+  {
+    LOG2 (level,
+          "CHN destination %s ready %s local-id: %u\n",
+          GSC_2s (ch->dest->c),
+          ch->dest->client_ready ? "YES" : "NO",
+          ntohl (ch->dest->ccn.channel_of_client));
+  }
+  LOG2 (level,
+        "CHN  Message IDs recv: %d (%LLX), send: %d\n",
+        ntohl (ch->mid_recv.mid),
+        (unsigned long long) ch->mid_futures,
+        ntohl (ch->mid_send.mid));
 }
 
 
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch)
-{
-  static char buf[64];
-
-  if (NULL == ch)
-    return "(NULL Channel)";
 
-  SPRINTF (buf,
-           "%s:%s gid:%X (%X / %X)",
-           GCT_2s (ch->t),
-           GNUNET_h2s (&ch->port),
-           ntohl (ch->gid.cn),
-           ntohl (ch->lid_root.channel_of_client),
-           ntohl (ch->lid_dest.channel_of_client));
-
-  return buf;
-}
+/* end of gnunet-service-cadet-new_channel.c */
diff --git a/src/cadet/gnunet-service-cadet_channel.h b/src/cadet/gnunet-service-cadet_channel.h
index 9d4893269..16517c457 100644
--- a/src/cadet/gnunet-service-cadet_channel.h
+++ b/src/cadet/gnunet-service-cadet_channel.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,344 +20,261 @@
 
 /**
  * @file cadet/gnunet-service-cadet_channel.h
- * @brief cadet service; dealing with end-to-end channels
+ * @brief GNUnet CADET service with encryption
  * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMCH (Gnunet Cadet CHannel)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_CHANNEL_H
 #define GNUNET_SERVICE_CADET_CHANNEL_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_peer.h"
 #include "cadet_protocol.h"
-#include "cadet.h"
+
 
 /**
- * Struct containing all information regarding a channel to a remote client.
+ * A channel is a bidirectional connection between two CADET
+ * clients.  Communiation can be reliable, unreliable, in-order
+ * or out-of-order.  One client is the "local" client, this
+ * one initiated the connection.   The other client is the
+ * "incoming" client, this one listened on a port to accept
+ * the connection from the "local" client.
  */
 struct CadetChannel;
 
 
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_local.h"
-
-
 /**
- * Destroy a channel and free all resources.
+ * Hash the @a port and @a initiator and @a listener to 
+ * calculate the "challenge" @a h_port we send to the other
+ * peer on #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN.
  *
- * @param ch Channel to destroy.
+ * @param[out] h_port set to the hash of @a port, @a initiator and @a listener
+ * @param port cadet port, as seen by CADET clients
+ * @param listener peer that is listining on @a port
  */
 void
-GCCH_destroy (struct CadetChannel *ch);
+GCCH_hash_port (struct GNUNET_HashCode *h_port,
+                const struct GNUNET_HashCode *port,
+                const struct GNUNET_PeerIdentity *listener);
 
 
 /**
- * Get the channel's public ID.
+ * Get the static string for identification of the channel.
  *
  * @param ch Channel.
  *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch);
-
-
-/**
- * Get the channel tunnel.
- *
- * @param ch Channel to get the tunnel from.
- *
- * @return tunnel of the channel.
+ * @return Static string with the channel IDs.
  */
-struct CadetTunnel *
-GCCH_get_tunnel (const struct CadetChannel *ch);
+const char *
+GCCH_2s (const struct CadetChannel *ch);
 
 
 /**
- * Get free buffer space towards the client on a specific channel.
+ * Log channel info.
  *
  * @param ch Channel.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - 64]
+ * @param level Debug level to use.
  */
-unsigned int
-GCCH_get_buffer (struct CadetChannel *ch, int fwd);
+void
+GCCH_debug (struct CadetChannel *ch,
+            enum GNUNET_ErrorType level);
 
 
 /**
- * Get flow control status of end point: is client allow to send?
+ * Get the channel's public ID.
  *
  * @param ch Channel.
- * @param fwd Is query about FWD traffic? (Request root status).
  *
- * @return #GNUNET_YES if client is allowed to send us data.
+ * @return ID used to identify the channel with the remote peer.
  */
-int
-GCCH_get_allowed (struct CadetChannel *ch, int fwd);
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch);
 
 
 /**
- * Is the root client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
- *
- * @return #GNUNET_YES in case it is.
+ * Create a new channel.
+ *
+ * @param owner local client owning the channel
+ * @param owner_id local chid of this channel at the @a owner
+ * @param destination peer to which we should build the channel
+ * @param port desired port at @a destination
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-int
-GCCH_is_origin (struct CadetChannel *ch, int fwd);
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber owner_id,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options);
 
-/**
- * Is the destination client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
- *
- * @return #GNUNET_YES in case it is.
- */
-int
-GCCH_is_terminal (struct CadetChannel *ch, int fwd);
 
 /**
- * Send an end-to-end ACK message for the most recent in-sequence payload.
- *
- * If channel is not reliable, do nothing.
+ * A client is bound to the port that we have a channel
+ * open to.  Send the acknowledgement for the connection
+ * request and establish the link with the client.
  *
- * @param ch Channel this is about.
- * @param fwd Is for FWD traffic? (ACK dest->owner)
+ * @param ch open incoming channel
+ * @param c client listening on the respective @a port
+ * @param port port number @a c is listening on
  */
 void
-GCCH_send_data_ack (struct CadetChannel *ch, int fwd);
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c,
+           const struct GNUNET_HashCode *port);
 
-/**
- * Notify the destination client that a new incoming channel was created.
- *
- * @param ch Channel that was created.
- */
-void
-GCCH_send_create (struct CadetChannel *ch);
 
 /**
- * Allow a client to send us more data, in case it was choked.
+ * Destroy locally created channel.  Called by the
+ * local client, so no need to tell the client.
  *
- * @param ch Channel.
- * @param fwd Is this about FWD traffic? (Root client).
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
  */
 void
-GCCH_allow_client (struct CadetChannel *ch, int fwd);
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn);
 
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch, enum GNUNET_ErrorType level);
 
 /**
- * Handle an ACK given by a client.
- *
- * Mark client as ready and send him any buffered data we could have for him.
- *
- * @param ch Channel.
- * @param fwd Is this a "FWD ACK"? (FWD ACKs are sent by root and go BCK)
- */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch, int fwd);
-
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if channel
- * is reliable and send an ACK to the client if there is still buffer space
- * in the tunnel.
- *
- * @param ch Channel.
- * @param c Client which sent the data.
- * @param fwd Is this a FWD data?
- * @param message Data message.
- * @param size Size of data.
- *
- * @return GNUNET_OK if everything goes well, GNUNET_SYSERR in case of en error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct CadetClient *c, int fwd,
-                        const struct GNUNET_MessageHeader *message,
-                        size_t size);
-
-/**
- * Handle a channel destroy requested by a client.
- *
- * Destroy the channel and the tunnel in case this was the last channel.
- *
- * @param ch Channel.
- * @param c Client that requested the destruction (to avoid notifying him).
- * @param is_root Is the request coming from root?
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
  */
 void
-GCCH_handle_local_destroy (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int is_root);
-
+GCCH_tunnel_up (struct CadetChannel *ch);
 
-/**
- * Handle a channel create requested by a client.
- *
- * Create the channel and the tunnel in case this was the first0 channel.
- *
- * @param c Client that requested the creation (will be the root).
- * @param msg Create Channel message.
- *
- * @return #GNUNET_OK if everything went fine, #GNUNET_SYSERR otherwise.
- */
-int
-GCCH_handle_local_create (struct CadetClient *c,
-                          struct GNUNET_CADET_LocalChannelCreateMessage *msg);
 
 /**
- * Handler for cadet network payload traffic.
- *
- * @param ch Channel for the message.
- * @param msg Unencryted data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * Create a new channel based on a request coming in over the network.
+ *
+ * @param t tunnel to the remote peer
+ * @param chid identifier of this channel in the tunnel
+ * @param origin peer to who initiated the channel
+ * @param h_port hash of desired local port
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-void
-GCCH_handle_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd);
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber chid,
+                           const struct GNUNET_HashCode *h_port,
+                           uint32_t options);
 
 
 /**
- * Handler for cadet network traffic end-to-end ACKs.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
  *
- * @param ch Channel on which we got this message.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
  */
 void
-GCCH_handle_data_ack (struct CadetChannel *ch,
-                      const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                      int fwd);
+GCCH_handle_duplicate_open (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
 
 
-/**
- * Handler for channel create messages.
- *
- * Does not have fwd parameter because it's always 'FWD': channel is incoming.
- *
- * @param t Tunnel this channel will be in.
- * @param msg Channel crate message.
- */
-struct CadetChannel *
-GCCH_handle_create (struct CadetTunnel *t,
-                    const struct GNUNET_CADET_ChannelOpenMessage *msg);
-
 
 /**
- * Handler for channel NACK messages.
- *
- * NACK messages always go dest -> root, no need for 'fwd' or 'msg' parameter.
+ * We got payload data for a channel.  Pass it on to the client.
  *
- * @param ch Channel.
+ * @param ch channel that got data
+ * @param cti identifier of the connection that delivered the message
+ * @param msg message that was received
  */
 void
-GCCH_handle_nack (struct CadetChannel *ch);
+GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
+                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg);
 
 
 /**
- * Handler for channel ack messages.
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
  *
- * @param ch Channel this channel is to be created in.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
  */
 void
-GCCH_handle_ack (struct CadetChannel *ch,
-                 const struct GNUNET_CADET_ChannelManageMessage *msg,
-                 int fwd);
+GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
+                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack);
 
 
 /**
- * Handler for channel destroy messages.
+ * We got an acknowledgement for the creation of the channel
+ * (the port is open on the other side). Begin transmissions.
  *
- * @param ch Channel this channel is to be destroyed of.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *        NULL if the ACK was inferred because we got payload or are on loopback
+ * @param port port number (needed to verify receiver knows the port)
  */
 void
-GCCH_handle_destroy (struct CadetChannel *ch,
-                     const struct GNUNET_CADET_ChannelManageMessage *msg,
-                     int fwd);
+GCCH_handle_channel_open_ack (struct CadetChannel *ch,
+                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                              const struct GNUNET_HashCode *port);
 
 
 /**
- * Sends an already built message on a channel.
- *
- * If the channel is on a loopback tunnel, notifies the appropriate destination
- * client locally.
+ * Destroy channel, based on the other peer closing the
+ * connection.  Also needs to remove this channel from
+ * the tunnel.
  *
- * On a normal channel passes the message to the tunnel for encryption and
- * sending on a connection.
+ * FIXME: need to make it possible to defer destruction until we have
+ * received all messages up to the destroy, and right now the destroy
+ * message (and this API) fails to give is the information we need!
  *
- * This function DOES NOT save the message for retransmission.
+ * FIXME: also need to know if the other peer got a destroy from
+ * us before!
  *
- * @param message Message to send. Function makes a copy of it.
- * @param ch Channel on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param existing_copy This is a retransmission, don't save a copy.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *            NULL during shutdown
  */
 void
-GCCH_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                            struct CadetChannel *ch, int fwd,
-                            void *existing_copy);
+GCCH_handle_remote_destroy (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
 
 
 /**
- * Get the static string for identification of the channel.
+ * Handle data given by a client.
  *
- * @param ch Channel.i
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
  *
- * @return Static string with the channel IDs.
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
  */
-const char *
-GCCH_2s (const struct CadetChannel *ch);
-
-
+int
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len);
 
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
+/**
+ * Handle ACK from client on local channel.
+ *
+ * @param ch channel to destroy
+ * @param client_ccn ccn of the client sending the ack
+ */
+void
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn);
 
-/* ifndef GNUNET_SERVICE_CADET_CHANNEL_H */
 #endif
-/* end of gnunet-service-cadet_channel.h */
diff --git a/src/cadet/gnunet-service-cadet_connection.c b/src/cadet/gnunet-service-cadet_connection.c
index af27647b3..7b66f61a2 100644
--- a/src/cadet/gnunet-service-cadet_connection.c
+++ b/src/cadet/gnunet-service-cadet_connection.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001-2015 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,242 +17,126 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
+
 /**
  * @file cadet/gnunet-service-cadet_connection.c
- * @brief GNUnet CADET service connection handling
+ * @brief management of CORE-level end-to-end connections; establishes
+ *        end-to-end routes and transmits messages along the route
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
 #include "platform.h"
-#include "gnunet_util_lib.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet_cadet_service.h"
 #include "gnunet_statistics_service.h"
-#include "cadet_path.h"
 #include "cadet_protocol.h"
-#include "cadet.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_peer.h"
-#include "gnunet-service-cadet_tunnel.h"
-
-
-/**
- * Should we run somewhat expensive checks on our invariants?
- */
-#define CHECK_INVARIANTS 0
-
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-con",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
-
-
-#define CADET_MAX_POLL_TIME      GNUNET_TIME_relative_multiply (\
-                                  GNUNET_TIME_UNIT_MINUTES,\
-                                  10)
-#define AVG_MSGS                32
-
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetConnectionQueue
-{
 
-  struct CadetConnectionQueue *next;
-  struct CadetConnectionQueue *prev;
 
-  /**
-   * Peer queue handle, to cancel if necessary.
-   */
-  struct CadetPeerQueue *peer_q;
-
-  /**
-   * Continuation to call once sent.
-   */
-  GCC_sent cont;
-
-  /**
-   * Closure for @e cont.
-   */
-  void *cont_cls;
-
-  /**
-   * Was this a forced message? (Do not account for it)
-   */
-  int forced;
-};
+#define LOG(level, ...) GNUNET_log_from(level,"cadet-con",__VA_ARGS__)
 
 
 /**
- * Struct to encapsulate all the Flow Control information to a peer to which
- * we are directly connected (on a core level).
+ * All the states a connection can be in.
  */
-struct CadetFlowControl
+enum CadetConnectionState
 {
   /**
-   * Connection this controls.
-   */
-  struct CadetConnection *c;
-
-  struct CadetConnectionQueue *q_head;
-  struct CadetConnectionQueue *q_tail;
-
-  /**
-   * How many messages are in the queue on this connection.
-   */
-  unsigned int queue_n;
-
-  /**
-   * How many messages do we accept in the queue.
-   * If 0, the connection is broken in this direction (next hop disconnected).
-   */
-  unsigned int queue_max;
-
-  /**
-   * ID of the next packet to send.
-   */
-  struct CadetEncryptedMessageIdentifier next_pid;
-
-  /**
-   * ID of the last packet sent towards the peer.
-   */
-  struct CadetEncryptedMessageIdentifier last_pid_sent;
-
-  /**
-   * ID of the last packet received from the peer.
-   */
-  struct CadetEncryptedMessageIdentifier last_pid_recv;
-
-  /**
-   * Bitmap of past 32 messages received:
-   * - LSB being @c last_pid_recv.
-   * - MSB being @c last_pid_recv - 31 (mod UINTMAX).
-   */
-  uint32_t recv_bitmap;
-
-  /**
-   * Last ACK sent to the peer (peer is not allowed to send
-   * messages with PIDs higher than this value).
+   * Uninitialized status, we have not yet even gotten the message queue.
    */
-  struct CadetEncryptedMessageIdentifier last_ack_sent;
+  CADET_CONNECTION_NEW,
 
   /**
-   * Last ACK sent towards the origin (for traffic towards leaf node).
+   * Connection create message in queue, awaiting transmission by CORE.
    */
-  struct CadetEncryptedMessageIdentifier last_ack_recv;
+  CADET_CONNECTION_SENDING_CREATE,
 
   /**
-   * Task to poll the peer in case of a lost ACK causes stall.
+   * Connection create message sent, waiting for ACK.
    */
-  struct GNUNET_SCHEDULER_Task *poll_task;
+  CADET_CONNECTION_SENT,
 
   /**
-   * How frequently to poll for ACKs.
+   * We are an inbound connection, and received a CREATE. Need to
+   * send an CREATE_ACK back.
    */
-  struct GNUNET_TIME_Relative poll_time;
+  CADET_CONNECTION_CREATE_RECEIVED,
 
   /**
-   * Queued poll message, to cancel if not necessary anymore (got ACK).
+   * Connection confirmed, ready to carry traffic.
    */
-  struct CadetConnectionQueue *poll_msg;
+  CADET_CONNECTION_READY
 
-  /**
-   * Queued poll message, to cancel if not necessary anymore (got ACK).
-   */
-  struct CadetConnectionQueue *ack_msg;
 };
 
+
 /**
- * Keep a record of the last messages sent on this connection.
+ * Low-level connection to a destination.
  */
-struct CadetConnectionPerformance
+struct CadetConnection
 {
-  /**
-   * Circular buffer for storing measurements.
-   */
-  double usecsperbyte[AVG_MSGS];
 
   /**
-   * Running average of @c usecsperbyte.
-   */
-  double avg;
-
-  /**
-   * How many values of @c usecsperbyte are valid.
-   */
-  uint16_t size;
-
-  /**
-   * Index of the next "free" position in @c usecsperbyte.
+   * ID of the connection.
    */
-  uint16_t idx;
-};
-
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 
-/**
- * Struct containing all information regarding a connection to a peer.
- */
-struct CadetConnection
-{
   /**
-   * Tunnel this connection is part of.
+   * To which peer does this connection go?
    */
-  struct CadetTunnel *t;
+  struct CadetPeer *destination;
 
   /**
-   * Flow control information for traffic fwd.
+   * Which tunnel is using this connection?
    */
-  struct CadetFlowControl fwd_fc;
+  struct CadetTConnection *ct;
 
   /**
-   * Flow control information for traffic bck.
+   * Path we are using to our destination.
    */
-  struct CadetFlowControl bck_fc;
+  struct CadetPeerPath *path;
 
   /**
-   * Measure connection performance on the endpoint.
+   * Pending message, NULL if we are ready to transmit.
    */
-  struct CadetConnectionPerformance *perf;
+  struct GNUNET_MQ_Envelope *env;
 
   /**
-   * ID of the connection.
+   * Handle for calling #GCP_request_mq_cancel() once we are finished.
    */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier id;
+  struct GCP_MessageQueueManager *mq_man;
 
   /**
-   * Path being used for the tunnel. At the origin of the connection
-   * it's a pointer to the destination's path pool, otherwise just a copy.
+   * Task for connection maintenance.
    */
-  struct CadetPeerPath *path;
+  struct GNUNET_SCHEDULER_Task *task;
 
   /**
-   * Task to keep the used paths alive at the owner,
-   * time tunnel out on all the other peers.
+   * Queue entry for keepalive messages.
    */
-  struct GNUNET_SCHEDULER_Task *fwd_maintenance_task;
+  struct CadetTunnelQueueEntry *keepalive_qe;
 
   /**
-   * Task to keep the used paths alive at the destination,
-   * time tunnel out on all the other peers.
+   * Function to call once we are ready to transmit.
    */
-  struct GNUNET_SCHEDULER_Task *bck_maintenance_task;
+  GCC_ReadyCallback ready_cb;
 
   /**
-   * Queue handle for maintainance traffic. One handle for FWD and BCK since
-   * one peer never needs to maintain both directions (no loopback connections).
+   * Closure for @e ready_cb.
    */
-  struct CadetPeerQueue *maintenance_q;
+  void *ready_cb_cls;
 
   /**
-   * Should equal #get_next_hop(), or NULL if that peer disconnected.
+   * How long do we wait before we try again with a CREATE message?
    */
-  struct CadetPeer *next_peer;
+  struct GNUNET_TIME_Relative retry_delay;
 
   /**
-   * Should equal #get_prev_hop(), or NULL if that peer disconnected.
+   * Performance metrics for this connection.
    */
-  struct CadetPeer *prev_peer;
+  struct CadetConnectionMetrics metrics;
 
   /**
    * State of the connection.
@@ -260,221 +144,36 @@ struct CadetConnection
   enum CadetConnectionState state;
 
   /**
-   * Position of the local peer in the path.
+   * Options for the route, control buffering.
    */
-  unsigned int own_pos;
+  enum GNUNET_CADET_ChannelOption options;
 
   /**
-   * Pending message count.
+   * How many latency observations did we make for this connection?
    */
-  unsigned int pending_messages;
+  unsigned int latency_datapoints;
 
   /**
-   * Destroy flag:
-   * - if 0, connection in use.
-   * - if 1, destroy on last message.
-   * - if 2, connection is being destroyed don't re-enter.
+   * Offset of our @e destination in @e path.
    */
-  int destroy;
+  unsigned int off;
 
   /**
-   * In-connection-map flag. Sometimes, when @e destroy is set but
-   * actual destruction is delayed to enable us to finish processing
-   * queues (i.e. in the direction that is still working), we remove
-   * the connection from the map to prevent it from still being
-   * found (and used) by accident. This flag is set to #GNUNET_YES
-   * for a connection that is not in the #connections map.  Should
-   * only be #GNUNET_YES if #destroy is also non-zero.
+   * Are we ready to transmit via @e mq_man right now?
    */
-  int was_removed;
+  int mqm_ready;
 
-  /**
-   * Counter to do exponential backoff when creating a connection (max 64).
-   */
-  unsigned short create_retry;
-
-  /**
-   * Task to check if connection has duplicates.
-   */
-  struct GNUNET_SCHEDULER_Task *check_duplicates_task;
 };
 
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-/**
- * Connections known, indexed by cid (CadetConnection).
- */
-static struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-/**
- * How many connections are we willing to maintain.
- *  Local connections are always allowed,
- * even if there are more connections than max.
- */
-static unsigned long long max_connections;
-
-/**
- * How many messages *in total* are we willing to queue, divide by number of
- * connections to get connection queue size.
- */
-static unsigned long long max_msgs_queue;
-
-/**
- * How often to send path keepalives. Paths timeout after 4 missed.
- */
-static struct GNUNET_TIME_Relative refresh_connection_time;
-
-/**
- * How often to send path create / ACKs.
- */
-static struct GNUNET_TIME_Relative create_connection_time;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
-
-#if 0 // avoid compiler warning for unused static function
-static void
-fc_debug (struct CadetFlowControl *fc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    IN: %u/%u\n",
-       ntohl (fc->last_pid_recv.pid),
-       ntohl (fc->last_ack_sent.pid));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    OUT: %u/%u\n",
-              fc->last_pid_sent, fc->last_ack_recv);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    QUEUE: %u/%u\n",
-              fc->queue_n, fc->queue_max);
-}
-
-static void
-connection_debug (struct CadetConnection *c)
-{
-  if (NULL == c)
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, "DEBUG NULL CONNECTION\n");
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connection %s:%X\n",
-              peer2s (c->t->peer), GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  state: %u, pending msgs: %u\n",
-              c->state, c->pending_messages);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  FWD FC\n");
-  fc_debug (&c->fwd_fc);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  BCK FC\n");
-  fc_debug (&c->bck_fc);
-}
-#endif
-
-
-/**
- * Schedule next keepalive task, taking in consideration
- * the connection state and number of retries.
- *
- * @param c Connection for which to schedule the next keepalive.
- * @param fwd Direction for the next keepalive.
- */
-static void
-schedule_next_keepalive (struct CadetConnection *c, int fwd);
-
-
-/**
- * Resets the connection timeout task, some other message has done the
- * task's job.
- * - For the first peer on the direction this means to send
- *   a keepalive or a path confirmation message (either create or ACK).
- * - For all other peers, this means to destroy the connection,
- *   due to lack of activity.
- * Starts the timeout if no timeout was running (connection just created).
- *
- * @param c Connection whose timeout to reset.
- * @param fwd Is this forward?
- */
-static void
-connection_reset_timeout (struct CadetConnection *c, int fwd);
-
-
-/**
- * Get string description for tunnel state. Reentrant.
- *
- * @param s Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-GCC_state2s (enum CadetConnectionState s)
-{
-  switch (s)
-  {
-    case CADET_CONNECTION_NEW:
-      return "CADET_CONNECTION_NEW";
-    case CADET_CONNECTION_SENT:
-      return "CADET_CONNECTION_SENT";
-    case CADET_CONNECTION_ACK:
-      return "CADET_CONNECTION_ACK";
-    case CADET_CONNECTION_READY:
-      return "CADET_CONNECTION_READY";
-    case CADET_CONNECTION_DESTROYED:
-      return "CADET_CONNECTION_DESTROYED";
-    case CADET_CONNECTION_BROKEN:
-      return "CADET_CONNECTION_BROKEN";
-    default:
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_ERROR, " conn state %u unknown!\n", s);
-      return "CADET_CONNECTION_STATE_ERROR";
-  }
-}
-
-
 /**
- * Initialize a Flow Control structure to the initial state.
+ * Lookup a connection by its identifier.
  *
- * @param fc Flow Control structure to initialize.
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
  */
-static void
-fc_init (struct CadetFlowControl *fc)
-{
-  fc->next_pid.pid = 0;
-  fc->last_pid_sent.pid = htonl (UINT32_MAX);
-  fc->last_pid_recv.pid = htonl (UINT32_MAX);
-  fc->last_ack_sent.pid = (uint32_t) 0;
-  fc->last_ack_recv.pid = (uint32_t) 0;
-  fc->poll_task = NULL;
-  fc->poll_time = GNUNET_TIME_UNIT_SECONDS;
-  fc->queue_n = 0;
-  fc->queue_max = (max_msgs_queue / max_connections) + 1;
-}
-
-
-/**
- * Find a connection.
- *
- * @param cid Connection ID.
- *
- * @return conntection with the given ID @cid or NULL if not found.
- */
-static struct CadetConnection *
-connection_get (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
   return GNUNET_CONTAINER_multishortmap_get (connections,
                                              &cid->connection_of_tunnel);
@@ -482,3232 +181,911 @@ connection_get (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 
 
 /**
- * Change the connection state. Cannot change a connection marked as destroyed.
+ * Update the connection state. Also triggers the necessary
+ * MQM notifications.
  *
- * @param c Connection to change.
- * @param state New state to set.
+ * @param cc connection to update the state for
+ * @param new_state new state for @a cc
+ * @param new_mqm_ready new `mqm_ready` state for @a cc
  */
 static void
-connection_change_state (struct CadetConnection* c,
-                         enum CadetConnectionState state)
+update_state (struct CadetConnection *cc,
+              enum CadetConnectionState new_state,
+              int new_mqm_ready)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Connection %s state %s -> %s\n",
-       GCC_2s (c), GCC_state2s (c->state), GCC_state2s (state));
-  if (CADET_CONNECTION_DESTROYED <= c->state) /* Destroyed or broken. */
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "state not changing anymore\n");
-    return;
-  }
-  c->state = state;
-  if (CADET_CONNECTION_READY == state)
-    c->create_retry = 1;
-}
-
+  int old_ready;
+  int new_ready;
 
-/**
- * Mark a connection as "destroyed", to send all pending traffic and freeing
- * all associated resources, without accepting new status changes on it.
- *
- * @param c Connection to mark as destroyed.
- */
-static void
-mark_destroyed (struct CadetConnection *c)
-{
-  c->destroy = GNUNET_YES;
-  connection_change_state (c, CADET_CONNECTION_DESTROYED);
+  if ( (new_state == cc->state) &&
+       (new_mqm_ready == cc->mqm_ready) )
+    return; /* no change, nothing to do */
+  old_ready = ( (CADET_CONNECTION_READY == cc->state) &&
+                (GNUNET_YES == cc->mqm_ready) );
+  new_ready = ( (CADET_CONNECTION_READY == new_state) &&
+                (GNUNET_YES == new_mqm_ready) );
+  cc->state = new_state;
+  cc->mqm_ready = new_mqm_ready;
+  if (old_ready != new_ready)
+    cc->ready_cb (cc->ready_cb_cls,
+                  new_ready);
 }
 
 
 /**
- * Function called if a connection has been stalled for a while,
- * possibly due to a missed ACK. Poll the neighbor about its ACK status.
- *
- * @param cls Closure (poll ctx).
- */
-static void
-send_poll (void *cls);
-
-
-/**
- * Send an ACK on the connection, informing the predecessor about
- * the available buffer space. Should not be called in case the peer
- * is origin (no predecessor) in the @c fwd direction.
- *
- * Note that for fwd ack, the FWD mean forward *traffic* (root->dest),
- * the ACK itself goes "back" (dest->root).
+ * Destroy a connection, part of the internal implementation.  Called
+ * only from #GCC_destroy_from_core() or #GCC_destroy_from_tunnel().
  *
- * @param c Connection on which to send the ACK.
- * @param buffer How much space free to advertise?
- * @param fwd Is this FWD ACK? (Going dest -> root)
- * @param force Don't optimize out.
+ * @param cc connection to destroy
  */
 static void
-send_ack (struct CadetConnection *c,
-          unsigned int buffer,
-          int fwd,
-          int force)
+GCC_destroy (struct CadetConnection *cc)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetFlowControl *next_fc;
-  struct CadetFlowControl *prev_fc;
-  struct GNUNET_CADET_ConnectionEncryptedAckMessage msg;
-  struct CadetEncryptedMessageIdentifier ack_cemi;
-  int delta;
-
-  GCC_check_connections ();
-  GNUNET_assert (GNUNET_NO == GCC_is_origin (c, fwd));
-
-  next_fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  prev_fc = fwd ? &c->bck_fc : &c->fwd_fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send %s ack on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  /* Check if we need to transmit the ACK. */
-  delta = ntohl (prev_fc->last_ack_sent.pid) - ntohl (prev_fc->last_pid_recv.pid);
-  if (3 < delta && buffer < delta && GNUNET_NO == force)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Not sending ACK, delta > 3\n");
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  last pid recv: %u, last ack sent: %u\n",
-         ntohl (prev_fc->last_pid_recv.pid),
-         ntohl (prev_fc->last_ack_sent.pid));
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Ok, ACK might be necessary, what PID to ACK? */
-  ack_cemi.pid = htonl (ntohl (prev_fc->last_pid_recv.pid) + buffer);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " ACK %u, last PID %u, last ACK %u, qmax %u, q %u\n",
-       ntohl (ack_cemi.pid),
-       ntohl (prev_fc->last_pid_recv.pid),
-       ntohl (prev_fc->last_ack_sent.pid),
-       next_fc->queue_max, next_fc->queue_n);
-  if ( (ack_cemi.pid == prev_fc->last_ack_sent.pid) &&
-       (GNUNET_NO == force) )
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Not sending FWD ACK, not needed\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Check if message is already in queue */
-  if (NULL != prev_fc->ack_msg)
-  {
-    if (GC_is_pid_bigger (ntohl (ack_cemi.pid),
-                          ntohl (prev_fc->last_ack_sent.pid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " canceling old ACK\n");
-      GCC_cancel (prev_fc->ack_msg);
-      /* GCC_cancel triggers ack_sent(), which clears fc->ack_msg */
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " same ACK already in queue\n");
-      GCC_check_connections ();
-      return;
-    }
-  }
-  GNUNET_break (GC_is_pid_bigger (ntohl (ack_cemi.pid),
-                                  ntohl (prev_fc->last_ack_sent.pid)));
-  prev_fc->last_ack_sent = ack_cemi;
-
-  /* Build ACK message and send on conn */
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK);
-  msg.cemi_max = ack_cemi;
-  msg.cid = c->id;
-
-  prev_fc->ack_msg = GCC_send_prebuilt_message (&msg.header,
-                                                UINT16_MAX,
-                                                zero,
-                                                c,
-                                                !fwd,
-                                                GNUNET_YES,
-                                                NULL, NULL);
-  GNUNET_assert (NULL != prev_fc->ack_msg);
-  GCC_check_connections ();
-}
-
-
-/**
- * Update performance information if we are a connection's endpoint.
- *
- * @param c Connection to update.
- * @param wait How much time did we wait to send the last message.
- * @param size Size of the last message.
- */
-static void
-update_perf (struct CadetConnection *c,
-             struct GNUNET_TIME_Relative wait,
-             uint16_t size)
-{
-  struct CadetConnectionPerformance *p;
-  double usecsperbyte;
-
-  if (NULL == c->perf)
-    return; /* Only endpoints are interested in timing. */
-
-  p = c->perf;
-  usecsperbyte = ((double) wait.rel_value_us) / size;
-  if (p->size == AVG_MSGS)
-  {
-    /* Array is full. Substract oldest value, add new one and store. */
-    p->avg -= (p->usecsperbyte[p->idx] / AVG_MSGS);
-    p->usecsperbyte[p->idx] = usecsperbyte;
-    p->avg += (p->usecsperbyte[p->idx] / AVG_MSGS);
-  }
-  else
-  {
-    /* Array not yet full. Add current value to avg and store. */
-    p->usecsperbyte[p->idx] = usecsperbyte;
-    p->avg *= p->size;
-    p->avg += p->usecsperbyte[p->idx];
-    p->size++;
-    p->avg /= p->size;
-  }
-  p->idx = (p->idx + 1) % AVG_MSGS;
-}
-
-
-/**
- * Callback called when a connection queued message is sent.
- *
- * Calculates the average time and connection packet tracking.
- *
- * @param cls Closure (ConnectionQueue Handle), can be NULL.
- * @param c Connection this message was on.
- * @param fwd Was this a FWD going message?
- * @param sent Was it really sent? (Could have been canceled)
- * @param type Type of message sent.
- * @param payload_type Type of payload, if applicable.
- * @param pid Message ID, or 0 if not applicable (create, destroy, etc).
- * @param size Size of the message.
- * @param wait Time spent waiting for core (only the time for THIS message)
- */
-static void
-conn_message_sent (void *cls,
-                   struct CadetConnection *c,
-                   int fwd,
-                   int sent,
-                   uint16_t type,
-                   uint16_t payload_type,
-                   struct CadetEncryptedMessageIdentifier pid,
-                   size_t size,
-                   struct GNUNET_TIME_Relative wait)
-{
-  struct CadetConnectionQueue *q = cls;
-  struct CadetFlowControl *fc;
-  int forced;
-
-  GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         ">>> %s (%s %4u) on conn %s (%p) %s [%5u] in queue %s\n",
-         GC_m2s (type), GC_m2s (payload_type),
-         ntohl (pid.pid),
-         GCC_2s (c),
-         c,
-         GC_f2s (fwd), size,
-         GNUNET_STRINGS_relative_time_to_string (wait, GNUNET_YES));
-
-  /* If c is NULL, nothing to update. */
-  if (NULL == c)
-  {
-    if (type != GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
-        && type != GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR, "Message %s sent on NULL connection!\n",
-           GC_m2s (type));
-    }
-    GCC_check_connections ();
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " %ssent %s %s pid %u\n",
-       sent ? "" : "not ", GC_f2s (fwd),
-       GC_m2s (type), GC_m2s (payload_type),
-       ntohl (pid.pid));
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  /* Update flow control info. */
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  if (NULL != q)
+       "Destroying %s\n",
+       GCC_2s (cc));
+  if (NULL != cc->mq_man)
   {
-    GNUNET_CONTAINER_DLL_remove (fc->q_head, fc->q_tail, q);
-    forced = q->forced;
-    if (NULL != q->cont)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " calling cont\n");
-      q->cont (q->cont_cls, c, q, type, fwd, size);
-    }
-    GNUNET_free (q);
+    GCP_request_mq_cancel (cc->mq_man,
+                           NULL);
+    cc->mq_man = NULL;
   }
-  else /* CONN_CREATE or CONN_ACK */
+  if (NULL != cc->task)
   {
-    GNUNET_assert (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED != type);
-    forced = GNUNET_YES;
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
   }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " C_P- %p %u\n", c, c->pending_messages);
-  c->pending_messages--;
-  if ( (GNUNET_YES == c->destroy) &&
-       (0 == c->pending_messages) )
+  if (NULL != cc->keepalive_qe)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "!  destroying connection!\n");
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
+    GCT_send_cancel (cc->keepalive_qe);
+    cc->keepalive_qe = NULL;
   }
-
-  /* Send ACK if needed, after accounting for sent ID in fc->queue_n */
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-      c->maintenance_q = NULL;
-      /* Don't trigger a keepalive for sent ACKs, only SYN and SYNACKs */
-      if (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE == type || !fwd)
-        schedule_next_keepalive (c, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      if (GNUNET_YES == sent)
-      {
-        fc->last_pid_sent = pid;
-        if (GC_is_pid_bigger (ntohl (fc->last_pid_sent.pid) + 1,
-                              ntohl (fc->last_ack_recv.pid)) )
-          GCC_start_poll (c, fwd);
-        GCC_send_ack (c, fwd, GNUNET_NO);
-        connection_reset_timeout (c, fwd);
-      }
-
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "!  Q_N- %p %u\n", fc, fc->queue_n);
-      if (GNUNET_NO == forced)
-      {
-        fc->queue_n--;
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "!   accounting pid %u\n",
-             ntohl (fc->last_pid_sent.pid));
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "!   forced, Q_N not accounting pid %u\n",
-             ntohl (fc->last_pid_sent.pid));
-      }
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      if (GNUNET_YES == sent)
-        connection_reset_timeout (c, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-      fc->poll_msg = NULL;
-      if (2 == c->destroy)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL canceled on shutdown\n");
-        return;
-      }
-      if (0 == fc->queue_max)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL cancelled: neighbor disconnected\n");
-        return;
-      }
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL sent for %s, scheduling new one!\n",
-          GCC_2s (c));
-      GNUNET_assert (NULL == fc->poll_task);
-      fc->poll_time = GNUNET_TIME_STD_BACKOFF (fc->poll_time);
-      fc->poll_task = GNUNET_SCHEDULER_add_delayed (fc->poll_time,
-                                                    &send_poll, fc);
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " task %u\n", fc->poll_task);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-      fc->ack_msg = NULL;
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      break;
-
-    default:
-      LOG (GNUNET_ERROR_TYPE_ERROR, "%s unknown\n", GC_m2s (type));
-      GNUNET_break (0);
-      break;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "!  message sent!\n");
-
-  update_perf (c, wait, size);
-  GCC_check_connections ();
-}
-
-
-/**
- * Get the previous hop in a connection
- *
- * @param c Connection.
- *
- * @return Previous peer in the connection.
- */
-static struct CadetPeer *
-get_prev_hop (const struct CadetConnection *c)
-{
-  GNUNET_PEER_Id id;
-
-  if (NULL == c->path)
-    return NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " get prev hop %s [%u/%u]\n",
-       GCC_2s (c), c->own_pos, c->path->length);
-  if (0 == c->own_pos || c->path->length < 2)
-    id = c->path->peers[0];
-  else
-    id = c->path->peers[c->own_pos - 1];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  ID: %s (%u)\n",
-       GNUNET_i2s (GNUNET_PEER_resolve2 (id)), id);
-
-  return GCP_get_short (id, GNUNET_YES);
-}
-
-
-/**
- * Get the next hop in a connection
- *
- * @param c Connection.
- *
- * @return Next peer in the connection.
- */
-static struct CadetPeer *
-get_next_hop (const struct CadetConnection *c)
-{
-  GNUNET_PEER_Id id;
-
-  if (NULL == c->path)
-    return NULL;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " get next hop %s [%u/%u]\n",
-       GCC_2s (c), c->own_pos, c->path->length);
-  if ((c->path->length - 1) == c->own_pos || c->path->length < 2)
-    id = c->path->peers[c->path->length - 1];
-  else
-    id = c->path->peers[c->own_pos + 1];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  ID: %s (%u)\n",
-       GNUNET_i2s (GNUNET_PEER_resolve2 (id)), id);
-
-  return GCP_get_short (id, GNUNET_YES);
+  GCPP_del_connection (cc->path,
+                       cc->off,
+                       cc);
+  for (unsigned int i=0;i<cc->off;i++)
+    GCP_remove_connection (GCPP_get_peer_at_offset (cc->path,
+                                                    i),
+                           cc);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (connections,
+                                                        &GCC_get_id (cc)->connection_of_tunnel,
+                                                        cc));
+  GNUNET_free (cc);
 }
 
 
-/**
- * Check that the direct neighbours (previous and next hop)
- * are properly associated with this connection.
- *
- * @param c connection to check
- */
-static void
-check_neighbours (const struct CadetConnection *c)
-{
-  if (NULL == c->path)
-    return; /* nothing to check */
-  GCP_check_connection (get_next_hop (c), c);
-  GCP_check_connection (get_prev_hop (c), c);
-}
-
 
 /**
- * Helper for #GCC_check_connections().  Calls #check_neighbours().
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
  *
- * @param cls NULL
- * @param key ignored
- * @param value the `struct CadetConnection` to check
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-check_connection (void *cls,
-                  const struct GNUNET_ShortHashCode *key,
-                  void *value)
-{
-  struct CadetConnection *c = value;
-
-  check_neighbours (c);
-  return GNUNET_OK;
-}
-
-
-/**
- * Check invariants for all connections using #check_neighbours().
+ * @param cc connection to destroy
  */
 void
-GCC_check_connections ()
-{
-  if (0 == CHECK_INVARIANTS)
-    return;
-  if (NULL == connections)
-    return;
-  GNUNET_CONTAINER_multishortmap_iterate (connections,
-                                          &check_connection,
-                                          NULL);
-}
-
-
-/**
- * Get the hop in a connection.
- *
- * @param c Connection.
- * @param fwd Next in the FWD direction?
- *
- * @return Next peer in the connection.
- */
-static struct CadetPeer *
-get_hop (struct CadetConnection *c, int fwd)
-{
-  return (fwd) ? get_next_hop (c) : get_prev_hop (c);
-}
-
-
-/**
- * Get a bit mask for a message received out-of-order.
- *
- * @param last_pid_recv Last PID we received prior to the out-of-order.
- * @param ooo_pid PID of the out-of-order message.
- */
-static uint32_t
-get_recv_bitmask (struct CadetEncryptedMessageIdentifier last_pid_recv,
-                  struct CadetEncryptedMessageIdentifier ooo_pid)
-{
-  // FIXME: should assert that the delta is in range...
-  return 1 << (ntohl (last_pid_recv.pid) - ntohl (ooo_pid.pid));
-}
-
-
-/**
- * Check is an out-of-order message is ok:
- * - at most 31 messages behind.
- * - not duplicate.
- *
- * @param last_pid_recv Last in-order PID received.
- */
-static int
-is_ooo_ok (struct CadetEncryptedMessageIdentifier last_pid_recv,
-           struct CadetEncryptedMessageIdentifier ooo_pid,
-           uint32_t ooo_bitmap)
-{
-  uint32_t mask;
-
-  if (GC_is_pid_bigger (ntohl (last_pid_recv.pid) - 31,
-                        ntohl (ooo_pid.pid)))
-    return GNUNET_NO;
-
-  mask = get_recv_bitmask (last_pid_recv,
-                           ooo_pid);
-  if (0 != (ooo_bitmap & mask))
-    return GNUNET_NO;
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Is traffic coming from this sender 'FWD' traffic?
- *
- * @param c Connection to check.
- * @param sender Short peer identity of neighbor.
- *
- * @return #GNUNET_YES in case the sender is the 'prev' hop and therefore
- *         the traffic is 'FWD'.
- *         #GNUNET_NO for BCK.
- *         #GNUNET_SYSERR for errors (sender isn't a hop in the connection).
- */
-static int
-is_fwd (const struct CadetConnection *c,
-        const struct CadetPeer *sender)
-{
-  GNUNET_PEER_Id id;
-
-  id = GCP_get_short_id (sender);
-  if (GCP_get_short_id (get_prev_hop (c)) == id)
-    return GNUNET_YES;
-
-  if (GCP_get_short_id (get_next_hop (c)) == id)
-    return GNUNET_NO;
-
-  return GNUNET_SYSERR;
-}
-
-
-/**
- * Sends a CONNECTION ACK message in reponse to a received CONNECTION_CREATE
- * or a first CONNECTION_ACK directed to us.
- *
- * @param c Connection to confirm.
- * @param fwd Should we send it FWD? (root->dest)
- *            (First (~SYNACK) goes BCK, second (~ACK) goes FWD)
- */
-static void
-send_connection_ack (struct CadetConnection *c, int fwd)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionCreateAckMessage msg;
-  struct CadetTunnel *t;
-  const uint16_t size = sizeof (struct GNUNET_CADET_ConnectionCreateAckMessage);
-  const uint16_t type = GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK;
-
-  GCC_check_connections ();
-  t = c->t;
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "==> %s ({ C %s ACK}    0) on conn %s (%p) %s [%5u]\n",
-       GC_m2s (type), GC_f2s (!fwd), GCC_2s (c), c, GC_f2s (fwd), size);
-
-  msg.header.size = htons (size);
-  msg.header.type = htons (type);
-  msg.reserved = htonl (0);
-  msg.cid = c->id;
-
-  GNUNET_assert (NULL == c->maintenance_q);
-  c->maintenance_q = GCP_send (get_hop (c, fwd),
-                               &msg.header,
-                               GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK,
-                               zero,
-                               c,
-                               fwd,
-                               &conn_message_sent, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %p %u (conn`ACK)\n",
-       c, c->pending_messages);
-  c->pending_messages++;
-
-  if (CADET_TUNNEL_NEW == GCT_get_cstate (t))
-    GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-  if (CADET_CONNECTION_READY != c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-  GCC_check_connections ();
-}
-
-
-/**
- * Send a notification that a connection is broken.
- *
- * @param c Connection that is broken.
- * @param id1 Peer that has disconnected.
- * @param id2 Peer that has disconnected.
- * @param fwd Direction towards which to send it.
- */
-static void
-send_broken (struct CadetConnection *c,
-             const struct GNUNET_PeerIdentity *id1,
-             const struct GNUNET_PeerIdentity *id2,
-             int fwd)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionBrokenMessage msg;
-
-  GCC_check_connections ();
-  msg.header.size = htons (sizeof (struct GNUNET_CADET_ConnectionBrokenMessage));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
-  msg.cid = c->id;
-  msg.reserved = htonl (0);
-  msg.peer1 = *id1;
-  msg.peer2 = *id2;
-  (void) GCC_send_prebuilt_message (&msg.header,
-                                    UINT16_MAX,
-                                    zero,
-                                    c,
-                                    fwd,
-                                    GNUNET_YES,
-                                    NULL, NULL);
-  GCC_check_connections ();
-}
-
-
-/**
- * Send a notification that a connection is broken, when a connection
- * isn't even known to the local peer or soon to be destroyed.
- *
- * @param connection_id Connection ID.
- * @param id1 Peer that has disconnected, probably local peer.
- * @param id2 Peer that has disconnected can be NULL if unknown.
- * @param neighbor Peer to notify (neighbor who sent the connection).
- */
-static void
-send_broken_unknown (const struct GNUNET_CADET_ConnectionTunnelIdentifier *connection_id,
-                     const struct GNUNET_PeerIdentity *id1,
-                     const struct GNUNET_PeerIdentity *id2,
-                     struct CadetPeer *neighbor)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionBrokenMessage msg;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_INFO, "--> BROKEN on unknown connection %s\n",
-       GNUNET_sh2s (&connection_id->connection_of_tunnel));
-
-  msg.header.size = htons (sizeof (struct GNUNET_CADET_ConnectionBrokenMessage));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
-  msg.cid = *connection_id;
-  msg.reserved = htonl (0);
-  msg.peer1 = *id1;
-  if (NULL != id2)
-    msg.peer2 = *id2;
-  else
-    memset (&msg.peer2, 0, sizeof (msg.peer2));
-  GNUNET_assert (NULL != GCP_send (neighbor,
-                                   &msg.header,
-                                   UINT16_MAX,
-                                   zero,
-                                   NULL,
-                                   GNUNET_SYSERR, /* connection, fwd */
-                                   NULL, NULL)); /* continuation */
-  GCC_check_connections ();
-}
-
-
-/**
- * Send keepalive packets for a connection.
- *
- * @param c Connection to keep alive..
- * @param fwd Is this a FWD keepalive? (owner -> dest).
- */
-static void
-send_connection_keepalive (struct CadetConnection *c, int fwd)
+GCC_destroy_without_core (struct CadetConnection *cc)
 {
-  struct GNUNET_MessageHeader msg;
-  struct CadetFlowControl *fc;
-  int tunnel_ready;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "keepalive %s for connection %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  GNUNET_assert (NULL != c->t);
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  tunnel_ready = GNUNET_YES == GCT_has_queued_traffic (c->t)
-                 && CADET_TUNNEL_KEY_OK <= GCT_get_estate (c->t);
-  if (0 < fc->queue_n || tunnel_ready)
+  if (NULL != cc->ct)
   {
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, traffic in queue\n");
-    return;
+    GCT_connection_lost (cc->ct);
+    cc->ct = NULL;
   }
-
-  GNUNET_STATISTICS_update (stats, "# keepalives sent", 1, GNUNET_NO);
-
-  GNUNET_assert (NULL != c->t);
-  msg.size = htons (sizeof (msg));
-  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
-
-  GNUNET_assert (NULL ==
-                 GCT_send_prebuilt_message (&msg, c->t, c,
-                                            GNUNET_NO, NULL, NULL));
-  GCC_check_connections ();
-}
-
-
-/**
- * Send CONNECTION_{CREATE/ACK} packets for a connection.
- *
- * @param c Connection for which to send the message.
- * @param fwd If #GNUNET_YES, send CREATE, otherwise send ACK.
- */
-static void
-connection_recreate (struct CadetConnection *c, int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "sending connection recreate\n");
-  if (fwd)
-    GCC_send_create (c);
-  else
-    send_connection_ack (c, GNUNET_NO);
+  GCC_destroy (cc);
 }
 
 
 /**
- * Generic connection timer management.
- * Depending on the role of the peer in the connection will send the
- * appropriate message (build or keepalive)
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
  *
- * @param c Conncetion to maintain.
- * @param fwd Is FWD?
+ * @param cc connection to destroy
  */
-static void
-connection_maintain (struct CadetConnection *c, int fwd)
+void
+GCC_destroy_without_tunnel (struct CadetConnection *cc)
 {
-  if (GNUNET_NO != c->destroy)
+  cc->ct = NULL;
+  if ( (CADET_CONNECTION_SENDING_CREATE != cc->state) &&
+       (NULL != cc->mq_man) )
   {
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, being destroyed\n");
-    return;
-  }
-
-  if (NULL == c->t)
-  {
-    GNUNET_break (0);
-    GCC_debug (c, GNUNET_ERROR_TYPE_ERROR);
-    return;
-  }
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_ConnectionDestroyMessage *destroy_msg;
 
-  if (CADET_TUNNEL_SEARCHING == GCT_get_cstate (c->t))
-  {
-    /* If status is SEARCHING, why is there a connection? Should be WAITING */
-    GNUNET_break (0);
-    GCT_debug (c->t, GNUNET_ERROR_TYPE_ERROR);
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, tunnel SEARCHING\n");
-    schedule_next_keepalive (c, fwd);
-    return;
-  }
-  switch (c->state)
-  {
-    case CADET_CONNECTION_NEW:
-      GNUNET_break (0);
-      /* fall-through */
-    case CADET_CONNECTION_SENT:
-      connection_recreate (c, fwd);
-      break;
-    case CADET_CONNECTION_READY:
-      send_connection_keepalive (c, fwd);
-      break;
-    default:
-      break;
+    /* Need to notify next hop that we are down. */
+    env = GNUNET_MQ_msg (destroy_msg,
+                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
+    destroy_msg->cid = cc->cid;
+    GCP_request_mq_cancel (cc->mq_man,
+                           env);
+    cc->mq_man = NULL;
   }
+  GCC_destroy (cc);
 }
 
 
 /**
- * Keep the connection alive.
+ * Return the tunnel associated with this connection.
  *
- * @param c Connection to keep alive.
- * @param fwd Direction.
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
  */
-static void
-connection_keepalive (struct CadetConnection *c,
-                      int fwd)
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc)
 {
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s keepalive for %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  if (fwd)
-    c->fwd_maintenance_task = NULL;
-  else
-    c->bck_maintenance_task = NULL;
-  connection_maintain (c, fwd);
-  GCC_check_connections ();
-  /* Next execution will be scheduled by message_sent or _maintain*/
+  return cc->ct;
 }
 
 
 /**
- * Keep the connection alive in the FWD direction.
+ * Obtain performance @a metrics from @a cc.
  *
- * @param cls Closure (connection to keepalive).
+ * @param cc connection to query
+ * @return the metrics
  */
-static void
-connection_fwd_keepalive (void *cls)
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc)
 {
-  struct CadetConnection *c = cls;
-
-  GCC_check_connections ();
-  connection_keepalive (c,
-                        GNUNET_YES);
-  GCC_check_connections ();
+  return &cc->metrics;
 }
 
 
 /**
- * Keep the connection alive in the BCK direction.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
  *
- * @param cls Closure (connection to keepalive).
+ * @param cls the `struct CadetConnection` to keep alive.
  */
 static void
-connection_bck_keepalive (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  GCC_check_connections ();
-  connection_keepalive (c,
-                        GNUNET_NO);
-  GCC_check_connections ();
-}
+send_keepalive (void *cls);
 
 
 /**
- * Schedule next keepalive task, taking in consideration
- * the connection state and number of retries.
+ * Keepalive was transmitted.  Remember this, and possibly
+ * schedule the next one.
  *
- * If the peer is not the origin, do nothing.
- *
- * @param c Connection for which to schedule the next keepalive.
- * @param fwd Direction for the next keepalive.
+ * @param cls the `struct CadetConnection` to keep alive.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-schedule_next_keepalive (struct CadetConnection *c, int fwd)
+keepalive_done (void *cls,
+                const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct GNUNET_TIME_Relative delay;
-  struct GNUNET_SCHEDULER_Task * *task_id;
-  GNUNET_SCHEDULER_TaskCallback keepalive_task;
-
-  GCC_check_connections ();
-  if (GNUNET_NO == GCC_is_origin (c, fwd))
-    return;
-
-  /* Calculate delay to use, depending on the state of the connection */
-  if (CADET_CONNECTION_READY == c->state)
-  {
-    delay = refresh_connection_time;
-  }
-  else
-  {
-    if (1 > c->create_retry)
-      c->create_retry = 1;
-    delay = GNUNET_TIME_relative_saturating_multiply (create_connection_time,
-                                                      c->create_retry);
-    if (c->create_retry < 64) // TODO make configurable
-      c->create_retry *= 2;
-  }
-
-  /* Select direction-dependent parameters */
-  if (GNUNET_YES == fwd)
-  {
-    task_id = &c->fwd_maintenance_task;
-    keepalive_task = &connection_fwd_keepalive;
-  }
-  else
-  {
-    task_id = &c->bck_maintenance_task;
-    keepalive_task = &connection_bck_keepalive;
-  }
+  struct CadetConnection *cc = cls;
 
-  /* Check that no one scheduled it before us */
-  if (NULL != *task_id)
-  {
-    /* No need for a _break. It can happen for instance when sending a SYNACK
-     * for a duplicate SYN: the first SYNACK scheduled the task. */
-    GNUNET_SCHEDULER_cancel (*task_id);
-  }
-
-  /* Schedule the task */
-  *task_id = GNUNET_SCHEDULER_add_delayed (delay,
-                                           keepalive_task,
-                                           c);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "next keepalive for %s in in %s\n",
-       GCC_2s (c), GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
-  GCC_check_connections ();
+  cc->keepalive_qe = NULL;
+  if ( (GNUNET_YES == cc->mqm_ready) &&
+       (NULL == cc->task) )
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
 }
 
 
 /**
- * Cancel all transmissions that belong to a certain connection.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
  *
- * If the connection is scheduled for destruction and no more messages are left,
- * the connection will be destroyed by the continuation call.
- *
- * @param c Connection which to cancel. Might be destroyed during this call.
- * @param fwd Cancel fwd traffic?
+ * @param cls the `struct CadetConnection` to keep alive.
  */
 static void
-connection_cancel_queues (struct CadetConnection *c,
-                          int fwd)
+send_keepalive (void *cls)
 {
-  struct CadetFlowControl *fc;
+  struct CadetConnection *cc = cls;
+  struct GNUNET_MessageHeader msg;
 
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Cancel %s queues for connection %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-  if (NULL == c)
+  cc->task = NULL;
+  if (CADET_TUNNEL_KEY_OK != GCT_get_estate (cc->ct->t))
   {
-    GNUNET_break (0);
+    /* Tunnel not yet ready, wait with keepalives... */
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
     return;
   }
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (NULL != fc->poll_task)
-  {
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cancelled POLL task for fc %p\n", fc);
-  }
-  if (NULL != fc->poll_msg)
-  {
-    GCC_cancel (fc->poll_msg);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cancelled POLL msg for fc %p\n", fc);
-  }
-
-  while (NULL != fc->q_head)
-  {
-    GCC_cancel (fc->q_head);
-  }
-  GCC_check_connections ();
-}
-
-
-/**
- * Function called if a connection has been stalled for a while,
- * possibly due to a missed ACK. Poll the neighbor about its ACK status.
- *
- * @param cls Closure (poll ctx).
- */
-static void
-send_poll (void *cls)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetFlowControl *fc = cls;
-  struct GNUNET_CADET_ConnectionHopByHopPollMessage msg;
-  struct CadetConnection *c;
-  int fwd;
-
-  fc->poll_task = NULL;
-  GCC_check_connections ();
-  c = fc->c;
-  fwd = fc == &c->fwd_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Polling connection %s %s\n",
-       GCC_2s (c),  GC_f2s (fwd));
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL);
-  msg.header.size = htons (sizeof (msg));
-  msg.cid = c->id;
-  msg.cemi = fc->last_pid_sent;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " last pid sent: %u\n", ntohl (fc->last_pid_sent.pid));
-  fc->poll_msg
-    = GCC_send_prebuilt_message (&msg.header,
-                                 UINT16_MAX,
-                                 zero,
-                                 c,
-                                 fc == &c->fwd_fc,
-                                 GNUNET_YES,
-                                 NULL,
-                                 NULL);
-  GNUNET_assert (NULL != fc->poll_msg);
-  GCC_check_connections ();
-}
-
-
-/**
- * Generic connection timeout implementation.
- *
- * Timeout function due to lack of keepalive/traffic from an endpoint.
- * Destroys connection if called.
- *
- * @param c Connection to destroy.
- * @param fwd Was the timeout from the origin? (FWD timeout)
- */
-static void
-connection_timeout (struct CadetConnection *c, int fwd)
-{
-  GCC_check_connections ();
-
+  GNUNET_assert (NULL != cc->ct);
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (NULL == cc->keepalive_qe);
   LOG (GNUNET_ERROR_TYPE_INFO,
-       "Connection %s %s timed out. Destroying.\n",
-       GCC_2s (c),
-       GC_f2s (fwd));
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  if (GCC_is_origin (c, fwd)) /* Loopback? Something is wrong! */
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* If dest, send "broken" notification. */
-  if (GCC_is_terminal (c, fwd))
-  {
-    struct CadetPeer *next_hop;
-
-    next_hop = fwd ? get_prev_hop (c) : get_next_hop (c);
-    send_broken_unknown (&c->id, &my_full_id, NULL, next_hop);
-  }
-
-  GCC_destroy (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Timeout function due to lack of keepalive/traffic from the owner.
- * Destroys connection if called.
- *
- * @param cls Closure (connection to destroy).
- */
-static void
-connection_fwd_timeout (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  c->fwd_maintenance_task = NULL;
-  GCC_check_connections ();
-  connection_timeout (c, GNUNET_YES);
-  GCC_check_connections ();
-}
-
-
-/**
- * Timeout function due to lack of keepalive/traffic from the destination.
- * Destroys connection if called.
- *
- * @param cls Closure (connection to destroy).
- */
-static void
-connection_bck_timeout (void *cls)
-{
-  struct CadetConnection *c = cls;
+       "Sending KEEPALIVE on behalf of %s via %s\n",
+       GCC_2s (cc),
+       GCT_2s (cc->ct->t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives sent",
+                            1,
+                            GNUNET_NO);
+  msg.size = htons (sizeof (msg));
+  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
 
-  c->bck_maintenance_task = NULL;
-  GCC_check_connections ();
-  connection_timeout (c, GNUNET_NO);
-  GCC_check_connections ();
+  cc->keepalive_qe
+    = GCT_send (cc->ct->t,
+                &msg,
+                &keepalive_done,
+                cc);
 }
 
 
 /**
- * Resets the connection timeout task, some other message has done the
- * task's job.
- * - For the first peer on the direction this means to send
- *   a keepalive or a path confirmation message (either create or ACK).
- * - For all other peers, this means to destroy the connection,
- *   due to lack of activity.
- * Starts the timeout if no timeout was running (connection just created).
- *
- * @param c Connection whose timeout to reset.
- * @param fwd Is this forward?
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
  *
- * TODO use heap to improve efficiency of scheduler.
+ * @param cid connection identifier where we expect an ACK
  */
-static void
-connection_reset_timeout (struct CadetConnection *c, int fwd)
+void
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connection %s reset timeout\n", GC_f2s (fwd));
-  if (GCC_is_origin (c, fwd)) /* Startpoint */
-  {
-    schedule_next_keepalive (c, fwd);
-    if (NULL != c->maintenance_q)
-    {
-      GCP_send_cancel (c->maintenance_q);
-      c->maintenance_q = NULL; /* Is set to NULL by conn_message_sent anyway */
-    }
-  }
-  else /* Relay, endpoint. */
-  {
-    struct GNUNET_TIME_Relative delay;
-    struct GNUNET_SCHEDULER_Task * *ti;
-    GNUNET_SCHEDULER_TaskCallback f;
-
-    ti = fwd ? &c->fwd_maintenance_task : &c->bck_maintenance_task;
+  struct CadetConnection *cc;
 
-    if (NULL != *ti)
-      GNUNET_SCHEDULER_cancel (*ti);
-    delay = GNUNET_TIME_relative_saturating_multiply (refresh_connection_time, 4);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  timing out in %s\n",
-         GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_NO));
-    f = fwd ? &connection_fwd_timeout : &connection_bck_timeout;
-    *ti = GNUNET_SCHEDULER_add_delayed (delay, f, c);
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  cc->metrics.num_acked_transmissions++;
 }
 
 
 /**
- * Iterator to compare each connection's path with the path of a new connection.
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cti.
  *
- * If the connection coincides, the c member of path is set to the connection
- * and the destroy flag of the connection is set.
- *
- * @param cls Closure (new path).
- * @param c Connection in the tunnel to check.
+ * @param cti connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
  */
-static void
-check_path (void *cls, struct CadetConnection *c)
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct CadetConnection *new_conn = cls;
-  struct CadetPeerPath *path = new_conn->path;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  checking %s (%p), length %u\n",
-       GCC_2s (c), c, c->path->length);
+  struct CadetConnection *cc;
 
-  if (c != new_conn
-      && GNUNET_NO == c->destroy
-      && CADET_CONNECTION_BROKEN != c->state
-      && CADET_CONNECTION_DESTROYED != c->state
-      && path_equivalent (path, c->path))
-  {
-    new_conn->destroy = GNUNET_YES; /* Do not mark_destroyed, */
-    new_conn->path->c = c;          /* this is only a flag for the Iterator. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  MATCH!\n");
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  cc->metrics.num_successes++;
 }
 
 
 /**
- * Finds out if this path is already being used by an existing connection.
- *
- * Checks the tunnel towards the destination to see if it contains
- * any connection with the same path.
+ * We observed some the given @a latency on the connection
+ * identified by @a cti.  (The same connection was taken
+ * in both directions.)
  *
- * If the existing connection is ready, it is kept.
- * Otherwise if the sender has a smaller ID that ours, we accept it (and
- * the peer will eventually reject our attempt).
- *
- * @param path Path to check.
- * @return #GNUNET_YES if the tunnel has a connection with the same path,
- *         #GNUNET_NO otherwise.
+ * @param cid connection identifier where we measured latency
+ * @param latency the observed latency
  */
-static int
-does_connection_exist (struct CadetConnection *conn)
+void
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                      struct GNUNET_TIME_Relative latency)
 {
-  struct CadetPeer *p;
-  struct CadetTunnel *t;
-  struct CadetConnection *c;
-
-  p = GCP_get_short (conn->path->peers[0], GNUNET_NO);
-  if (NULL == p)
-    return GNUNET_NO;
-  t = GCP_get_tunnel (p);
-  if (NULL == t)
-    return GNUNET_NO;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Checking for duplicates\n");
+  struct CadetConnection *cc;
+  double weight;
+  double result;
 
-  GCT_iterate_connections (t, &check_path, conn);
-
-  if (GNUNET_YES == conn->destroy)
-  {
-    c = conn->path->c;
-    conn->destroy = GNUNET_NO;
-    conn->path->c = conn;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " found duplicate of %s\n", GCC_2s (conn));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " duplicate: %s\n", GCC_2s (c));
-    GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-    if (CADET_CONNECTION_READY == c->state)
-    {
-      /* The other peer confirmed a live connection with this path,
-       * why are they trying to duplicate it? */
-      GNUNET_STATISTICS_update (stats, "# duplicate connections", 1, GNUNET_NO);
-      return GNUNET_YES;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " duplicate not ready, connection unique\n");
-    return GNUNET_NO;
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  GNUNET_STATISTICS_update (stats,
+                            "# latencies observed",
+                            1,
+                            GNUNET_NO);
+  cc->latency_datapoints++;
+  if (cc->latency_datapoints >= 7)
+    weight = 7.0;
   else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " %s has no duplicates\n", GCC_2s (conn));
-    return GNUNET_NO;
-  }
+    weight = cc->latency_datapoints;
+  /* Compute weighted average, giving at MOST weight 7 to the
+     existing values, or less if that value is based on fewer than 7
+     measurements. */
+  result = (weight * cc->metrics.aged_latency.rel_value_us) + 1.0 * latency.rel_value_us;
+  result /= (weight + 1.0);
+  cc->metrics.aged_latency.rel_value_us = (uint64_t) result;
 }
 
 
 /**
- * @brief Check if the tunnel this connection belongs to has any other
- * connection with the same path, and destroy one if so.
+ * A #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK was received for this connection, implying
+ * that the end-to-end connection is up.  Process it.
  *
- * @param cls Closure (connection to check).
- */
-static void
-check_duplicates (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  c->check_duplicates_task = NULL;
-  if (GNUNET_YES == does_connection_exist (c))
-  {
-    GCT_debug (c->t, GNUNET_ERROR_TYPE_DEBUG);
-    send_broken (c, &my_full_id, &my_full_id, GCC_is_origin (c, GNUNET_YES));
-    GCC_destroy (c);
-  }
-}
-
-
-/**
- * Wait for enough time to let any dead connections time out and check for
- * any remaining duplicates.
- *
- * @param c Connection that is a potential duplicate.
- */
-static void
-schedule_check_duplicates (struct CadetConnection *c)
-{
-  struct GNUNET_TIME_Relative delay;
-
-  if (NULL != c->check_duplicates_task)
-    return;
-  delay = GNUNET_TIME_relative_saturating_multiply (refresh_connection_time, 5);
-  c->check_duplicates_task = GNUNET_SCHEDULER_add_delayed (delay,
-                                                           &check_duplicates,
-                                                           c);
-}
-
-
-/**
- * Add the connection to the list of both neighbors.
- *
- * @param c Connection.
- *
- * @return #GNUNET_OK if everything went fine
- *         #GNUNET_SYSERR if the was an error and @c c is malformed.
- */
-static int
-register_neighbors (struct CadetConnection *c)
-{
-  c->next_peer = get_next_hop (c);
-  c->prev_peer = get_prev_hop (c);
-  GNUNET_assert (c->next_peer != c->prev_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "register neighbors for connection %s\n",
-       GCC_2s (c));
-  path_debug (c->path);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "own pos %u\n", c->own_pos);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "putting connection %s to next peer %p\n",
-       GCC_2s (c),
-       c->next_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "next peer %p %s\n",
-       c->next_peer,
-       GCP_2s (c->next_peer));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "putting connection %s to prev peer %p\n",
-       GCC_2s (c),
-       c->prev_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "prev peer %p %s\n",
-       c->prev_peer,
-       GCP_2s (c->prev_peer));
-
-  if ( (GNUNET_NO == GCP_is_neighbor (c->next_peer)) ||
-       (GNUNET_NO == GCP_is_neighbor (c->prev_peer)) )
-  {
-    if (GCC_is_origin (c, GNUNET_YES))
-      GNUNET_STATISTICS_update (stats, "# local bad paths", 1, GNUNET_NO);
-    GNUNET_STATISTICS_update (stats, "# bad paths", 1, GNUNET_NO);
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  register neighbors failed\n");
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  prev: %s, neighbor?: %d\n",
-         GCP_2s (c->prev_peer),
-         GCP_is_neighbor (c->prev_peer));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  next: %s, neighbor?: %d\n",
-         GCP_2s (c->next_peer),
-         GCP_is_neighbor (c->next_peer));
-    return GNUNET_SYSERR;
-  }
-  GCP_add_connection (c->next_peer, c, GNUNET_NO);
-  GCP_add_connection (c->prev_peer, c, GNUNET_YES);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Remove the connection from the list of both neighbors.
- *
- * @param c Connection.
- */
-static void
-unregister_neighbors (struct CadetConnection *c)
-{
-//  struct CadetPeer *peer; FIXME dont use next_peer, prev_peer
-  /* Either already unregistered or never got registered, it's ok either way. */
-  if (NULL == c->path)
-    return;
-  if (NULL != c->next_peer)
-  {
-    GCP_remove_connection (c->next_peer, c);
-    c->next_peer = NULL;
-  }
-  if (NULL != c->prev_peer)
-  {
-    GCP_remove_connection (c->prev_peer, c);
-    c->prev_peer = NULL;
-  }
-}
-
-
-/**
- * Invalidates all paths towards all peers that comprise the connection which
- * rely on the disconnected peer.
- *
- * ~O(n^3) (peers in connection * paths/peer * links/path)
- *
- * @param c Connection whose peers' paths to clean.
- * @param disconnected Peer that disconnected.
- */
-static void
-invalidate_paths (struct CadetConnection *c,
-                  struct CadetPeer *disconnected)
-{
-  struct CadetPeer *peer;
-  unsigned int i;
-
-  for (i = 0; i < c->path->length; i++)
-  {
-    peer = GCP_get_short (c->path->peers[i], GNUNET_NO);
-    if (NULL != peer)
-      GCP_notify_broken_link (peer, &my_full_id, GCP_get_id (disconnected));
-  }
-}
-
-
-/**
- * Bind the connection to the peer and the tunnel to that peer.
- *
- * If the peer has no tunnel, create one. Update tunnel and connection
- * data structres to reflect new status.
- *
- * @param c Connection.
- * @param peer Peer.
- */
-static void
-add_to_peer (struct CadetConnection *c,
-             struct CadetPeer *peer)
-{
-  GCP_add_tunnel (peer);
-  c->t = GCP_get_tunnel (peer);
-  GCT_add_connection (c->t, c);
-}
-
-
-/**
- * Log receipt of message on stderr (INFO level).
- *
- * @param message Message received.
- * @param peer    Peer who sent the message.
- * @param conn_id Connection ID of the message.
- */
-static void
-log_message (const struct GNUNET_MessageHeader *message,
-             const struct CadetPeer *peer,
-             const struct GNUNET_CADET_ConnectionTunnelIdentifier *conn_id)
-{
-  uint16_t size;
-  uint16_t type;
-  char *arrow;
-
-  size = ntohs (message->size);
-  type = ntohs (message->type);
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      arrow = "==";
-      break;
-    default:
-      arrow = "--";
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<%s %s on conn %s from %s, %6u bytes\n",
-       arrow,
-       GC_m2s (type),
-       GNUNET_sh2s (&conn_id->connection_of_tunnel),
-       GCP_2s(peer),
-       (unsigned int) size);
-}
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Handler for connection creation.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc the connection that got the ACK.
  */
 void
-GCC_handle_create (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionCreateMessage *msg)
+GCC_handle_connection_create_ack (struct CadetConnection *cc)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid;
-  struct GNUNET_PeerIdentity *id;
-  struct CadetPeerPath *path;
-  struct CadetPeer *dest_peer;
-  struct CadetPeer *orig_peer;
-  struct CadetConnection *c;
-  unsigned int own_pos;
-  uint16_t size;
-
-  GCC_check_connections ();
-  size = ntohs (msg->header.size);
-
-  /* Calculate hops */
-  size -= sizeof (struct GNUNET_CADET_ConnectionCreateMessage);
-  if (0 != size % sizeof (struct GNUNET_PeerIdentity))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  size /= sizeof (struct GNUNET_PeerIdentity);
-  if (1 > size)
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    path has %u hops.\n", size);
-
-  /* Get parameters */
-  cid = &msg->cid;
-  log_message (&msg->header, peer, cid);
-  id = (struct GNUNET_PeerIdentity *) &msg[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    origin: %s\n", GNUNET_i2s (id));
-
-  /* Create connection */
-  c = connection_get (cid);
-  if (NULL == c)
-  {
-    path = path_build_from_peer_ids ((struct GNUNET_PeerIdentity *) &msg[1],
-                                     size, myid, &own_pos);
-    if (NULL == path)
-    {
-      /* Path was malformed, probably our own ID was not in it. */
-      GNUNET_STATISTICS_update (stats, "# malformed paths", 1, GNUNET_NO);
-      GNUNET_break_op (0);
-      return;
-    }
-    if (0 == own_pos)
-    {
-      /* We received this request from a neighbor, we cannot be origin */
-      GNUNET_STATISTICS_update (stats, "# fake paths", 1, GNUNET_NO);
-      GNUNET_break_op (0);
-      path_destroy (path);
-      return;
-    }
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Own position: %u\n", own_pos);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Creating connection\n");
-    c = GCC_new (cid, NULL, path, own_pos);
-    if (NULL == c)
-    {
-      if (path->length - 1 == own_pos)
-      {
-        /* If we are destination, why did the creation fail? */
-        GNUNET_break (0);
-        path_destroy (path);
-        GCC_check_connections ();
-        return;
-      }
-      send_broken_unknown (cid, &my_full_id,
-                           GNUNET_PEER_resolve2 (path->peers[own_pos + 1]),
-                           peer);
-      path_destroy (path);
-      GCC_check_connections ();
-      return;
-    }
-    GCP_add_path_to_all (path, GNUNET_NO);
-    connection_reset_timeout (c, GNUNET_YES);
-  }
-  else
-  {
-    path = path_duplicate (c->path);
-  }
-  if (CADET_CONNECTION_NEW == c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-
-  /* Remember peers */
-  dest_peer = GCP_get (&id[size - 1], GNUNET_YES);
-  orig_peer = GCP_get (&id[0], GNUNET_YES);
-
-  /* Is it a connection to us? */
-  if (c->own_pos == path->length - 1)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  It's for us!\n");
-    GCP_add_path_to_origin (orig_peer, path_duplicate (path), GNUNET_YES);
-
-    add_to_peer (c, orig_peer);
-    if (GNUNET_YES == does_connection_exist (c))
-    {
-      /* Peer created a connection equal to one we think exists
-       * and is fine.
-       * Solution: Keep both and postpone disambiguation. In the meantime
-       * the connection will time out or peer will inform us it is broken.
-       *
-       * Other options:
-       * - Use explicit duplicate.
-       * - Accept new conn and destroy the old. (interruption in higher level)
-       * - Keep the one with higher ID / created by peer with higher ID. */
-       schedule_check_duplicates (c);
-    }
-
-    if (CADET_TUNNEL_NEW == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t,  CADET_TUNNEL_WAITING);
-    if (NULL == c->maintenance_q)
-      send_connection_ack (c, GNUNET_NO);
-    if (CADET_CONNECTION_SENT == c->state)
-      connection_change_state (c, CADET_CONNECTION_ACK);
-  }
-  else
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CADET_CONNECTION_CREATE_ACK for %s in state %d (%s)\n",
+       GCC_2s (cc),
+       cc->state,
+       (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+  if (CADET_CONNECTION_READY == cc->state)
+    return; /* Duplicate ACK, ignore */
+  if (NULL != cc->task)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-    GCP_add_path (dest_peer, path_duplicate (path), GNUNET_NO);
-    GCP_add_path_to_origin (orig_peer, path_duplicate (path), GNUNET_NO);
-    (void) GCC_send_prebuilt_message (&msg->header,
-                                      0,
-                                      zero,
-                                      c,
-                                      GNUNET_YES, GNUNET_YES,
-                                      NULL, NULL);
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
   }
-  path_destroy (path);
-  GCC_check_connections ();
+  cc->metrics.age = GNUNET_TIME_absolute_get ();
+  update_state (cc,
+                CADET_CONNECTION_READY,
+                cc->mqm_ready);
+  if ( (NULL == cc->keepalive_qe) &&
+       (GNUNET_YES == cc->mqm_ready) &&
+       (NULL == cc->task) )
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
 }
 
 
 /**
- * Handler for connection confirmations.
+ * Handle KX message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_handle_confirm (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionCreateAckMessage *msg)
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  enum CadetConnectionState oldstate;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats, "# control on unknown connection",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  don't know the connection!\n");
-    send_broken_unknown (&msg->cid, &my_full_id, NULL, peer);
-    GCC_check_connections ();
-    return;
-  }
-  if (GNUNET_NO != c->destroy)
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    GNUNET_assert (CADET_CONNECTION_DESTROYED == c->state);
-    GNUNET_STATISTICS_update (stats, "# control on dying connection",
-                              1, GNUNET_NO);
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "connection %s being destroyed, ignoring confirm\n",
-         GCC_2s (c));
-    GCC_check_connections ();
-    return;
-  }
-
-  oldstate = c->state;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  via peer %s\n", GCP_2s (peer));
-  if (get_next_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  SYNACK\n");
-    fwd = GNUNET_NO;
-    if (CADET_CONNECTION_SENT == oldstate)
-      connection_change_state (c, CADET_CONNECTION_ACK);
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  FINAL ACK\n");
-    fwd = GNUNET_YES;
-    connection_change_state (c, CADET_CONNECTION_READY);
-  }
-  else
-  {
-    GNUNET_STATISTICS_update (stats, "# control on connection from wrong peer",
-                              1, GNUNET_NO);
-    GNUNET_break_op (0);
-    return;
-  }
-
-  connection_reset_timeout (c, fwd);
-
-  GNUNET_assert (NULL != c->path);
-  GCP_add_path_to_all (c->path, GNUNET_YES);
-
-  /* Message for us as creator? */
-  if (GNUNET_YES == GCC_is_origin (c, GNUNET_YES))
-  {
-    if (GNUNET_NO != fwd)
-    {
-      GNUNET_break (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Connection (SYN)ACK for us!\n");
-
-    /* If just created, cancel the short timeout and start a long one */
-    if (CADET_CONNECTION_SENT == oldstate)
-    {
-      c->create_retry = 1;
-      connection_reset_timeout (c, GNUNET_YES);
-    }
-
-    /* Change connection state, send ACK */
-    connection_change_state (c, CADET_CONNECTION_READY);
-    send_connection_ack (c, GNUNET_YES);
-
-    /* Change tunnel state, trigger KX */
-    if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Message for us as destination? */
-  if (GCC_is_terminal (c, GNUNET_YES))
-  {
-    if (GNUNET_YES != fwd)
-    {
-      GNUNET_break (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Connection ACK for us!\n");
-
-    /* If just created, cancel the short timeout and start a long one */
-    if (CADET_CONNECTION_ACK == oldstate)
-      connection_reset_timeout (c, GNUNET_NO);
-
-    /* Change tunnel state */
-    if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    GCC_check_connections ();
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero,
-                                      c,
-                                      fwd,
-                                      GNUNET_YES, NULL, NULL);
-  }
-  GCC_check_connections ();
+  GCT_handle_kx (cc->ct,
+                 msg);
 }
 
 
 /**
- * Handler for notifications of broken connections.
+ * Handle KX_AUTH message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_handle_broken (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionBrokenMessage *msg)
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  struct CadetTunnel *t;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  regarding %s\n", GNUNET_i2s (&msg->peer1));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  regarding %s\n", GNUNET_i2s (&msg->peer2));
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  duplicate CONNECTION_BROKEN\n");
-    GNUNET_STATISTICS_update (stats, "# duplicate CONNECTION_BROKEN",
-                              1, GNUNET_NO);
-    GCC_check_connections ();
-    return;
-  }
-
-  t = c->t;
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
-  mark_destroyed (c);
-  if (GCC_is_terminal (c, fwd))
-  {
-    struct CadetPeer *endpoint;
-
-    if (NULL == t)
-    {
-      /* A terminal connection should not have 't' set to NULL. */
-      GNUNET_break (0);
-      GCC_debug (c, GNUNET_ERROR_TYPE_ERROR);
-      return;
-    }
-    endpoint = GCP_get_short (c->path->peers[c->path->length - 1], GNUNET_YES);
-    if (2 < c->path->length)
-      path_invalidate (c->path);
-    GCP_notify_broken_link (endpoint, &msg->peer1, &msg->peer2);
-
-    connection_change_state (c, CADET_CONNECTION_BROKEN);
-    GCT_remove_connection (t, c);
-    c->t = NULL;
-
-    GCC_destroy (c);
-  }
-  else
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero, c, fwd,
-                                      GNUNET_YES, NULL, NULL);
-    connection_cancel_queues (c, !fwd);
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  GCC_check_connections ();
-  return;
+  GCT_handle_kx_auth (cc->ct,
+                      msg);
 }
 
 
 /**
- * Handler for notifications of destroyed connections.
+ * Handle encrypted message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
  */
 void
-GCC_handle_destroy (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionDestroyMessage *msg)
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    /* Probably already got the message from another path,
-     * destroyed the tunnel and retransmitted to children.
-     * Safe to ignore.
-     */
-    GNUNET_STATISTICS_update (stats,
-                              "# control on unknown connection",
-                              1, GNUNET_NO);
+    /* We didn't get the CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  connection unknown destroyed: previously destroyed?\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
-
-  if (GNUNET_NO == GCC_is_terminal (c, fwd))
-  {
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero, c, fwd,
-                                      GNUNET_YES, NULL, NULL);
+         "Faking connection ACK for %s due to ENCRYPTED payload\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  else if (0 == c->pending_messages)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  directly destroying connection!\n");
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
-  }
-  mark_destroyed (c);
-  if (NULL != c->t)
-  {
-    GCT_remove_connection (c->t, c);
-    c->t = NULL;
-  }
-  GCC_check_connections ();
-  return;
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  GCT_handle_encrypted (cc->ct,
+                        msg);
 }
 
 
 /**
- * Handler for cadet network traffic hop-by-hop acks.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE message to the
+ * first hop.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection` to initiate
  */
-void
-GCC_handle_ack (struct CadetPeer *peer,
-                const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg)
-{
-  struct CadetConnection *c;
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier ack;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# ack on unknown connection",
-                              1,
-                              GNUNET_NO);
-    send_broken_unknown (&msg->cid,
-                         &my_full_id,
-                         NULL,
-                         peer);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Is this a forward or backward ACK? */
-  if (get_next_hop (c) == peer)
-  {
-    fc = &c->fwd_fc;
-    fwd = GNUNET_YES;
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    fc = &c->bck_fc;
-    fwd = GNUNET_NO;
-  }
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  ack = msg->cemi_max;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " %s ACK %u (was %u)\n",
-       GC_f2s (fwd),
-       ntohl (ack.pid),
-       ntohl (fc->last_ack_recv.pid));
-  if (GC_is_pid_bigger (ntohl (ack.pid),
-                        ntohl (fc->last_ack_recv.pid)))
-    fc->last_ack_recv = ack;
-
-  /* Cancel polling if the ACK is big enough. */
-  if ( (NULL != fc->poll_task) &
-       GC_is_pid_bigger (ntohl (fc->last_ack_recv.pid),
-                         ntohl (fc->last_pid_sent.pid)))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Cancel poll\n");
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-    fc->poll_time = GNUNET_TIME_UNIT_SECONDS;
-  }
-
-  GCC_check_connections ();
+static void
+send_create (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateMessage *create_msg;
+  struct GNUNET_PeerIdentity *pids;
+  struct GNUNET_MQ_Envelope *env;
+  unsigned int path_length;
+
+  cc->task = NULL;
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  path_length = GCPP_get_length (cc->path);
+  env = GNUNET_MQ_msg_extra (create_msg,
+                             (1 + path_length) * sizeof (struct GNUNET_PeerIdentity),
+                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
+  create_msg->options = htonl ((uint32_t) cc->options);
+  create_msg->cid = cc->cid;
+  pids = (struct GNUNET_PeerIdentity *) &create_msg[1];
+  pids[0] = my_full_id;
+  for (unsigned int i=0;i<path_length;i++)
+    pids[i + 1] = *GCP_get_id (GCPP_get_peer_at_offset (cc->path,
+                                                        i));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_CONNECTION_CREATE message for %s\n",
+       GCC_2s (cc));
+  cc->env = env;
+  update_state (cc,
+                CADET_CONNECTION_SENT,
+                GNUNET_NO);
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * Handler for cadet network traffic hop-by-hop data counter polls.
+ * Send a CREATE_ACK message towards the origin.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection` to initiate
  */
-void
-GCC_handle_poll (struct CadetPeer *peer,
-                 const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg)
+static void
+send_create_ack (void *cls)
 {
-  struct CadetConnection *c;
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier pid;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats, "# poll on unknown connection", 1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "POLL message on unknown connection %s!\n",
-         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
-    send_broken_unknown (&msg->cid,
-                         &my_full_id,
-                         NULL,
-                         peer);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Is this a forward or backward ACK?
-   * Note: a poll should never be needed in a loopback case,
-   * since there is no possiblility of packet loss there, so
-   * this way of discerining FWD/BCK should not be a problem.
-   */
-  if (get_next_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  FWD FC\n");
-    fc = &c->fwd_fc;
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  BCK FC\n");
-    fc = &c->bck_fc;
-  }
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateAckMessage *ack_msg;
+  struct GNUNET_MQ_Envelope *env;
 
-  pid = msg->cemi;
+  cc->task = NULL;
+  GNUNET_assert (CADET_CONNECTION_CREATE_RECEIVED == cc->state);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  PID %u, OLD %u\n",
-       ntohl (pid.pid),
-       ntohl (fc->last_pid_recv.pid));
-  fc->last_pid_recv = pid;
-  fwd = fc == &c->bck_fc;
-  GCC_send_ack (c, fwd, GNUNET_YES);
-  GCC_check_connections ();
+       "Sending CONNECTION_CREATE_ACK message for %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  env = GNUNET_MQ_msg (ack_msg,
+                       GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK);
+  ack_msg->cid = cc->cid;
+  cc->env = env;
+  update_state (cc,
+                CADET_CONNECTION_READY,
+                GNUNET_NO);
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * Check the message against internal state and test if it goes FWD or BCK.
- *
- * Updates the PID, state and timeout values for the connection.
- *
- * @param message Message to check. It must belong to an existing connection.
- * @param cid Connection ID (even if @a c is NULL, the ID is still needed).
- * @param c Connection this message should belong. If NULL, check fails.
- * @param sender Neighbor that sent the message.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
  *
- * @return #GNUNET_YES if the message goes FWD.
- *         #GNUNET_NO if it goes BCK.
- *         #GNUNET_SYSERR if there is an error (unauthorized sender, ...).
+ * @param cc connection that got the duplicate CREATE
  */
-static int
-check_message (const struct GNUNET_MessageHeader *message,
-               const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid,
-               struct CadetConnection *c,
-               struct CadetPeer *sender,
-               struct CadetEncryptedMessageIdentifier pid)
+void
+GCC_handle_duplicate_create (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-  struct CadetPeer *hop;
-  int fwd;
-  uint16_t type;
-
-  /* Check connection */
-  if (NULL == c)
+  if (GNUNET_YES == cc->mqm_ready)
   {
-    GNUNET_STATISTICS_update (stats,
-                              "# unknown connection",
-                              1, GNUNET_NO);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "%s on unknown connection %s\n",
-         GC_m2s (ntohs (message->type)),
-         GNUNET_sh2s (&cid->connection_of_tunnel));
-    GNUNET_break_op (0);
-    send_broken_unknown (cid,
-                         &my_full_id,
-                         NULL,
-                         sender);
-    return GNUNET_SYSERR;
-  }
-
-  /* Check if origin is as expected */
-  hop = get_prev_hop (c);
-  if (sender == hop)
-  {
-    fwd = GNUNET_YES;
+         "Got duplicate CREATE for %s, scheduling another ACK (%s)\n",
+         GCC_2s (cc),
+         (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+    /* Revert back to the state of having only received the 'CREATE',
+       and immediately proceed to send the CREATE_ACK. */
+    update_state (cc,
+                  CADET_CONNECTION_CREATE_RECEIVED,
+                  cc->mqm_ready);
+    if (NULL != cc->task)
+      GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
+                                         cc);
   }
   else
   {
-    hop = get_next_hop (c);
-    GNUNET_break (hop == c->next_peer);
-    if (sender == hop)
-    {
-      fwd = GNUNET_NO;
-    }
-    else
-    {
-      /* Unexpected peer sending traffic on a connection. */
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-  }
-
-  /* Check PID for payload messages */
-  type = ntohs (message->type);
-  if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == type)
-  {
-    fc = fwd ? &c->bck_fc : &c->fwd_fc;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " PID %u (expected in interval [%u,%u])\n",
-         ntohl (pid.pid),
-         ntohl (fc->last_pid_recv.pid) + 1,
-         ntohl (fc->last_ack_sent.pid));
-    if (GC_is_pid_bigger (ntohl (pid.pid),
-                          ntohl (fc->last_ack_sent.pid)))
-    {
-      GNUNET_STATISTICS_update (stats,
-                                "# unsolicited message",
-                                1,
-                                GNUNET_NO);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "Received PID %u, (prev %u), ACK %u\n",
-           pid, fc->last_pid_recv, fc->last_ack_sent);
-      return GNUNET_SYSERR;
-    }
-    if (GC_is_pid_bigger (ntohl (pid.pid),
-                          ntohl (fc->last_pid_recv.pid)))
-    {
-      unsigned int delta;
-
-      delta = ntohl (pid.pid) - ntohl (fc->last_pid_recv.pid);
-      fc->last_pid_recv = pid;
-      fc->recv_bitmap <<= delta;
-      fc->recv_bitmap |= 1;
-    }
-    else
-    {
-      GNUNET_STATISTICS_update (stats,
-                                "# out of order PID",
-                                1,
-                                GNUNET_NO);
-      if (GNUNET_NO == is_ooo_ok (fc->last_pid_recv,
-                                  pid,
-                                  fc->recv_bitmap))
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "PID %u unexpected (%u+), dropping!\n",
-             ntohl (pid.pid),
-             ntohl (fc->last_pid_recv.pid) - 31);
-        return GNUNET_SYSERR;
-      }
-      fc->recv_bitmap |= get_recv_bitmask (fc->last_pid_recv,
-                                           pid);
-    }
-  }
-
-  /* Count as connection confirmation. */
-  if ( (CADET_CONNECTION_SENT == c->state) ||
-       (CADET_CONNECTION_ACK == c->state) )
-  {
-    connection_change_state (c, CADET_CONNECTION_READY);
-    if (NULL != c->t)
-    {
-      if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-        GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    }
+    /* We are currently sending something else back, which
+       can only be an ACK or payload, either of which would
+       do. So actually no need to do anything. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate CREATE for %s. MQ is busy, not queueing another ACK\n",
+         GCC_2s (cc));
   }
-  connection_reset_timeout (c, fwd);
-
-  return fwd;
 }
 
 
 /**
- * Handler for key exchange traffic (Axolotl KX).
+ * There has been a change in the message queue existence for our
+ * peer at the first hop.  Adjust accordingly.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection`
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
  */
-void
-GCC_handle_kx (struct CadetPeer *peer,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+static void
+manage_first_hop_mq (void *cls,
+                     int available)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid;
-  struct CadetConnection *c;
-  int fwd;
-
-  GCC_check_connections ();
-  cid = &msg->cid;
-  log_message (&msg->header, peer, cid);
-
-  c = connection_get (cid);
-  fwd = check_message (&msg->header,
-                       cid,
-                       c,
-                       peer,
-                       zero);
-
-  /* If something went wrong, discard message. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
+  struct CadetConnection *cc = cls;
 
-  /* Is this message for us? */
-  if (GCC_is_terminal (c, fwd))
+  if (GNUNET_YES != available)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  message for us!\n");
-    GNUNET_STATISTICS_update (stats, "# received KX", 1, GNUNET_NO);
-    if (NULL == c->t)
+    /* Connection is down, for now... */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Core MQ for %s went down\n",
+         GCC_2s (cc));
+    update_state (cc,
+                  CADET_CONNECTION_NEW,
+                  GNUNET_NO);
+    cc->retry_delay = GNUNET_TIME_UNIT_ZERO;
+    if (NULL != cc->task)
     {
-      GNUNET_break (0);
-      return;
+      GNUNET_SCHEDULER_cancel (cc->task);
+      cc->task = NULL;
     }
-    GCT_handle_kx (c->t, msg);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Message not for us: forward to next hop */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-  GNUNET_STATISTICS_update (stats, "# messages forwarded", 1, GNUNET_NO);
-  (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                    zero, c, fwd,
-                                    GNUNET_NO, NULL, NULL);
-  GCC_check_connections ();
-}
-
-
-/**
- * Handler for encrypted cadet network traffic (channel mgmt, data).
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_encrypted (struct CadetPeer *peer,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid;
-  struct CadetConnection *c;
-  struct CadetEncryptedMessageIdentifier pid;
-  int fwd;
-
-  GCC_check_connections ();
-  cid = &msg->cid;
-  pid = msg->cemi;
-  log_message (&msg->header, peer, cid);
-
-  c = connection_get (cid);
-  fwd = check_message (&msg->header,
-                       cid,
-                       c,
-                       peer,
-                       pid);
-
-  /* If something went wrong, discard message. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    GCC_check_connections ();
     return;
   }
 
-  /* Is this message for us? */
-  if (GCC_is_terminal (c, fwd))
-  {
-    GNUNET_STATISTICS_update (stats, "# received encrypted", 1, GNUNET_NO);
-
-    if (NULL == c->t)
+  update_state (cc,
+                cc->state,
+                GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Core MQ for %s became available in state %d\n",
+       GCC_2s (cc),
+       cc->state);
+  switch (cc->state)
+  {
+  case CADET_CONNECTION_NEW:
+    /* Transmit immediately */
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create,
+                                         cc);
+    break;
+  case CADET_CONNECTION_SENDING_CREATE:
+    /* Should not be possible to be called in this state. */
+    GNUNET_assert (0);
+    break;
+  case CADET_CONNECTION_SENT:
+    /* Retry a bit later... */
+    cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
+    cc->task = GNUNET_SCHEDULER_add_delayed (cc->retry_delay,
+                                             &send_create,
+                                             cc);
+    break;
+  case CADET_CONNECTION_CREATE_RECEIVED:
+    /* We got the 'CREATE' (incoming connection), should send the CREATE_ACK */
+    cc->metrics.age = GNUNET_TIME_absolute_get ();
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
+                                         cc);
+    break;
+  case CADET_CONNECTION_READY:
+    if ( (NULL == cc->keepalive_qe) &&
+         (GNUNET_YES == cc->mqm_ready) &&
+         (NULL == cc->task) )
     {
-      GNUNET_break (GNUNET_NO != c->destroy);
-      return;
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Scheduling keepalive for %s in %s\n",
+           GCC_2s (cc),
+           GNUNET_STRINGS_relative_time_to_string (keepalive_period,
+                                                   GNUNET_YES));
+      cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                               &send_keepalive,
+                                               cc);
     }
-    GCT_handle_encrypted (c->t, msg);
-    GCC_send_ack (c, fwd, GNUNET_NO);
-    GCC_check_connections ();
-    return;
+    break;
   }
-
-  /* Message not for us: forward to next hop */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-  GNUNET_STATISTICS_update (stats, "# messages forwarded", 1, GNUNET_NO);
-  (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                    zero, c, fwd,
-                                    GNUNET_NO, NULL, NULL);
-  GCC_check_connections ();
 }
 
 
 /**
- * Initialize the connections subsystem
+ * Create a connection to @a destination via @a path and notify @a cb
+ * whenever we are ready for more data.  Shared logic independent of
+ * who is initiating the connection.
  *
- * @param c Configuration handle.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param init_state initial state for the connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_MSGS_QUEUE",
-                                             &max_msgs_queue))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "MAX_MSGS_QUEUE", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_CONNECTIONS",
-                                             &max_connections))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "MAX_CONNECTIONS", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c, "CADET", "REFRESH_CONNECTION_TIME",
-                                           &refresh_connection_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "REFRESH_CONNECTION_TIME", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  create_connection_time = GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_SECONDS,
-                                                     refresh_connection_time);
-  connections = GNUNET_CONTAINER_multishortmap_create (1024,
-                                                       GNUNET_YES);
-}
-
-
-/**
- * Destroy each connection on shutdown.
- *
- * @param cls Closure (unused).
- * @param key Current key code (CID, unused).
- * @param value Value in the hash map (`struct CadetConnection`)
- *
- * @return #GNUNET_YES, because we should continue to iterate
- */
-static int
-shutdown_iterator (void *cls,
-                   const struct GNUNET_ShortHashCode *key,
-                   void *value)
-{
-  struct CadetConnection *c = value;
-
-  c->state = CADET_CONNECTION_DESTROYED;
-  GCC_destroy (c);
-  return GNUNET_YES;
-}
-
-
-/**
- * Shut down the connections subsystem.
- */
-void
-GCC_shutdown (void)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down connections\n");
-  GCC_check_connections ();
-  GNUNET_CONTAINER_multishortmap_iterate (connections,
-                                          &shutdown_iterator,
-                                          NULL);
-  GNUNET_CONTAINER_multishortmap_destroy (connections);
-  connections = NULL;
-}
-
-
-/**
- * Create a connection.
- *
- * @param cid Connection ID (either created locally or imposed remotely).
- * @param t Tunnel this connection belongs to (or NULL for transit connections);
- * @param path Path this connection has to use (copy is made).
- * @param own_pos Own position in the @c path path.
- *
- * @return Newly created connection.
- *         NULL in case of error: own id not in path, wrong neighbors, ...
-*/
-struct CadetConnection *
-GCC_new (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-         struct CadetTunnel *t,
-         struct CadetPeerPath *path,
-         unsigned int own_pos)
-{
-  struct CadetConnection *c;
-  struct CadetPeerPath *cpath;
-
-  GCC_check_connections ();
-  cpath = path_duplicate (path);
-  GNUNET_assert (NULL != cpath);
-  c = GNUNET_new (struct CadetConnection);
-  c->id = *cid;
+static struct CadetConnection *
+connection_create (struct CadetPeer *destination,
+                   struct CadetPeerPath *path,
+                   unsigned int off,
+                   enum GNUNET_CADET_ChannelOption options,
+                   struct CadetTConnection *ct,
+                   const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                   enum CadetConnectionState init_state,
+                   GCC_ReadyCallback ready_cb,
+                   void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  struct CadetPeer *first_hop;
+
+  cc = GNUNET_new (struct CadetConnection);
+  cc->options = options;
+  cc->state = init_state;
+  cc->ct = ct;
+  cc->cid = *cid;
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multishortmap_put (connections,
-                                                     &c->id.connection_of_tunnel,
-                                                     c,
+                                                     &GCC_get_id (cc)->connection_of_tunnel,
+                                                     cc,
                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  fc_init (&c->fwd_fc);
-  fc_init (&c->bck_fc);
-  c->fwd_fc.c = c;
-  c->bck_fc.c = c;
-
-  c->t = t;
-  GNUNET_assert (own_pos <= cpath->length - 1);
-  c->own_pos = own_pos;
-  c->path = cpath;
-  cpath->c = c;
-  if (GNUNET_OK != register_neighbors (c))
-  {
-    if (0 == own_pos)
-    {
-      /* We were the origin of this request, this means we have invalid
-       * info about the paths to reach the destination. We must invalidate
-       * the *original* path to avoid trying it again in the next minute.
-       */
-      if (2 < path->length)
-        path_invalidate (path);
-      else
-      {
-        GNUNET_break (0);
-        GCT_debug(t, GNUNET_ERROR_TYPE_WARNING);
-      }
-      c->t = NULL;
-    }
-    path_destroy (c->path);
-    c->path = NULL;
-    GCC_destroy (c);
-    return NULL;
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO, "New connection %s\n", GCC_2s (c));
-  GCC_check_connections ();
-  return c;
-}
-
-
-/**
- * Connection is no longer needed: destroy it.
- *
- * Cancels all pending traffic (including possible DESTROY messages), all
- * maintenance tasks and removes the connection from neighbor peers and tunnel.
- *
- * @param c Connection to destroy.
- */
-void
-GCC_destroy (struct CadetConnection *c)
-{
-  GCC_check_connections ();
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (2 == c->destroy) /* cancel queues -> GCP_queue_cancel -> q_destroy -> */
-    return;            /* -> message_sent -> GCC_destroy. Don't loop. */
-  c->destroy = 2;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying connection %s\n",
-       GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " fc's f: %p, b: %p\n",
-       &c->fwd_fc, &c->bck_fc);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " fc tasks f: %u, b: %u\n",
-       c->fwd_fc.poll_task,
-       c->bck_fc.poll_task);
-
-  /* Cancel all traffic */
-  if (NULL != c->path)
-  {
-    connection_cancel_queues (c, GNUNET_YES);
-    connection_cancel_queues (c, GNUNET_NO);
-    if (NULL != c->maintenance_q)
-    {
-      GCP_send_cancel (c->maintenance_q);
-      c->maintenance_q = NULL;
-    }
-  }
-  unregister_neighbors (c);
-  path_destroy (c->path);
-  c->path = NULL;
-
-  /* Delete from tunnel */
-  if (NULL != c->t)
-    GCT_remove_connection (c->t, c);
-
-  if (NULL != c->check_duplicates_task)
-    GNUNET_SCHEDULER_cancel (c->check_duplicates_task);
-  if (NULL != c->fwd_maintenance_task)
-    GNUNET_SCHEDULER_cancel (c->fwd_maintenance_task);
-  if (NULL != c->bck_maintenance_task)
-    GNUNET_SCHEDULER_cancel (c->bck_maintenance_task);
-
-  if (GNUNET_NO == c->was_removed)
-  {
-    GNUNET_break (GNUNET_YES ==
-                  GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                         &c->id.connection_of_tunnel,
-                                                         c));
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# connections",
-                            -1,
-                            GNUNET_NO);
-  GNUNET_free (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Get the connection ID.
- *
- * @param c Connection to get the ID from.
- *
- * @return ID of the connection.
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (const struct CadetConnection *c)
-{
-  return &c->id;
-}
-
-
-/**
- * Get the connection path.
- *
- * @param c Connection to get the path from.
- *
- * @return path used by the connection.
- */
-const struct CadetPeerPath *
-GCC_get_path (const struct CadetConnection *c)
-{
-  if (GNUNET_NO == c->destroy)
-    return c->path;
-  return NULL;
-}
-
-
-/**
- * Get the connection state.
- *
- * @param c Connection to get the state from.
- *
- * @return state of the connection.
- */
-enum CadetConnectionState
-GCC_get_state (const struct CadetConnection *c)
-{
-  return c->state;
-}
-
-/**
- * Get the connection tunnel.
- *
- * @param c Connection to get the tunnel from.
- *
- * @return tunnel of the connection.
- */
-struct CadetTunnel *
-GCC_get_tunnel (const struct CadetConnection *c)
-{
-  return c->t;
-}
-
-
-/**
- * Get free buffer space in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - max_msgs_queue/max_connections]
- */
-unsigned int
-GCC_get_buffer (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Get %s buffer on %s: %u - %u\n",
-       GC_f2s (fwd), GCC_2s (c), fc->queue_max, fc->queue_n);
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  return (fc->queue_max - fc->queue_n);
-}
-
-
-/**
- * Get how many messages have we allowed to send to us from a direction.
- *
- * @param c Connection.
- * @param fwd Are we asking about traffic from FWD (BCK messages)?
- *
- * @return last_ack_sent - last_pid_recv
- */
-unsigned int
-GCC_get_allowed (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if ( (CADET_CONNECTION_READY != c->state) ||
-       GC_is_pid_bigger (ntohl (fc->last_pid_recv.pid),
-                         ntohl (fc->last_ack_sent.pid)) )
-  {
-    return 0;
-  }
-  return (ntohl (fc->last_ack_sent.pid) - ntohl (fc->last_pid_recv.pid));
-}
-
-
-/**
- * Get messages queued in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Number of messages queued.
- */
-unsigned int
-GCC_get_qn (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  return fc->queue_n;
-}
-
-
-/**
- * Get next PID to use.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- * @return Next PID to use.
- */
-struct CadetEncryptedMessageIdentifier
-GCC_get_pid (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier pid;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  pid = fc->next_pid;
-  fc->next_pid.pid = htonl (1 + ntohl (pid.pid));
-  return pid;
-}
-
-
-/**
- * Allow the connection to advertise a buffer of the given size.
- *
- * The connection will send an @c fwd ACK message (so: in direction !fwd)
- * allowing up to last_pid_recv + buffer.
- *
- * @param c Connection.
- * @param buffer How many more messages the connection can accept.
- * @param fwd Is this about FWD traffic? (The ack will go dest->root).
- */
-void
-GCC_allow (struct CadetConnection *c, unsigned int buffer, int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  allowing %s %u messages %s\n",
-       GCC_2s (c), buffer, GC_f2s (fwd));
-  send_ack (c, buffer, fwd, GNUNET_NO);
-}
-
-
-/**
- * Notify other peers on a connection of a broken link. Mark connections
- * to destroy after all traffic has been sent.
- *
- * @param c Connection on which there has been a disconnection.
- * @param peer Peer that disconnected.
- */
-void
-GCC_neighbor_disconnected (struct CadetConnection *c, struct CadetPeer *peer)
-{
-  struct CadetFlowControl *fc;
-  char peer_name[16];
-  int fwd;
-
-  GCC_check_connections ();
-  strncpy (peer_name, GCP_2s (peer), 16);
-  peer_name[15] = '\0';
+  cc->ready_cb = ready_cb;
+  cc->ready_cb_cls = ready_cb_cls;
+  cc->path = path;
+  cc->off = off;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "shutting down %s, %s disconnected\n",
-       GCC_2s (c), peer_name);
-
-  invalidate_paths (c, peer);
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if ( (GNUNET_YES == GCC_is_terminal (c, fwd)) ||
-       (GNUNET_NO != c->destroy) )
-  {
-    /* Local shutdown, or other peer already down (hence 'c->destroy');
-       so there is no one to notify about this, just clean up. */
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
-  }
-  /* Mark FlowControl towards the peer as unavaliable. */
-  fc = fwd ? &c->bck_fc : &c->fwd_fc;
-  fc->queue_max = 0;
-
-  send_broken (c, &my_full_id, GCP_get_id (peer), fwd);
-
-  /* Connection will have at least one pending message
-   * (the one we just scheduled), so delay destruction
-   * and remove from map so we don't use accidentally. */
-  mark_destroyed (c);
-  GNUNET_assert (GNUNET_NO == c->was_removed);
-  c->was_removed = GNUNET_YES;
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                       &c->id.connection_of_tunnel,
-                                                       c));
-  /* Cancel queue in the direction that just died. */
-  connection_cancel_queues (c, ! fwd);
-  GCC_stop_poll (c, ! fwd);
-  unregister_neighbors (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Is this peer the first one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES if origin, #GNUNET_NO if relay/terminal.
+       "Creating %s using path %s\n",
+       GCC_2s (cc),
+       GCPP_2s (path));
+  GCPP_add_connection (path,
+                       off,
+                       cc);
+  for (unsigned int i=0;i<off;i++)
+    GCP_add_connection (GCPP_get_peer_at_offset (path,
+                                                 i),
+                        cc);
+
+  first_hop = GCPP_get_peer_at_offset (path,
+                                       0);
+  cc->mq_man = GCP_request_mq (first_hop,
+                               &manage_first_hop_mq,
+                               cc);
+  return cc;
+}
+
+
+/**
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
  */
-int
-GCC_is_origin (struct CadetConnection *c, int fwd)
-{
-  if (!fwd && c->path->length - 1 == c->own_pos )
-    return GNUNET_YES;
-  if (fwd && 0 == c->own_pos)
-    return GNUNET_YES;
-  return GNUNET_NO;
-}
-
-
-/**
- * Is this peer the last one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *            Note that the ROOT is the terminal for BCK traffic!
- *
- * @return #GNUNET_YES if terminal, #GNUNET_NO if relay/origin.
- */
-int
-GCC_is_terminal (struct CadetConnection *c, int fwd)
-{
-  return GCC_is_origin (c, ! fwd);
-}
-
-
-/**
- * See if we are allowed to send by the next hop in the given direction.
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES in case it's OK to send.
- */
-int
-GCC_is_sendable (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " checking sendability of %s traffic on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return GNUNET_YES;
-  }
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " last ack recv: %u, last pid sent: %u\n",
-       ntohl (fc->last_ack_recv.pid),
-       ntohl (fc->last_pid_sent.pid));
-  if (GC_is_pid_bigger (ntohl (fc->last_ack_recv.pid),
-                        ntohl (fc->last_pid_sent.pid)))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " sendable\n");
-    return GNUNET_YES;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " not sendable\n");
-  return GNUNET_NO;
-}
-
-
-/**
- * Check if this connection is a direct one (never trim a direct connection).
- *
- * @param c Connection.
- *
- * @return #GNUNET_YES in case it's a direct connection, #GNUNET_NO otherwise.
- */
-int
-GCC_is_direct (struct CadetConnection *c)
-{
-  return (c->path->length == 2) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Sends a completely built message on a connection, properly registering
- * all used resources.
- *
- * @param message Message to send.
- * @param payload_type Type of payload, in case the message is encrypted.
- *                     0 for restransmissions (when type is no longer known)
- *                     UINT16_MAX when not applicable.
- * @param payload_id ID of the payload (PID, ACK, ...).
- * @param c Connection on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param force Force the connection to accept the message (buffer overfill).
- * @param cont Continuation called once message is sent. Can be NULL.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel the message before it's sent.
- *         NULL on error.
- *         Invalid on @c cont call.
- */
-struct CadetConnectionQueue *
-GCC_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           uint16_t payload_type,
-                           struct CadetEncryptedMessageIdentifier payload_id,
-                           struct CadetConnection *c, int fwd, int force,
-                           GCC_sent cont, void *cont_cls)
-{
-  struct CadetFlowControl *fc;
-  struct CadetConnectionQueue *q;
-  uint16_t size;
-  uint16_t type;
-
-  size = ntohs (message->size);
-  type = ntohs (message->type);
-
-  GCC_check_connections ();
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (0 == fc->queue_max)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "--> %s (%s %4u) on conn %s (%p) %s [%5u]\n",
-       GC_m2s (type), GC_m2s (payload_type), payload_id, GCC_2s (c), c,
-       GC_f2s(fwd), size);
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  Q_N+ %p %u, PIDsnt: %u, ACKrcv: %u\n",
-           fc,
-           fc->queue_n,
-           ntohl (fc->last_pid_sent.pid),
-           ntohl (fc->last_ack_recv.pid));
-      if (GNUNET_NO == force)
-      {
-        fc->queue_n++;
-      }
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      /* nothing to do here */
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-       /* Should've only be used for restransmissions. */
-      GNUNET_break (0 == payload_type);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-      GNUNET_assert (GNUNET_YES == force);
-      break;
-
-    default:
-      GNUNET_break (0);
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    enum GNUNET_CADET_ChannelOption options,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  unsigned int off;
+
+  off = GCPP_find_peer (path,
+                        destination);
+  GNUNET_assert (UINT_MAX != off);
+  cc = GCPP_get_connection (path,
+                            destination,
+                            off);
+  if (NULL != cc)
+  {
+    int cmp;
+
+    cmp = memcmp (cid,
+                  &cc->cid,
+                  sizeof (*cid));
+    if (0 == cmp)
+    {
+      /* Two peers picked the SAME random connection identifier at the
+         same time for the same path? Must be malicious.  Drop
+         connection (existing and inbound), even if it is the only
+         one. */
+      GNUNET_break_op (0);
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
       return NULL;
-  }
-
-  if (fc->queue_n > fc->queue_max && GNUNET_NO == force)
-  {
-    GNUNET_STATISTICS_update (stats, "# messages dropped (buffer full)",
-                              1, GNUNET_NO);
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "queue full: %u/%u\n",
-         fc->queue_n, fc->queue_max);
-    if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == type)
+    }
+    if (0 < cmp)
     {
-      fc->queue_n--;
+      /* drop existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, dropping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
     }
-    return NULL; /* Drop this message */
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %s %u\n",
-       GCC_2s (c), c->pending_messages);
-  c->pending_messages++;
-
-  q = GNUNET_new (struct CadetConnectionQueue);
-  q->cont = cont;
-  q->cont_cls = cont_cls;
-  q->forced = force;
-  GNUNET_CONTAINER_DLL_insert (fc->q_head, fc->q_tail, q);
-  q->peer_q = GCP_send (get_hop (c, fwd),
-                        message,
-                        payload_type,
-                        payload_id,
-                        c,
-                        fwd,
-                        &conn_message_sent, q);
-  if (NULL == q->peer_q)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "dropping msg on %s, NULL q\n", GCC_2s (c));
-    GNUNET_CONTAINER_DLL_remove (fc->q_head, fc->q_tail, q);
-    GNUNET_free (q);
-    GCC_check_connections ();
-    return NULL;
-  }
-  GCC_check_connections ();
-  return q;
-}
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCC_cancel (struct CadetConnectionQueue *q)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "!  GCC cancel message\n");
-
-  /* send_cancel calls message_sent, which calls q->cont and frees q */
-  GCP_send_cancel (q->peer_q);
-  GCC_check_connections ();
-}
-
-
-/**
- * Sends a CREATE CONNECTION message for a path to a peer.
- * Changes the connection and tunnel states if necessary.
- *
- * @param c Connection to create.
- */
-void
-GCC_send_create (struct CadetConnection *c)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  enum CadetTunnelCState state;
-  size_t size;
-
-  GCC_check_connections ();
-  size = sizeof (struct GNUNET_CADET_ConnectionCreateMessage);
-  size += c->path->length * sizeof (struct GNUNET_PeerIdentity);
-  {
-    /* Allocate message on the stack */
-    unsigned char cbuf[size];
-    struct GNUNET_CADET_ConnectionCreateMessage *msg;
-    struct GNUNET_PeerIdentity *peers;
-
-
-    msg = (struct GNUNET_CADET_ConnectionCreateMessage *) cbuf;
-    msg->header.size = htons (size);
-    msg->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
-    msg->reserved = htonl (0);
-    msg->cid = *GCC_get_id (c);
-    peers = (struct GNUNET_PeerIdentity *) &msg[1];
-    for (int i = 0; i < c->path->length; i++)
+    else
     {
-      GNUNET_PEER_resolve (c->path->peers[i], peers++);
+      /* keep existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, keeping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      return NULL;
     }
-    GNUNET_assert (NULL == c->maintenance_q);
-    c->maintenance_q = GCP_send (get_next_hop (c),
-                                 &msg->header,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE,
-                                 zero,
-                                 c, GNUNET_YES,
-                                 &conn_message_sent, NULL);
   }
 
-  LOG (GNUNET_ERROR_TYPE_INFO, "==> %s %19s on conn %s (%p) FWD [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE), "",
-       GCC_2s (c), c, size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %p %u (create)\n",
-         c, c->pending_messages);
-  c->pending_messages++;
-
-  state = GCT_get_cstate (c->t);
-  if (CADET_TUNNEL_SEARCHING == state || CADET_TUNNEL_NEW == state)
-    GCT_change_cstate (c->t, CADET_TUNNEL_WAITING);
-  if (CADET_CONNECTION_NEW == c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-  GCC_check_connections ();
+  return connection_create (destination,
+                            path,
+                            off,
+                            options,
+                            ct,
+                            cid,
+                            CADET_CONNECTION_CREATE_RECEIVED,
+                            ready_cb,
+                            ready_cb_cls);
 }
 
 
 /**
- * Send an ACK on the appropriate connection/channel, depending on
- * the direction and the position of the peer.
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.
  *
- * @param c Which connection to send the hop-by-hop ACK.
- * @param fwd Is this a fwd ACK? (will go dest->root).
- * @param force Send the ACK even if suboptimal (e.g. requested by POLL).
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct tunnel that uses the connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_send_ack (struct CadetConnection *c, int fwd, int force)
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            enum GNUNET_CADET_ChannelOption options,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls)
 {
-  unsigned int buffer;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCC send %s ACK on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 
-  if (GNUNET_NO != c->destroy)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  being destroyed, why bother...\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Get available buffer space */
-  if (GCC_is_terminal (c, fwd))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  getting from all channels\n");
-    buffer = GCT_get_channels_buffer (c->t);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  getting from one connection\n");
-    buffer = GCC_get_buffer (c, fwd);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer available: %u\n", buffer);
-  if (0 == buffer && GNUNET_NO == force)
-  {
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Send available buffer space */
-  if (GNUNET_YES == GCC_is_origin (c, fwd))
-  {
-    GNUNET_assert (NULL != c->t);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on channels...\n");
-    GCT_unchoke_channels (c->t);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on connection\n");
-    send_ack (c, buffer, fwd, force);
-  }
-  GCC_check_connections ();
+  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE,
+                              &cid,
+                              sizeof (cid));
+  return connection_create (destination,
+                            path,
+                            off,
+                            options,
+                            ct,
+                            &cid,
+                            CADET_CONNECTION_NEW,
+                            ready_cb,
+                            ready_cb_cls);
 }
 
 
 /**
- * Send a message to all peers in this connection that the connection
- * is no longer valid.
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
  *
- * If some peer should not receive the message, it should be zero'ed out
- * before calling this function.
- *
- * @param c The connection whose peers to notify.
+ * @param cc connection identification
+ * @param env envelope with message to transmit; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
 void
-GCC_send_destroy (struct CadetConnection *c)
+GCC_transmit (struct CadetConnection *cc,
+              struct GNUNET_MQ_Envelope *env)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionDestroyMessage msg;
-
-  if (GNUNET_YES == c->destroy)
-    return;
-  GCC_check_connections ();
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
-  msg.cid = c->id;
-  msg.reserved = htonl (0);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-              "  sending connection destroy for connection %s\n",
-              GCC_2s (c));
-
-  if (GNUNET_NO == GCC_is_terminal (c, GNUNET_YES))
-    (void) GCC_send_prebuilt_message (&msg.header,
-                                      UINT16_MAX,
-                                      zero,
-                                      c,
-                                      GNUNET_YES, GNUNET_YES, NULL, NULL);
-  if (GNUNET_NO == GCC_is_terminal (c, GNUNET_NO))
-    (void) GCC_send_prebuilt_message (&msg.header,
-                                      UINT16_MAX,
-                                      zero,
-                                      c,
-                                      GNUNET_NO, GNUNET_YES, NULL, NULL);
-  mark_destroyed (c);
-  GCC_check_connections ();
+       "Scheduling message for transmission on %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (CADET_CONNECTION_READY == cc->state);
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  cc->mqm_ready = GNUNET_NO;
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * @brief Start a polling timer for the connection.
- *
- * When a neighbor does not accept more traffic on the connection it could be
- * caused by a simple congestion or by a lost ACK. Polling enables to check
- * for the lastest ACK status for a connection.
+ * Obtain the path used by this connection.
  *
- * @param c Connection.
- * @param fwd Should we poll in the FWD direction?
+ * @param cc connection
+ * @return path to @a cc
  */
-void
-GCC_start_poll (struct CadetConnection *c, int fwd)
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL %s requested\n",
-       GC_f2s (fwd));
-  if (NULL != fc->poll_task || NULL != fc->poll_msg)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  POLL already in progress (t: %p, m: %p)\n",
-         fc->poll_task, fc->poll_msg);
-    return;
-  }
-  if (0 == fc->queue_max)
-  {
-    /* Should not be needed, traffic should've been cancelled. */
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  POLL not possible, peer disconnected\n");
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL started on request\n");
-  fc->poll_task = GNUNET_SCHEDULER_add_delayed (fc->poll_time, &send_poll, fc);
+  return cc->path;
 }
 
 
 /**
- * @brief Stop polling a connection for ACKs.
+ * Obtain unique ID for the connection.
  *
- * Once we have enough ACKs for future traffic, polls are no longer necessary.
- *
- * @param c Connection.
- * @param fwd Should we stop the poll in the FWD direction?
+ * @param cc connection.
+ * @return unique number of the connection
  */
-void
-GCC_stop_poll (struct CadetConnection *c, int fwd)
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (NULL != fc->poll_task)
-  {
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-  }
-  if (NULL != fc->poll_msg)
-  {
-    GCC_cancel (fc->poll_msg);
-    fc->poll_msg = NULL;
-  }
+  return &cc->cid;
 }
 
 
 /**
  * Get a (static) string for a connection.
  *
- * @param c Connection.
+ * @param cc Connection.
  */
 const char *
-GCC_2s (const struct CadetConnection *c)
+GCC_2s (const struct CadetConnection *cc)
 {
-  if (NULL == c)
-    return "NULL";
+  static char buf[128];
 
-  if (NULL != c->t)
-  {
-    static char buf[128];
+  if (NULL == cc)
+    return "Connection(NULL)";
 
-    SPRINTF (buf, "%s (->%s)",
-             GNUNET_sh2s (&GCC_get_id (c)->connection_of_tunnel),
-             GCT_2s (c->t));
+  if (NULL != cc->ct)
+  {
+    GNUNET_snprintf (buf,
+                     sizeof (buf),
+                     "Connection %s (%s)",
+                     GNUNET_sh2s (&cc->cid.connection_of_tunnel),
+                     GCT_2s (cc->ct->t));
     return buf;
   }
-  return GNUNET_sh2s (&c->id.connection_of_tunnel);
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Connection %s",
+                   GNUNET_sh2s (&cc->cid.connection_of_tunnel));
+  return buf;
 }
 
 
+#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
+
+
 /**
- * Log all possible info about the connection state.
+ * Log connection info.
  *
- * @param c Connection to debug.
+ * @param cc connection
  * @param level Debug level to use.
  */
 void
-GCC_debug (const struct CadetConnection *c, enum GNUNET_ErrorType level)
+GCC_debug (struct CadetConnection *cc,
+           enum GNUNET_ErrorType level)
 {
   int do_log;
-  char *s;
 
   do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
                                        "cadet-con",
                                        __FILE__, __FUNCTION__, __LINE__);
   if (0 == do_log)
     return;
-
-  if (NULL == c)
+  if (NULL == cc)
   {
-    LOG2 (level, "CCC DEBUG NULL CONNECTION\n");
+    LOG2 (level,
+          "Connection (NULL)\n");
     return;
   }
-
-  LOG2 (level, "CCC DEBUG CONNECTION %s\n", GCC_2s (c));
-  s = path_2s (c->path);
-  LOG2 (level, "CCC  path %s, own pos: %u\n", s, c->own_pos);
-  GNUNET_free (s);
-  LOG2 (level, "CCC  state: %s, destroy: %u\n",
-        GCC_state2s (c->state), c->destroy);
-  LOG2 (level, "CCC  pending messages: %u\n", c->pending_messages);
-  if (NULL != c->perf)
-    LOG2 (level, "CCC  us/byte: %f\n", c->perf->avg);
-
-  LOG2 (level, "CCC  FWD flow control:\n");
-  LOG2 (level, "CCC   queue: %u/%u\n", c->fwd_fc.queue_n, c->fwd_fc.queue_max);
-  LOG2 (level, "CCC   last PID sent: %5u, recv: %5u\n",
-        ntohl (c->fwd_fc.last_pid_sent.pid),
-        ntohl (c->fwd_fc.last_pid_recv.pid));
-  LOG2 (level, "CCC   last ACK sent: %5u, recv: %5u\n",
-        ntohl (c->fwd_fc.last_ack_sent.pid),
-        ntohl (c->fwd_fc.last_ack_recv.pid));
-  LOG2 (level, "CCC   recv PID bitmap: %X\n", c->fwd_fc.recv_bitmap);
-  LOG2 (level, "CCC   poll: task %d, msg  %p, msg_ack %p)\n",
-        c->fwd_fc.poll_task, c->fwd_fc.poll_msg, c->fwd_fc.ack_msg);
-
-  LOG2 (level, "CCC  BCK flow control:\n");
-  LOG2 (level, "CCC   queue: %u/%u\n", c->bck_fc.queue_n, c->bck_fc.queue_max);
-  LOG2 (level, "CCC   last PID sent: %5u, recv: %5u\n",
-        ntohl (c->bck_fc.last_pid_sent.pid),
-        ntohl (c->bck_fc.last_pid_recv.pid));
-  LOG2 (level, "CCC   last ACK sent: %5u, recv: %5u\n",
-        ntohl (c->bck_fc.last_ack_sent.pid),
-        ntohl (c->bck_fc.last_ack_recv.pid));
-  LOG2 (level, "CCC   recv PID bitmap: %X\n", c->bck_fc.recv_bitmap);
-  LOG2 (level, "CCC   poll: task %d, msg  %p, msg_ack %p)\n",
-        c->bck_fc.poll_task, c->bck_fc.poll_msg, c->bck_fc.ack_msg);
-
-  LOG2 (level, "CCC DEBUG CONNECTION END\n");
+  LOG2 (level,
+        "%s to %s via path %s in state %d is %s\n",
+        GCC_2s (cc),
+        GCP_2s (cc->destination),
+        GCPP_2s (cc->path),
+        cc->state,
+        (GNUNET_YES == cc->mqm_ready) ? "ready" : "busy");
 }
+
+/* end of gnunet-service-cadet-new_connection.c */
diff --git a/src/cadet/gnunet-service-cadet_connection.h b/src/cadet/gnunet-service-cadet_connection.h
index 307cb42c2..fdb184366 100644
--- a/src/cadet/gnunet-service-cadet_connection.h
+++ b/src/cadet/gnunet-service-cadet_connection.h
@@ -1,6 +1,7 @@
+
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,557 +21,319 @@
 
 /**
  * @file cadet/gnunet-service-cadet_connection.h
- * @brief cadet service; dealing with connections
+ * @brief A connection is a live end-to-end messaging mechanism
+ *       where the peers are identified by a path and know how
+ *       to forward along the route using a connection identifier
+ *       for routing the data.
  * @author Bartlomiej Polot
- *
- * All functions in this file use the prefix GCC (GNUnet Cadet Connection)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_CONNECTION_H
 #define GNUNET_SERVICE_CADET_CONNECTION_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
 #include "gnunet_util_lib.h"
-
-
-/**
- * All the states a connection can be in.
- */
-enum CadetConnectionState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_CONNECTION_NEW,
-
-  /**
-   * Connection create message sent, waiting for ACK.
-   */
-  CADET_CONNECTION_SENT,
-
-  /**
-   * Connection ACK sent, waiting for ACK.
-   */
-  CADET_CONNECTION_ACK,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CONNECTION_READY,
-
-  /**
-   * Connection to be destroyed, just waiting to empty queues.
-   */
-  CADET_CONNECTION_DESTROYED,
-
-  /**
-   * Connection to be destroyed because of a distant peer, same as DESTROYED.
-   */
-  CADET_CONNECTION_BROKEN,
-};
-
-
-/**
- * Struct containing all information regarding a connection to a peer.
- */
-struct CadetConnection;
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetConnectionQueue;
-
-#include "cadet_path.h"
-#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_peer.h"
+#include "cadet_protocol.h"
 
 
 /**
- * Check invariants for all connections using #check_neighbours().
- */
-void
-GCC_check_connections (void);
-
-
-/**
- * Callback called when a queued message is sent.
+ * Function called to notify tunnel about change in our readyness.
  *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
+ * @param cls closure
+ * @param is_ready #GNUNET_YES if the connection is now ready for transmission,
+ *                 #GNUNET_NO if the connection is no longer ready for transmission
  */
 typedef void
-(*GCC_sent) (void *cls,
-             struct CadetConnection *c,
-             struct CadetConnectionQueue *q,
-             uint16_t type,
-             int fwd,
-             size_t size);
+(*GCC_ReadyCallback)(void *cls,
+                     int is_ready);
 
 
 /**
- * Handler for connection creation.
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection to destroy
  */
 void
-GCC_handle_create (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionCreateMessage *msg);
+GCC_destroy_without_core (struct CadetConnection *cc);
 
 
 /**
- * Handler for connection confirmations.
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection to destroy
  */
 void
-GCC_handle_confirm (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionCreateAckMessage *msg);
+GCC_destroy_without_tunnel (struct CadetConnection *cc);
 
 
 /**
- * Handler for notifications of broken connections.
+ * Lookup a connection by its identifier.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
  */
-void
-GCC_handle_broken (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionBrokenMessage *msg);
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Handler for notifications of destroyed connections.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_destroy (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionDestroyMessage *msg);
 
 /**
- * Handler for cadet network traffic hop-by-hop acks.
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_handle_ack (struct CadetPeer *peer,
-                const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg);
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            enum GNUNET_CADET_ChannelOption options,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls);
 
-/**
- * Handler for cadet network traffic hop-by-hop data counter polls.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_poll (struct CadetPeer *peer,
-                 const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg);
 
 /**
- * Handler for key exchange traffic (Axolotl KX).
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
  */
-void
-GCC_handle_kx (struct CadetPeer *peer,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    enum GNUNET_CADET_ChannelOption options,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls);
+
 
 /**
- * Handler for encrypted cadet network traffic (channel mgmt, data).
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection identification
+ * @param env envelope with message to transmit;
+ *            the #GNUNET_MQ_notify_send() must not have yet been used
+ *            for the envelope.  Also, the message better match the
+ *            connection identifier of this connection...
  */
 void
-GCC_handle_encrypted (struct CadetPeer *peer,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
+GCC_transmit (struct CadetConnection *cc,
+              struct GNUNET_MQ_Envelope *env);
 
-/**
- * Core handler for axolotl key exchange traffic.
- *
- * @param cls Closure (unused).
- * @param message Message received.
- * @param peer Neighbor who sent the message.
- *
- * @return GNUNET_OK, to keep the connection open.
- */
-int
-GCC_handle_ax_kx (void *cls, const struct GNUNET_PeerIdentity *peer,
-                  const struct GNUNET_MessageHeader *message);
 
 /**
- * Core handler for axolotl encrypted cadet network traffic.
+ * A CREATE_ACK was received for this connection, process it.
  *
- * @param cls Closure (unused).
- * @param message Message received.
- * @param peer Neighbor who sent the message.
- *
- * @return GNUNET_OK, to keep the connection open.
+ * @param cc the connection that got the ACK.
  */
-int
-GCC_handle_ax (void *cls, const struct GNUNET_PeerIdentity *peer,
-               struct GNUNET_MessageHeader *message);
+void
+GCC_handle_connection_create_ack (struct CadetConnection *cc);
 
-/**
- * Core handler for cadet keepalives.
- *
- * @param cls closure
- * @param message message
- * @param peer peer identity this notification is about
- * @return GNUNET_OK to keep the connection open,
- *         GNUNET_SYSERR to close it (signal serious error)
- *
- * TODO: Check who we got this from, to validate route.
- */
-int
-GCC_handle_keepalive (void *cls, const struct GNUNET_PeerIdentity *peer,
-                      const struct GNUNET_MessageHeader *message);
 
 /**
- * Send an ACK on the appropriate connection/channel, depending on
- * the direction and the position of the peer.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
  *
- * @param c Which connection to send the hop-by-hop ACK.
- * @param fwd Is this a fwd ACK? (will go dest->root).
- * @param force Send the ACK even if suboptimal (e.g. requested by POLL).
+ * @param cc connection that got the duplicate CREATE
  */
 void
-GCC_send_ack (struct CadetConnection *c, int fwd, int force);
+GCC_handle_duplicate_create (struct CadetConnection *cc);
 
-/**
- * Initialize the connections subsystem
- *
- * @param c Configuration handle.
- */
-void
-GCC_init (const struct GNUNET_CONFIGURATION_Handle *c);
 
 /**
- * Shut down the connections subsystem.
+ * Handle KX message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_shutdown (void);
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
 
-/**
- * Create a connection.
- *
- * @param cid Connection ID (either created locally or imposed remotely).
- * @param t Tunnel this connection belongs to (or NULL for transit connections);
- * @param path Path this connection has to use (copy is made).
- * @param own_pos Own position in the @c path path.
- *
- * @return Newly created connection.
- *         NULL in case of error: own id not in path, wrong neighbors, ...
- */
-struct CadetConnection *
-GCC_new (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-         struct CadetTunnel *t,
-         struct CadetPeerPath *path,
-         unsigned int own_pos);
 
 /**
- * Connection is no longer needed: destroy it.
- *
- * Cancels all pending traffic (including possible DESTROY messages), all
- * maintenance tasks and removes the connection from neighbor peers and tunnel.
+ * Handle KX_AUTH message.
  *
- * @param c Connection to destroy.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_destroy (struct CadetConnection *c);
-
-/**
- * Get the connection ID.
- *
- * @param c Connection to get the ID from.
- *
- * @return ID of the connection.
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (const struct CadetConnection *c);
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
 
 
 /**
- * Get the connection path.
- *
- * @param c Connection to get the path from.
- *
- * @return path used by the connection.
+ * Performance metrics for a connection.
  */
-const struct CadetPeerPath *
-GCC_get_path (const struct CadetConnection *c);
+struct CadetConnectionMetrics
+{
 
-/**
- * Get the connection state.
- *
- * @param c Connection to get the state from.
- *
- * @return state of the connection.
- */
-enum CadetConnectionState
-GCC_get_state (const struct CadetConnection *c);
+  /**
+   * Our current best estimate of the latency, based on a weighted
+   * average of at least @a latency_datapoints values.
+   */
+  struct GNUNET_TIME_Relative aged_latency;
 
-/**
- * Get the connection tunnel.
- *
- * @param c Connection to get the tunnel from.
- *
- * @return tunnel of the connection.
- */
-struct CadetTunnel *
-GCC_get_tunnel (const struct CadetConnection *c);
+  /**
+   * When was this connection first established? (by us sending or
+   * receiving the CREATE_ACK for the first time)
+   */
+  struct GNUNET_TIME_Absolute age;
 
-/**
- * Get free buffer space in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - max_msgs_queue/max_connections]
- */
-unsigned int
-GCC_get_buffer (struct CadetConnection *c, int fwd);
+  /**
+   * When was this connection last used? (by us sending or
+   * receiving a PAYLOAD message on it)
+   */
+  struct GNUNET_TIME_Absolute last_use;
 
-/**
- * Get how many messages have we allowed to send to us from a direction.
- *
- * @param c Connection.
- * @param fwd Are we asking about traffic from FWD (BCK messages)?
- *
- * @return last_ack_sent - last_pid_recv
- */
-unsigned int
-GCC_get_allowed (struct CadetConnection *c, int fwd);
+  /**
+   * How many packets that ought to generate an ACK did we send via
+   * this connection?
+   */
+  unsigned long long num_acked_transmissions;
 
-/**
- * Get messages queued in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Number of messages queued.
- */
-unsigned int
-GCC_get_qn (struct CadetConnection *c, int fwd);
+  /**
+   * Number of packets that were sent via this connection did actually
+   * receive an ACK?  (Note: ACKs may be transmitted and lost via
+   * other connections, so this value should only be interpreted
+   * relative to @e num_acked_transmissions and in relation to other
+   * connections.)
+   */
+  unsigned long long num_successes;
 
-/**
- * Get next PID to use.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- * @return Next PID to use.
- */
-struct CadetEncryptedMessageIdentifier
-GCC_get_pid (struct CadetConnection *c, int fwd);
+};
 
-/**
- * Allow the connection to advertise a buffer of the given size.
- *
- * The connection will send an @c fwd ACK message (so: in direction !fwd)
- * allowing up to last_pid_recv + buffer.
- *
- * @param c Connection.
- * @param buffer How many more messages the connection can accept.
- * @param fwd Is this about FWD traffic? (The ack will go dest->root).
- */
-void
-GCC_allow (struct CadetConnection *c, unsigned int buffer, int fwd);
 
 /**
- * Send FWD keepalive packets for a connection.
+ * Obtain performance @a metrics from @a cc.
  *
- * @param cls Closure (connection for which to send the keepalive).
- * @param tc Notification context.
+ * @param cc connection to query
+ * @return the metrics
  */
-void
-GCC_fwd_keepalive (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc);
+
 
 /**
- * Send BCK keepalive packets for a connection.
+ * Handle encrypted message.
  *
- * @param cls Closure (connection for which to send the keepalive).
- * @param tc Notification context.
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
  */
 void
-GCC_bck_keepalive (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
 
 
 /**
- * Notify other peers on a connection of a broken link. Mark connections
- * to destroy after all traffic has been sent.
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
  *
- * @param c Connection on which there has been a disconnection.
- * @param peer Peer that disconnected.
+ * @param cid connection identifier where we expect an ACK
  */
 void
-GCC_neighbor_disconnected (struct CadetConnection *c, struct CadetPeer *peer);
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Is this peer the first one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES if origin, #GNUNET_NO if relay/terminal.
- */
-int
-GCC_is_origin (struct CadetConnection *c, int fwd);
 
 /**
- * Is this peer the last one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *            Note that the ROOT is the terminal for BCK traffic!
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cti.
  *
- * @return #GNUNET_YES if terminal, #GNUNET_NO if relay/origin.
+ * @param cid connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
  */
-int
-GCC_is_terminal (struct CadetConnection *c, int fwd);
-
-/**
- * See if we are allowed to send by the next hop in the given direction.
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES in case it's OK to send.
- */
-int
-GCC_is_sendable (struct CadetConnection *c, int fwd);
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Check if this connection is a direct one (never trim a direct connection).
- *
- * @param c Connection.
- *
- * @return #GNUNET_YES in case it's a direct connection, #GNUNET_NO otherwise.
- */
-int
-GCC_is_direct (struct CadetConnection *c);
 
 /**
- * Cancel a previously sent message while it's in the queue.
+ * We observed some the given @a latency on the connection
+ * identified by @a cti.  (The same connection was taken
+ * in both directions.)
  *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
+ * @param cti connection identifier where we measured latency
+ * @param latency the observed latency
  */
 void
-GCC_cancel (struct CadetConnectionQueue *q);
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                      struct GNUNET_TIME_Relative latency);
 
-/**
- * Sends an already built message on a connection, properly registering
- * all used resources.
- *
- * @param message Message to send.
- * @param payload_type Type of payload, in case the message is encrypted.
- *                     0 for restransmissions (when type is no longer known)
- *                     UINT16_MAX when not applicable.
- * @param payload_id ID of the payload (PID, ACK, ...).
- * @param c Connection on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param force Force the connection to accept the message (buffer overfill).
- * @param cont Continuation called once message is sent. Can be NULL.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel the message before it's sent.
- *         NULL on error.
- *         Invalid on @c cont call.
- */
-struct CadetConnectionQueue *
-GCC_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           uint16_t payload_type,
-                           struct CadetEncryptedMessageIdentifier payload_id,
-                           struct CadetConnection *c, int fwd, int force,
-                           GCC_sent cont, void *cont_cls);
 
 /**
- * Sends a CREATE CONNECTION message for a path to a peer.
- * Changes the connection and tunnel states if necessary.
+ * Return the tunnel associated with this connection.
  *
- * @param connection Connection to create.
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
  */
-void
-GCC_send_create (struct CadetConnection *connection);
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc);
 
-/**
- * Send a message to all peers in this connection that the connection
- * is no longer valid.
- *
- * If some peer should not receive the message, it should be zero'ed out
- * before calling this function.
- *
- * @param c The connection whose peers to notify.
- */
-void
-GCC_send_destroy (struct CadetConnection *c);
 
 /**
- * @brief Start a polling timer for the connection.
+ * Obtain the path used by this connection.
  *
- * When a neighbor does not accept more traffic on the connection it could be
- * caused by a simple congestion or by a lost ACK. Polling enables to check
- * for the lastest ACK status for a connection.
- *
- * @param c Connection.
- * @param fwd Should we poll in the FWD direction?
+ * @param cc connection
+ * @return path to @a cc
  */
-void
-GCC_start_poll (struct CadetConnection *c, int fwd);
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc);
 
 
 /**
- * @brief Stop polling a connection for ACKs.
- *
- * Once we have enough ACKs for future traffic, polls are no longer necessary.
+ * Obtain unique ID for the connection.
  *
- * @param c Connection.
- * @param fwd Should we stop the poll in the FWD direction?
+ * @param cc connection.
+ * @return unique number of the connection
  */
-void
-GCC_stop_poll (struct CadetConnection *c, int fwd);
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc);
+
 
 /**
  * Get a (static) string for a connection.
  *
- * @param c Connection.
+ * @param cc Connection.
  */
 const char *
-GCC_2s (const struct CadetConnection *c);
+GCC_2s (const struct CadetConnection *cc);
+
 
 /**
- * Log all possible info about the connection state.
+ * Log connection info.
  *
- * @param c Connection to debug.
+ * @param cc connection
  * @param level Debug level to use.
  */
 void
-GCC_debug (const struct CadetConnection *c, enum GNUNET_ErrorType level);
+GCC_debug (struct CadetConnection *cc,
+           enum GNUNET_ErrorType level);
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
 
-/* ifndef GNUNET_SERVICE_CADET_CONNECTION_H */
 #endif
-/* end of gnunet-service-cadet_connection.h */
diff --git a/src/cadet/gnunet-service-cadet-new_core.c b/src/cadet/gnunet-service-cadet_core.c
index 4a4ead05b..ae03b4f35 100644
--- a/src/cadet/gnunet-service-cadet-new_core.c
+++ b/src/cadet/gnunet-service-cadet_core.c
@@ -30,22 +30,64 @@
  * - Optimization: given BROKEN messages, destroy paths (?)
  */
 #include "platform.h"
-#include "gnunet-service-cadet-new_core.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
+#include "gnunet-service-cadet_core.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
 #include "gnunet_core_service.h"
+#include "gnunet_statistics_service.h"
 #include "cadet_protocol.h"
 
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-cor",__VA_ARGS__)
 
+/**
+ * Information we keep per direction for a route.
+ */
+struct RouteDirection;
+
 
 /**
- * Number of messages we are willing to buffer per route.
+ * Set of CadetRoutes that have exactly the same number of messages
+ * in their buffer.  Used so we can efficiently find all of those
+ * routes that have the current maximum of messages in the buffer (in
+ * case we have to purge).
  */
-#define ROUTE_BUFFER_SIZE 8
+struct Rung
+{
+
+  /**
+   * Rung of RouteDirections with one more buffer entry each.
+   */
+  struct Rung *next;
+
+  /**
+   * Rung of RouteDirections with one less buffer entry each.
+   */
+  struct Rung *prev;
+
+  /**
+   * DLL of route directions with a number of buffer entries matching this rung.
+   */
+  struct RouteDirection *rd_head;
+
+  /**
+   * DLL of route directions with a number of buffer entries matching this rung.
+   */
+  struct RouteDirection *rd_tail;
+
+  /**
+   * Total number of route directions in this rung.
+   */
+  unsigned int num_routes;
+
+  /**
+   * Number of messages route directions at this rung have
+   * in their buffer.
+   */
+  unsigned int rung_off;
+};
 
 
 /**
@@ -53,35 +95,46 @@
  */
 struct RouteDirection
 {
+
   /**
-   * Target peer.
+   * DLL of other route directions within the same `struct Rung`.
    */
-  struct CadetPeer *hop;
+  struct RouteDirection *prev;
 
   /**
-   * Route this direction is part of.
+   * DLL of other route directions within the same `struct Rung`.
    */
-  struct CadetRoute *my_route;
+  struct RouteDirection *next;
 
   /**
-   * Message queue manager for @e hop.
+   * Rung of this route direction (matches length of the buffer DLL).
    */
-  struct GCP_MessageQueueManager *mqm;
+  struct Rung *rung;
+
+  /**
+   * Head of DLL of envelopes we have in the buffer for this direction.
+   */
+  struct GNUNET_MQ_Envelope *env_head;
 
   /**
-   * Cyclic message buffer to @e hop.
+   * Tail of DLL of envelopes we have in the buffer for this direction.
    */
-  struct GNUNET_MQ_Envelope *out_buffer[ROUTE_BUFFER_SIZE];
+  struct GNUNET_MQ_Envelope *env_tail;
 
   /**
-   * Next write offset to use to append messages to @e out_buffer.
+   * Target peer.
+   */
+  struct CadetPeer *hop;
+
+  /**
+   * Route this direction is part of.
    */
-  unsigned int out_wpos;
+  struct CadetRoute *my_route;
 
   /**
-   * Next read offset to use to retrieve messages from @e out_buffer.
+   * Message queue manager for @e hop.
    */
-  unsigned int out_rpos;
+  struct GCP_MessageQueueManager *mqm;
 
   /**
    * Is @e mqm currently ready for transmission?
@@ -122,6 +175,15 @@ struct CadetRoute
    */
   struct GNUNET_TIME_Absolute last_use;
 
+  /**
+   * Position of this route in the #route_heap.
+   */
+  struct GNUNET_CONTAINER_HeapNode *hn;
+
+  /**
+   * Options for the route, control buffering.
+   */
+  enum GNUNET_CADET_ChannelOption options;
 };
 
 
@@ -135,6 +197,47 @@ static struct GNUNET_CORE_Handle *core;
  */
 static struct GNUNET_CONTAINER_MultiShortmap *routes;
 
+/**
+ * Heap of routes, MIN-sorted by last activity.
+ */
+static struct GNUNET_CONTAINER_Heap *route_heap;
+
+/**
+ * Rung zero (always pointed to by #rung_head).
+ */
+static struct Rung rung_zero;
+
+/**
+ * DLL of rungs, with the head always point to a rung of
+ * route directions with no messages in the queue.
+ */
+static struct Rung *rung_head = &rung_zero;
+
+/**
+ * Tail of the #rung_head DLL.
+ */
+static struct Rung *rung_tail = &rung_zero;
+
+/**
+ * Maximum number of concurrent routes this peer will support.
+ */
+static unsigned long long max_routes;
+
+/**
+ * Maximum number of envelopes we will buffer at this peer.
+ */
+static unsigned long long max_buffers;
+
+/**
+ * Current number of envelopes we have buffered at this peer.
+ */
+static unsigned long long cur_buffers;
+
+/**
+ * Task to timeout routes.
+ */
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
 
 /**
  * Get the route corresponding to a hash.
@@ -150,6 +253,91 @@ get_route (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 
 
 /**
+ * Lower the rung in which @a dir is by 1.
+ *
+ * @param dir direction to lower in rung.
+ */
+static void
+lower_rung (struct RouteDirection *dir)
+{
+  struct Rung *rung = dir->rung;
+  struct Rung *prev;
+
+  GNUNET_CONTAINER_DLL_remove (rung->rd_head,
+                               rung->rd_tail,
+                               dir);
+  prev = rung->prev;
+  GNUNET_assert (NULL != prev);
+  if (prev->rung_off != rung->rung_off - 1)
+  {
+    prev = GNUNET_new (struct Rung);
+    prev->rung_off = rung->rung_off - 1;
+    GNUNET_CONTAINER_DLL_insert_after (rung_head,
+                                       rung_tail,
+                                       rung->prev,
+                                       prev);
+  }
+  GNUNET_assert (NULL != prev);
+  GNUNET_CONTAINER_DLL_insert (prev->rd_head,
+                               prev->rd_tail,
+                               dir);
+  dir->rung = prev;
+}
+
+
+/**
+ * Discard the buffer @a env from the route direction @a dir and
+ * move @a dir down a rung.
+ *
+ * @param dir direction that contains the @a env in the buffer
+ * @param env envelope to discard
+ */
+static void
+discard_buffer (struct RouteDirection *dir,
+                struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_MQ_dll_remove (&dir->env_head,
+                        &dir->env_tail,
+                        env);
+  cur_buffers--;
+  GNUNET_MQ_discard (env);
+  lower_rung (dir);
+  GNUNET_STATISTICS_set (stats,
+                         "# buffer use",
+                         cur_buffers,
+                         GNUNET_NO);
+}
+
+
+/**
+ * Discard all messages from the highest rung, to make space.
+ */
+static void
+discard_all_from_rung_tail ()
+{
+  struct Rung *tail = rung_tail;
+  struct RouteDirection *dir;
+
+  while (NULL != (dir = tail->rd_head))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Queue full due new message %s on connection %s, dropping old message\n",
+         GNUNET_sh2s (&dir->my_route->cid.connection_of_tunnel));
+    GNUNET_STATISTICS_update (stats,
+                              "# messages dropped due to full buffer",
+                              1,
+                              GNUNET_NO);
+    discard_buffer (dir,
+                    dir->env_head);
+  }
+  GNUNET_CONTAINER_DLL_remove (rung_head,
+                               rung_tail,
+                               tail);
+  GNUNET_free (tail);
+}
+
+
+/**
  * We message @a msg from @a prev.  Find its route by @a cid and
  * forward to the next hop.  Drop and signal broken route if we do not
  * have a route.
@@ -165,6 +353,8 @@ route_message (struct CadetPeer *prev,
 {
   struct CadetRoute *route;
   struct RouteDirection *dir;
+  struct Rung *rung;
+  struct Rung *nxt;
   struct GNUNET_MQ_Envelope *env;
 
   route = get_route (cid);
@@ -178,6 +368,13 @@ route_message (struct CadetPeer *prev,
          ntohs (msg->type),
          GCP_2s (prev),
          GNUNET_sh2s (&cid->connection_of_tunnel));
+    switch (ntohs (msg->type))
+    {
+    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
+    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
+      /* No need to respond to these! */
+      return;
+    }
     env = GNUNET_MQ_msg (bm,
                          GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
     bm->cid = *cid;
@@ -186,6 +383,9 @@ route_message (struct CadetPeer *prev,
                   env);
     return;
   }
+  route->last_use = GNUNET_TIME_absolute_get ();
+  GNUNET_CONTAINER_heap_update_cost (route->hn,
+                                     route->last_use.abs_value_us);
   dir = (prev == route->prev.hop) ? &route->next : &route->prev;
   if (GNUNET_YES == dir->is_ready)
   {
@@ -200,22 +400,59 @@ route_message (struct CadetPeer *prev,
               GNUNET_MQ_msg_copy (msg));
     return;
   }
-  env = dir->out_buffer[dir->out_wpos];
-  if (NULL != env)
+  /* Check if buffering is disallowed, and if so, make sure we only queue
+     one message per direction. */
+  if ( (0 != (route->options & GNUNET_CADET_OPTION_NOBUFFER)) &&
+       (NULL != dir->env_head) )
+    discard_buffer (dir,
+                    dir->env_head);
+  rung = dir->rung;
+  if (cur_buffers == max_buffers)
   {
-    /* Queue full, drop earliest message in queue */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Queue full due to new message of type %u from %s to %s on connection %s, dropping old message\n",
-         ntohs (msg->type),
-         GCP_2s (prev),
-         GNUNET_i2s (GCP_get_id (dir->hop)),
-         GNUNET_sh2s (&cid->connection_of_tunnel));
-    GNUNET_assert (dir->out_rpos == dir->out_wpos);
-    GNUNET_MQ_discard (env);
-    dir->out_rpos++;
-    if (ROUTE_BUFFER_SIZE == dir->out_rpos)
-      dir->out_rpos = 0;
+    /* Need to make room. */
+    if (NULL != rung->next)
+    {
+      /* Easy case, drop messages from route directions in highest rung */
+      discard_all_from_rung_tail ();
+    }
+    else
+    {
+      /* We are in the highest rung, drop our own! */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Queue full due new message %s on connection %s, dropping old message\n",
+           GNUNET_sh2s (&dir->my_route->cid.connection_of_tunnel));
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to full buffer",
+                                1,
+                                GNUNET_NO);
+      discard_buffer (dir,
+                      dir->env_head);
+      rung = dir->rung;
+    }
+  }
+  /* remove 'dir' from current rung */
+  GNUNET_CONTAINER_DLL_remove (rung->rd_head,
+                               rung->rd_tail,
+                               dir);
+  /* make 'nxt' point to the next higher rung, creat if necessary */
+  nxt = rung->next;
+  if ( (NULL == nxt) ||
+       (rung->rung_off + 1 != nxt->rung_off) )
+  {
+    nxt = GNUNET_new (struct Rung);
+    nxt->rung_off = rung->rung_off + 1;
+    GNUNET_CONTAINER_DLL_insert_after (rung_head,
+                                       rung_tail,
+                                       rung,
+                                       nxt);
   }
+  /* insert 'dir' into next higher rung */
+  GNUNET_CONTAINER_DLL_insert (nxt->rd_head,
+                               nxt->rd_tail,
+                               dir);
+  dir->rung = nxt;
+
+  /* add message into 'dir' buffer */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Queueing new message of type %u from %s to %s on connection %s\n",
        ntohs (msg->type),
@@ -223,10 +460,23 @@ route_message (struct CadetPeer *prev,
        GNUNET_i2s (GCP_get_id (dir->hop)),
        GNUNET_sh2s (&cid->connection_of_tunnel));
   env = GNUNET_MQ_msg_copy (msg);
-  dir->out_buffer[dir->out_wpos] = env;
-  dir->out_wpos++;
-  if (ROUTE_BUFFER_SIZE == dir->out_wpos)
-    dir->out_wpos = 0;
+  GNUNET_MQ_dll_insert_tail (&dir->env_head,
+                             &dir->env_tail,
+                             env);
+  cur_buffers++;
+  GNUNET_STATISTICS_set (stats,
+                         "# buffer use",
+                         cur_buffers,
+                         GNUNET_NO);
+  /* Clean up 'rung' if now empty (and not head) */
+  if ( (NULL == rung->rd_head) &&
+       (rung != rung_head) )
+  {
+    GNUNET_CONTAINER_DLL_remove (rung_head,
+                                 rung_tail,
+                                 rung);
+    GNUNET_free (rung);
+  }
 }
 
 
@@ -261,18 +511,26 @@ check_connection_create (void *cls,
 static void
 destroy_direction (struct RouteDirection *dir)
 {
-  for (unsigned int i=0;i<ROUTE_BUFFER_SIZE;i++)
-    if (NULL != dir->out_buffer[i])
-    {
-      GNUNET_MQ_discard (dir->out_buffer[i]);
-      dir->out_buffer[i] = NULL;
-    }
+  struct GNUNET_MQ_Envelope *env;
+
+  while (NULL != (env = dir->env_head))
+  {
+    GNUNET_STATISTICS_update (stats,
+                              "# messages dropped due to route destruction",
+                              1,
+                              GNUNET_NO);
+    discard_buffer (dir,
+                    env);
+  }
   if (NULL != dir->mqm)
   {
     GCP_request_mq_cancel (dir->mqm,
                            NULL);
     dir->mqm = NULL;
   }
+  GNUNET_CONTAINER_DLL_remove (rung_head->rd_head,
+                               rung_head->rd_tail,
+                               dir);
 }
 
 
@@ -289,6 +547,16 @@ destroy_route (struct CadetRoute *route)
        GNUNET_i2s  (GCP_get_id (route->prev.hop)),
        GNUNET_i2s2 (GCP_get_id (route->next.hop)),
        GNUNET_sh2s (&route->cid.connection_of_tunnel));
+  GNUNET_assert (route ==
+                 GNUNET_CONTAINER_heap_remove_node (route->hn));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (routes,
+                                                        &route->cid.connection_of_tunnel,
+                                                        route));
+  GNUNET_STATISTICS_set (stats,
+                         "# routes",
+                         GNUNET_CONTAINER_multishortmap_size (routes),
+                         GNUNET_NO);
   destroy_direction (&route->prev);
   destroy_direction (&route->next);
   GNUNET_free (route);
@@ -328,7 +596,6 @@ send_broken (struct RouteDirection *target,
     bm->peer1 = *peer1;
   if (NULL != peer2)
     bm->peer2 = *peer2;
-
   GCP_request_mq_cancel (target->mqm,
                          env);
   target->mqm = NULL;
@@ -336,6 +603,49 @@ send_broken (struct RouteDirection *target,
 
 
 /**
+ * Function called to check if any routes have timed out, and if
+ * so, to clean them up.  Finally, schedules itself again at the
+ * earliest time where there might be more work.
+ *
+ * @param cls NULL
+ */
+static void
+timeout_cb (void *cls)
+{
+  struct CadetRoute *r;
+  struct GNUNET_TIME_Relative linger;
+  struct GNUNET_TIME_Absolute exp;
+
+  timeout_task = NULL;
+  linger = GNUNET_TIME_relative_multiply (keepalive_period,
+                                          3);
+  while (NULL != (r = GNUNET_CONTAINER_heap_peek (route_heap)))
+  {
+    exp = GNUNET_TIME_absolute_add (r->last_use,
+                                    linger);
+    if (0 != GNUNET_TIME_absolute_get_duration (exp).rel_value_us)
+    {
+      /* Route not yet timed out, wait until it does. */
+      timeout_task = GNUNET_SCHEDULER_add_at (exp,
+                                              &timeout_cb,
+                                              NULL);
+      return;
+    }
+    send_broken (&r->prev,
+                 &r->cid,
+                 NULL,
+                 NULL);
+    send_broken (&r->next,
+                 &r->cid,
+                 NULL,
+                 NULL);
+    destroy_route (r);
+  }
+  /* No more routes left, so no need for a #timeout_task */
+}
+
+
+/**
  * Function called when the message queue to the previous hop
  * becomes available/unavailable.  We expect this function to
  * be called immediately when we register, and then again
@@ -360,12 +670,17 @@ dir_ready_cb (void *cls,
     struct GNUNET_MQ_Envelope *env;
 
     dir->is_ready = GNUNET_YES;
-    if (NULL != (env = dir->out_buffer[dir->out_rpos]))
+    if (NULL != (env = dir->env_head))
     {
-      dir->out_buffer[dir->out_rpos] = NULL;
-      dir->out_rpos++;
-      if (ROUTE_BUFFER_SIZE == dir->out_rpos)
-        dir->out_rpos = 0;
+      GNUNET_MQ_dll_remove (&dir->env_head,
+                            &dir->env_tail,
+                            env);
+      cur_buffers--;
+      GNUNET_STATISTICS_set (stats,
+                             "# buffer use",
+                             cur_buffers,
+                             GNUNET_NO);
+      lower_rung (dir);
       dir->is_ready = GNUNET_NO;
       GCP_send (dir->mqm,
                 env);
@@ -398,11 +713,44 @@ dir_init (struct RouteDirection *dir,
   dir->mqm = GCP_request_mq (hop,
                              &dir_ready_cb,
                              dir);
+  GNUNET_CONTAINER_DLL_insert (rung_head->rd_head,
+                               rung_head->rd_tail,
+                               dir);
+  dir->rung = rung_head;
   GNUNET_assert (GNUNET_YES == dir->is_ready);
 }
 
 
 /**
+ * We could not create the desired route.  Send a
+ * #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
+ * message to @a target.
+ *
+ * @param target who should receive the message
+ * @param cid identifier of the connection/route that failed
+ * @param failure_at neighbour with which we failed to route,
+ *        or NULL.
+ */
+static void
+send_broken_without_mqm (struct CadetPeer *target,
+                         const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                         const struct GNUNET_PeerIdentity *failure_at)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_ConnectionBrokenMessage *bm;
+
+  env = GNUNET_MQ_msg (bm,
+                       GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
+  bm->cid = *cid;
+  bm->peer1 = my_full_id;
+  if (NULL != failure_at)
+    bm->peer2 = *failure_at;
+  GCP_send_ooo (target,
+                env);
+}
+
+
+/**
  * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE
  *
  * @param cls Closure (CadetPeer for neighbor that sent the message).
@@ -419,7 +767,9 @@ handle_connection_create (void *cls,
   uint16_t size = ntohs (msg->header.size) - sizeof (*msg);
   unsigned int path_length;
   unsigned int off;
+  enum GNUNET_CADET_ChannelOption options;
 
+  options = (enum GNUNET_CADET_ChannelOption) ntohl (msg->options);
   path_length = size / sizeof (struct GNUNET_PeerIdentity);
   /* Initiator is at offset 0. */
   for (off=1;off<path_length;off++)
@@ -460,8 +810,7 @@ handle_connection_create (void *cls,
     struct CadetPeerPath *path;
     struct CadetPeer *origin;
 
-    cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                             &msg->cid.connection_of_tunnel);
+    cc = GCC_lookup (&msg->cid);
     if (NULL != cc)
     {
       LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -479,10 +828,25 @@ handle_connection_create (void *cls,
          GNUNET_sh2s (&msg->cid.connection_of_tunnel));
     path = GCPP_get_path_from_route (path_length - 1,
                                      pids);
-    GCT_add_inbound_connection (GCP_get_tunnel (origin,
-                                                GNUNET_YES),
-                                &msg->cid,
-                                path);
+    if (GNUNET_OK !=
+        GCT_add_inbound_connection (GCP_get_tunnel (origin,
+                                                    GNUNET_YES),
+                                    &msg->cid,
+                                    (enum GNUNET_CADET_ChannelOption) ntohl (msg->options),
+                                    path))
+    {
+      /* Send back BROKEN: duplicate connection on the same path,
+         we will use the other one. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Received CADET_CONNECTION_CREATE from %s for %s, but %s already has a connection. Sending BROKEN\n",
+           GCP_2s (sender),
+           GNUNET_sh2s (&msg->cid.connection_of_tunnel),
+           GCPP_2s (path));
+      send_broken_without_mqm (sender,
+                               &msg->cid,
+                               NULL);
+      return;
+    }
     return;
   }
   /* We are merely a hop on the way, check if we can support the route */
@@ -492,22 +856,26 @@ handle_connection_create (void *cls,
        (GNUNET_NO == GCP_has_core_connection (next)) )
   {
     /* unworkable, send back BROKEN notification */
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_ConnectionBrokenMessage *bm;
-
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Received CADET_CONNECTION_CREATE from %s for %s. Next hop %s:%u is down. Sending BROKEN\n",
          GCP_2s (sender),
          GNUNET_sh2s (&msg->cid.connection_of_tunnel),
          GNUNET_i2s (&pids[off + 1]),
          off + 1);
-    env = GNUNET_MQ_msg (bm,
-                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
-    bm->cid = msg->cid;
-    bm->peer1 = pids[off + 1];
-    bm->peer2 = my_full_id;
-    GCP_send_ooo (sender,
-                  env);
+    send_broken_without_mqm (sender,
+                             &msg->cid,
+                             &pids[off + 1]);
+    return;
+  }
+  if (max_routes <= GNUNET_CONTAINER_multishortmap_size (routes))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CADET_CONNECTION_CREATE from %s for %s. We have reached our route limit. Sending BROKEN\n",
+         GCP_2s (sender),
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    send_broken_without_mqm (sender,
+                             &msg->cid,
+                             &pids[off - 1]);
     return;
   }
 
@@ -519,7 +887,9 @@ handle_connection_create (void *cls,
        GNUNET_i2s (&pids[off + 1]),
        off + 1);
   route = GNUNET_new (struct CadetRoute);
+  route->options = options;
   route->cid = msg->cid;
+  route->last_use = GNUNET_TIME_absolute_get ();
   dir_init (&route->prev,
             route,
             sender);
@@ -531,6 +901,18 @@ handle_connection_create (void *cls,
                                                      &route->cid.connection_of_tunnel,
                                                      route,
                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  GNUNET_STATISTICS_set (stats,
+                         "# routes",
+                         GNUNET_CONTAINER_multishortmap_size (routes),
+                         GNUNET_NO);
+  route->hn = GNUNET_CONTAINER_heap_insert (route_heap,
+                                            route,
+                                            route->last_use.abs_value_us);
+  if (NULL == timeout_task)
+    timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (keepalive_period,
+                                                                                3),
+                                                 &timeout_cb,
+                                                 NULL);
 }
 
 
@@ -548,8 +930,7 @@ handle_connection_create_ack (void *cls,
   struct CadetConnection *cc;
 
   /* First, check if ACK belongs to a connection that ends here. */
-  cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                           &msg->cid.connection_of_tunnel);
+  cc = GCC_lookup (&msg->cid);
   if (NULL != cc)
   {
     /* verify ACK came from the right direction */
@@ -593,8 +974,7 @@ handle_connection_broken (void *cls,
   struct CadetRoute *route;
 
   /* First, check if message belongs to a connection that ends here. */
-  cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                           &msg->cid.connection_of_tunnel);
+  cc = GCC_lookup (&msg->cid);
   if (NULL != cc)
   {
     /* verify message came from the right direction */
@@ -611,18 +991,19 @@ handle_connection_broken (void *cls,
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Received CONNECTION_BROKEN for connection %s. Destroying it.\n",
          GNUNET_sh2s (&msg->cid.connection_of_tunnel));
-    GCC_destroy (cc);
+    GCC_destroy_without_core (cc);
 
     /* FIXME: also destroy the path up to the specified link! */
     return;
   }
 
   /* We're just an intermediary peer, route the message along its path */
-  route = get_route (&msg->cid);
   route_message (peer,
                  &msg->cid,
                  &msg->header);
-  destroy_route (route);
+  route = get_route (&msg->cid);
+  if (NULL != route)
+    destroy_route (route);
   /* FIXME: also destroy paths we MAY have up to the specified link! */
 }
 
@@ -642,8 +1023,7 @@ handle_connection_destroy (void *cls,
   struct CadetRoute *route;
 
   /* First, check if message belongs to a connection that ends here. */
-  cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                           &msg->cid.connection_of_tunnel);
+  cc = GCC_lookup (&msg->cid);
   if (NULL != cc)
   {
     /* verify message came from the right direction */
@@ -661,7 +1041,7 @@ handle_connection_destroy (void *cls,
          "Received CONNECTION_DESTROY for connection %s. Destroying connection.\n",
          GNUNET_sh2s (&msg->cid.connection_of_tunnel));
 
-    GCC_destroy (cc);
+    GCC_destroy_without_core (cc);
     return;
   }
 
@@ -669,11 +1049,12 @@ handle_connection_destroy (void *cls,
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Received CONNECTION_DESTROY for connection %s. Destroying route.\n",
        GNUNET_sh2s (&msg->cid.connection_of_tunnel));
-  route = get_route (&msg->cid);
   route_message (peer,
                  &msg->cid,
                  &msg->header);
-  destroy_route (route);
+  route = get_route (&msg->cid);
+  if (NULL != route)
+    destroy_route (route);
 }
 
 
@@ -691,8 +1072,7 @@ handle_tunnel_kx (void *cls,
   struct CadetConnection *cc;
 
   /* First, check if message belongs to a connection that ends here. */
-  cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                           &msg->cid.connection_of_tunnel);
+  cc = GCC_lookup (&msg->cid);
   if (NULL != cc)
   {
     /* verify message came from the right direction */
@@ -719,6 +1099,46 @@ handle_tunnel_kx (void *cls,
 
 
 /**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_tunnel_kx_auth (void *cls,
+                       const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+
+  /* First, check if message belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->kx.cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    struct CadetPeerPath *path = GCC_get_path (cc);
+
+    if (peer !=
+        GCPP_get_peer_at_offset (path,
+                                 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    GCC_handle_kx_auth (cc,
+                        msg);
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer,
+                 &msg->kx.cid,
+                 &msg->kx.header);
+}
+
+
+/**
  * Check if the encrypted message has the appropriate size.
  *
  * @param cls Closure (unused).
@@ -748,8 +1168,7 @@ handle_tunnel_encrypted (void *cls,
   struct CadetConnection *cc;
 
   /* First, check if message belongs to a connection that ends here. */
-  cc = GNUNET_CONTAINER_multishortmap_get (connections,
-                                           &msg->cid.connection_of_tunnel);
+  cc = GCC_lookup (&msg->cid);
   if (NULL != cc)
   {
     /* verify message came from the right direction */
@@ -876,6 +1295,10 @@ GCO_init (const struct GNUNET_CONFIGURATION_Handle *c)
                              GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX,
                              struct GNUNET_CADET_TunnelKeyExchangeMessage,
                              NULL),
+    GNUNET_MQ_hd_fixed_size (tunnel_kx_auth,
+                             GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH,
+                             struct GNUNET_CADET_TunnelKeyExchangeAuthMessage,
+                             NULL),
     GNUNET_MQ_hd_var_size (tunnel_encrypted,
                            GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED,
                            struct GNUNET_CADET_TunnelEncryptedMessage,
@@ -883,8 +1306,21 @@ GCO_init (const struct GNUNET_CONFIGURATION_Handle *c)
     GNUNET_MQ_handler_end ()
   };
 
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "MAX_ROUTES",
+                                             &max_routes))
+    max_routes = 5000;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "MAX_MSGS_QUEUE",
+                                             &max_buffers))
+    max_buffers = 10000;
   routes = GNUNET_CONTAINER_multishortmap_create (1024,
                                                   GNUNET_NO);
+  route_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
   core = GNUNET_CORE_connect (c,
                               NULL,
                               &core_init_cb,
@@ -907,6 +1343,14 @@ GCO_shutdown ()
   }
   GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (routes));
   GNUNET_CONTAINER_multishortmap_destroy (routes);
+  routes = NULL;
+  GNUNET_CONTAINER_heap_destroy (route_heap);
+  route_heap = NULL;
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
 }
 
 /* end of gnunet-cadet-service_core.c */
diff --git a/src/cadet/gnunet-service-cadet-new_core.h b/src/cadet/gnunet-service-cadet_core.h
index 65b0a6ba5..65b0a6ba5 100644
--- a/src/cadet/gnunet-service-cadet-new_core.h
+++ b/src/cadet/gnunet-service-cadet_core.h
diff --git a/src/cadet/gnunet-service-cadet_dht.c b/src/cadet/gnunet-service-cadet_dht.c
index 22673b167..f00c0caf3 100644
--- a/src/cadet/gnunet-service-cadet_dht.c
+++ b/src/cadet/gnunet-service-cadet_dht.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,25 +17,41 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
+/**
+ * @file cadet/gnunet-service-cadet_dht.c
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
-
 #include "gnunet_dht_service.h"
 #include "gnunet_statistics_service.h"
-
-#include "cadet_path.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_dht.h"
-#include "gnunet-service-cadet_peer.h"
 #include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
+/**
+ * How long do we wait before first announcing our presence to the DHT.
+ * Used to wait for our HELLO to be available.  Note that we also get
+ * notifications when our HELLO is ready, so this is just the maximum
+ * we wait for the first notification.
+ */
+#define STARTUP_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 500)
 
+/**
+ * How long do we wait after we get an updated HELLO before publishing?
+ * Allows for the HELLO to be updated again quickly, for example in
+ * case multiple addresses changed and we got a partial update.
+ */
+#define CHANGE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100)
+
+
+#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
 
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
 
 /**
  * Handle for DHT searches.
@@ -47,42 +63,9 @@ struct GCD_search_handle
    */
   struct GNUNET_DHT_GetHandle *dhtget;
 
-  /**
-   * Provided callback to call when a path is found.
-   */
-  GCD_search_callback callback;
-
-  /**
-   * Provided closure.
-   */
-  void *cls;
-
-  /**
-   * Peer ID searched for
-   */
-  GNUNET_PEER_Id peer_id;
 };
 
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Own ID (short value).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
 /**
  * Handle to use DHT.
  */
@@ -94,69 +77,20 @@ static struct GNUNET_DHT_Handle *dht_handle;
 static struct GNUNET_TIME_Relative id_announce_time;
 
 /**
- * DHT replication level, see DHT API: GNUNET_DHT_get_start, GNUNET_DHT_put.
+ * DHT replication level, see DHT API: #GNUNET_DHT_get_start(), #GNUNET_DHT_put().
  */
 static unsigned long long dht_replication_level;
 
 /**
  * Task to periodically announce itself in the network.
  */
-static struct GNUNET_SCHEDULER_Task * announce_id_task;
+static struct GNUNET_SCHEDULER_Task *announce_id_task;
 
 /**
  * Delay for the next ID announce.
  */
 static struct GNUNET_TIME_Relative announce_delay;
 
-/**
- * GET requests to stop on shutdown.
- */
-static struct GNUNET_CONTAINER_MultiHashMap32 *get_requests;
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
-/**
- * Build a PeerPath from the paths returned from the DHT, reversing the paths
- * to obtain a local peer -> destination path and interning the peer ids.
- *
- * @return Newly allocated and created path
- *
- * FIXME refactor and use build_path_from_peer_ids
- */
-static struct CadetPeerPath *
-path_build_from_dht (const struct GNUNET_PeerIdentity *get_path,
-                     unsigned int get_path_length,
-                     const struct GNUNET_PeerIdentity *put_path,
-                     unsigned int put_path_length)
-{
-  size_t size = get_path_length + put_path_length + 1;
-  struct GNUNET_PeerIdentity peers[size];
-  const struct GNUNET_PeerIdentity *peer;
-  struct CadetPeerPath *p;
-  unsigned int own_pos;
-  int i;
-
-  peers[0] = my_full_id;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   GET has %d hops.\n", get_path_length);
-  for (i = 0 ; i < get_path_length; i++)
-  {
-    peer = &get_path[get_path_length - i - 1];
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   From GET: %s\n", GNUNET_i2s (peer));
-    peers[i + 1] = *peer;
-  }
-  for (i = 0 ; i < put_path_length; i++)
-  {
-    peer = &put_path[put_path_length - i - 1];
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   From PUT: %s\n", GNUNET_i2s (peer));
-    peers[i + get_path_length + 1] = *peer;
-  }
-  p = path_build_from_peer_ids (peers, size, myid, &own_pos);
-  return p;
-}
-
 
 /**
  * Function to process paths received for a new peer addition. The recorded
@@ -176,42 +110,34 @@ path_build_from_dht (const struct GNUNET_PeerIdentity *get_path,
  */
 static void
 dht_get_id_handler (void *cls, struct GNUNET_TIME_Absolute exp,
-                    const struct GNUNET_HashCode * key,
+                    const struct GNUNET_HashCode *key,
                     const struct GNUNET_PeerIdentity *get_path,
                     unsigned int get_path_length,
                     const struct GNUNET_PeerIdentity *put_path,
-                    unsigned int put_path_length, enum GNUNET_BLOCK_Type type,
-                    size_t size, const void *data)
+                    unsigned int put_path_length,
+                    enum GNUNET_BLOCK_Type type,
+                    size_t size,
+                    const void *data)
 {
-  struct GCD_search_handle *h = cls;
-  struct GNUNET_HELLO_Message *hello;
-  struct CadetPeerPath *p;
+  const struct GNUNET_HELLO_Message *hello = data;
   struct CadetPeer *peer;
-  char *s;
 
-  p = path_build_from_dht (get_path, get_path_length,
-                           put_path, put_path_length);
-  if (NULL == p)
+  GCPP_try_path_from_dht (get_path,
+                          get_path_length,
+                          put_path,
+                          put_path_length);
+  if ( (size >= sizeof (struct GNUNET_HELLO_Message)) &&
+       (ntohs (hello->header.size) == size) &&
+       (size == GNUNET_HELLO_size (hello)) )
   {
-    GNUNET_break_op (0);
-    return;
+    peer = GCP_get (&put_path[0],
+                    GNUNET_YES);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got HELLO for %s\n",
+         GCP_2s (peer));
+    GCP_set_hello (peer,
+                   hello);
   }
-
-  s = path_2s (p);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Got path from DHT: %s\n",
-       s);
-  GNUNET_free_non_null (s);
-
-  peer = GCP_get_short (p->peers[p->length - 1], GNUNET_YES);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got HELLO for %s\n",
-       GCP_2s (peer));
-  h->callback (h->cls, p);
-  path_destroy (p);
-  hello = (struct GNUNET_HELLO_Message *) data;
-  GCP_set_hello (peer, hello);
-  GCP_try_connect (peer);
 }
 
 
@@ -229,19 +155,10 @@ announce_id (void *cls)
   struct GNUNET_TIME_Absolute expiration;
   struct GNUNET_TIME_Relative next_put;
 
-  announce_id_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Announce ID\n");
   hello = GCH_get_mine ();
   size = (NULL != hello) ? GNUNET_HELLO_size (hello) : 0;
-  if ( (NULL == hello) || (0 == size) )
+  if (0 == size)
   {
-    /* Peerinfo gave us no hello yet, try again soon. */
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "  no hello, waiting!\n");
-    GNUNET_STATISTICS_update (stats,
-                              "# DHT announce skipped (no hello)",
-                              1,
-                              GNUNET_NO);
     expiration = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
                                            announce_delay);
     announce_delay = GNUNET_TIME_STD_BACKOFF (announce_delay);
@@ -252,71 +169,64 @@ announce_id (void *cls)
     announce_delay = GNUNET_TIME_UNIT_SECONDS;
   }
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Hello %p size: %u\n",
-       hello,
-       size);
-  if (NULL != hello)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# DHT announce",
-                              1, GNUNET_NO);
-    memset (&phash,
-            0,
-            sizeof (phash));
-    GNUNET_memcpy (&phash,
-                   &my_full_id,
-                   sizeof (my_full_id));
-    GNUNET_DHT_put (dht_handle,   /* DHT handle */
-                    &phash,       /* Key to use */
-                    dht_replication_level,     /* Replication level */
-                    GNUNET_DHT_RO_RECORD_ROUTE
-                    | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
-                    GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
-                    size,  /* Size of the data */
-                    (const char *) hello, /* Data itself */
-                    expiration,  /* Data expiration */
-                    NULL,         /* Continuation */
-                    NULL);        /* Continuation closure */
-  }
   /* Call again in id_announce_time, unless HELLO expires first,
    * but wait at least 1s. */
-  next_put = GNUNET_TIME_absolute_get_remaining (expiration);
-  next_put = GNUNET_TIME_relative_min (next_put,
-                                       id_announce_time);
-  next_put = GNUNET_TIME_relative_max (next_put,
-                                       GNUNET_TIME_UNIT_SECONDS);
-  announce_id_task = GNUNET_SCHEDULER_add_delayed (next_put,
-                                                   &announce_id,
-                                                   cls);
+  next_put
+    = GNUNET_TIME_absolute_get_remaining (expiration);
+  next_put
+    = GNUNET_TIME_relative_min (next_put,
+                                id_announce_time);
+  next_put
+    = GNUNET_TIME_relative_max (next_put,
+                                GNUNET_TIME_UNIT_SECONDS);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (next_put,
+                                    &announce_id,
+                                    cls);
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT announce",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof (phash));
+  GNUNET_memcpy (&phash,
+                 &my_full_id,
+                 sizeof (my_full_id));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Announcing my HELLO (%u bytes) in the DHT\n",
+       size);
+  GNUNET_DHT_put (dht_handle,   /* DHT handle */
+                  &phash,       /* Key to use */
+                  dht_replication_level,     /* Replication level */
+                  GNUNET_DHT_RO_RECORD_ROUTE
+                  | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
+                  GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
+                  size,  /* Size of the data */
+                  (const char *) hello, /* Data itself */
+                  expiration,  /* Data expiration */
+                  NULL,         /* Continuation */
+                  NULL);        /* Continuation closure */
 }
 
+
 /**
- * Iterator over hash map entries and stop GET requests before disconnecting
- * from the DHT.
- *
- * @param cls Closure (unused)
- * @param key Current peer ID.
- * @param value Value in the hash map (GCD_search_handle).
- *
- * @return #GNUNET_YES, we should continue to iterate,
+ * Function called by the HELLO subsystem whenever OUR hello
+ * changes. Re-triggers the DHT PUT immediately.
  */
-int
-stop_get (void *cls,
-          uint32_t key,
-          void *value)
+void
+GCD_hello_update ()
 {
-  struct GCD_search_handle *h = value;
-
-  GCD_search_stop (h);
-  return GNUNET_YES;
+  if (NULL == announce_id_task)
+    return; /* too early */
+  GNUNET_SCHEDULER_cancel (announce_id_task);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (CHANGE_DELAY,
+                                    &announce_id,
+                                    NULL);
 }
 
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
 /**
  * Initialize the DHT subsystem.
  *
@@ -325,36 +235,40 @@ stop_get (void *cls,
 void
 GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET",
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
                                              "DHT_REPLICATION_LEVEL",
                                              &dht_replication_level))
   {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING, "CADET",
-                               "DHT_REPLICATION_LEVEL", "USING DEFAULT");
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "DHT_REPLICATION_LEVEL",
+                               "USING DEFAULT");
     dht_replication_level = 3;
   }
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c, "CADET", "ID_ANNOUNCE_TIME",
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "ID_ANNOUNCE_TIME",
                                            &id_announce_time))
   {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, "CADET",
-                               "ID_ANNOUNCE_TIME", "MISSING");
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
+                               "CADET",
+                               "ID_ANNOUNCE_TIME",
+                               "MISSING");
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
 
-  dht_handle = GNUNET_DHT_connect (c, 64);
-  if (NULL == dht_handle)
-  {
-    GNUNET_break (0);
-  }
-
+  dht_handle = GNUNET_DHT_connect (c,
+                                   64);
+  GNUNET_break (NULL != dht_handle);
   announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  announce_id_task = GNUNET_SCHEDULER_add_now (&announce_id, NULL);
-  get_requests = GNUNET_CONTAINER_multihashmap32_create (32);
+  announce_id_task = GNUNET_SCHEDULER_add_delayed (STARTUP_DELAY,
+                                                   &announce_id,
+                                                   NULL);
 }
 
 
@@ -364,10 +278,7 @@ GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
 void
 GCD_shutdown (void)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down DHT\n");
-  GNUNET_CONTAINER_multihashmap32_iterate (get_requests, &stop_get, NULL);
-  GNUNET_CONTAINER_multihashmap32_destroy (get_requests);
-  if (dht_handle != NULL)
+  if (NULL != dht_handle)
   {
     GNUNET_DHT_disconnect (dht_handle);
     dht_handle = NULL;
@@ -379,22 +290,31 @@ GCD_shutdown (void)
   }
 }
 
+
+/**
+ * Search DHT for paths to @a peeR_id
+ *
+ * @param peer_id peer to search for
+ * @return handle to abort search
+ */
 struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id,
-            GCD_search_callback callback, void *cls)
+GCD_search (const struct GNUNET_PeerIdentity *peer_id)
 {
   struct GNUNET_HashCode phash;
   struct GCD_search_handle *h;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Starting DHT GET for peer %s\n",
-       GNUNET_i2s (peer_id));
-  GNUNET_STATISTICS_update (stats, "# DHT search", 1, GNUNET_NO);
-  memset (&phash, 0, sizeof (phash));
-  GNUNET_memcpy (&phash, peer_id, sizeof (*peer_id));
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT search",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof (phash));
+  GNUNET_memcpy (&phash,
+                 peer_id,
+                 sizeof (*peer_id));
+
   h = GNUNET_new (struct GCD_search_handle);
-  h->peer_id = GNUNET_PEER_intern (peer_id);
-  h->callback = callback;
-  h->cls = cls;
   h->dhtget = GNUNET_DHT_get_start (dht_handle,    /* handle */
                                     GNUNET_BLOCK_TYPE_DHT_HELLO, /* type */
                                     &phash,     /* key to search */
@@ -405,20 +325,27 @@ GCD_search (const struct GNUNET_PeerIdentity *peer_id,
                                     0,     /* xquery bits */
                                     &dht_get_id_handler,
                                     h);
-  GNUNET_CONTAINER_multihashmap32_put (get_requests,
-                                       h->peer_id,
-                                       h,
-                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Starting DHT GET for peer %s (%p)\n",
+       GNUNET_i2s (peer_id),
+       h);
   return h;
 }
 
 
+/**
+ * Stop DHT search started with #GCD_search().
+ *
+ * @param h handle to search to stop
+ */
 void
 GCD_search_stop (struct GCD_search_handle *h)
 {
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONTAINER_multihashmap32_remove (get_requests,
-                                                        h->peer_id, h));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Stopping DHT GET %p\n",
+       h);
   GNUNET_DHT_get_stop (h->dhtget);
   GNUNET_free (h);
 }
+
+/* end of gnunet-service-cadet_dht.c */
diff --git a/src/cadet/gnunet-service-cadet_dht.h b/src/cadet/gnunet-service-cadet_dht.h
index b70dfe975..5d7ab29a0 100644
--- a/src/cadet/gnunet-service-cadet_dht.h
+++ b/src/cadet/gnunet-service-cadet_dht.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,10 +22,10 @@
  * @file cadet/gnunet-service-cadet_dht.h
  * @brief cadet service; dealing with DHT requests and results
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  *
- * All functions in this file should use the prefix GMD (Gnunet Cadet Dht)
+ * All functions in this file should use the prefix GCD (Gnunet Cadet Dht)
  */
-
 #ifndef GNUNET_SERVICE_CADET_DHT_H
 #define GNUNET_SERVICE_CADET_DHT_H
 
@@ -40,23 +40,11 @@ extern "C"
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-struct GCD_search_handle;
-
-
 /**
- * Callback called on each path found over the DHT.
- *
- * @param cls Closure.
- * @param path An unchecked, unoptimized path to the target node.
- *             After callback will no longer be valid!
+ * Handle for DHT search operation.
  */
-typedef void
-(*GCD_search_callback) (void *cls,
-                        const struct CadetPeerPath *path);
+struct GCD_search_handle;
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
 
 /**
  * Initialize the DHT subsystem.
@@ -66,6 +54,7 @@ typedef void
 void
 GCD_init (const struct GNUNET_CONFIGURATION_Handle *c);
 
+
 /**
  * Shut down the DHT subsystem.
  */
@@ -73,14 +62,32 @@ void
 GCD_shutdown (void);
 
 
+/**
+ * Function called by the HELLO subsystem whenever OUR hello
+ * changes. Re-triggers the DHT PUT immediately.
+ */
+void
+GCD_hello_update (void);
+
+/**
+ * Search DHT for paths to @a peeR_id
+ *
+ * @param peer_id peer to search for
+ * @return handle to abort search
+ */
 struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id,
-            GCD_search_callback callback, void *cls);
+GCD_search (const struct GNUNET_PeerIdentity *peer_id);
 
 
+/**
+ * Stop DHT search started with #GCD_search().
+ *
+ * @param h handle to search to stop
+ */
 void
 GCD_search_stop (struct GCD_search_handle *h);
 
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif
@@ -88,6 +95,6 @@ GCD_search_stop (struct GCD_search_handle *h);
 }
 #endif
 
-/* ifndef GNUNET_CADET_SERVICE_LOCAL_H */
+/* ifndef GNUNET_CADET_SERVICE_DHT_H */
 #endif
-/* end of gnunet-cadet-service_LOCAL.h */
+/* end of gnunet-service-cadet_dht.h */
diff --git a/src/cadet/gnunet-service-cadet_hello.c b/src/cadet/gnunet-service-cadet_hello.c
index 3c63f3551..6d85de39f 100644
--- a/src/cadet/gnunet-service-cadet_hello.c
+++ b/src/cadet/gnunet-service-cadet_hello.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2014 GNUnet e.V.
+     Copyright (C) 2014, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,58 +17,33 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
+/**
+ * @file cadet/gnunet-service-cadet_hello.c
+ * @brief spread knowledge about how to contact other peers from PEERINFO
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - is most of this necessary/helpful?
+ * - should we not simply restrict this to OUR hello?
+ */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
 #include "gnunet_statistics_service.h"
 #include "gnunet_peerinfo_service.h"
-
 #include "cadet_protocol.h"
-#include "cadet_path.h"
-
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_dht.h"
 #include "gnunet-service-cadet_hello.h"
 #include "gnunet-service-cadet_peer.h"
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-hll",__VA_ARGS__)
 
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-
-/**
- * Don't try to recover tunnels if shutting down.
- */
-extern int shutting_down;
-
-
 /**
  * Hello message of local peer.
  */
-const struct GNUNET_HELLO_Message *mine;
+static struct GNUNET_HELLO_Message *mine;
 
 /**
  * Handle to peerinfo service.
@@ -78,13 +53,9 @@ static struct GNUNET_PEERINFO_Handle *peerinfo;
 /**
  * Iterator context.
  */
-struct GNUNET_PEERINFO_NotifyContext* nc;
+static struct GNUNET_PEERINFO_NotifyContext *nc;
 
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
 /**
  * Process each hello message received from peerinfo.
  *
@@ -94,31 +65,37 @@ struct GNUNET_PEERINFO_NotifyContext* nc;
  * @param err_msg Error message.
  */
 static void
-got_hello (void *cls, const struct GNUNET_PeerIdentity *id,
+got_hello (void *cls,
+           const struct GNUNET_PeerIdentity *id,
            const struct GNUNET_HELLO_Message *hello,
            const char *err_msg)
 {
   struct CadetPeer *peer;
 
-  if (NULL == id || NULL == hello)
+  if ( (NULL == id) ||
+       (NULL == hello) )
+    return;
+  if (0 == memcmp (id,
+                   &my_full_id,
+                   sizeof (struct GNUNET_PeerIdentity)))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " hello with id %p and msg %p\n", id, hello);
+    GNUNET_free_non_null (mine);
+    mine = (struct GNUNET_HELLO_Message *) GNUNET_copy_message (&hello->header);
+    GCD_hello_update ();
     return;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " hello for %s (%d bytes), expires on %s\n",
-       GNUNET_i2s (id), GNUNET_HELLO_size (hello),
-       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration(hello)));
-  peer = GCP_get (id, GNUNET_YES);
-  GCP_set_hello (peer, hello);
-
-  if (GCP_get_short_id (peer) == myid)
-    mine = GCP_get_hello (peer);
-}
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Hello for %s (%d bytes), expires on %s\n",
+       GNUNET_i2s (id),
+       GNUNET_HELLO_size (hello),
+       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration (hello)));
+  peer = GCP_get (id,
+                  GNUNET_YES);
+  GCP_set_hello (peer,
+                 hello);
+}
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
 
 /**
  * Initialize the hello subsystem.
@@ -128,10 +105,12 @@ got_hello (void *cls, const struct GNUNET_PeerIdentity *id,
 void
 GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
   GNUNET_assert (NULL == nc);
   peerinfo = GNUNET_PEERINFO_connect (c);
-  nc = GNUNET_PEERINFO_notify (c, GNUNET_NO, &got_hello, NULL);
+  nc = GNUNET_PEERINFO_notify (c,
+                               GNUNET_NO,
+                               &got_hello,
+                               NULL);
 }
 
 
@@ -141,7 +120,6 @@ GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
 void
 GCH_shutdown ()
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down channels\n");
   if (NULL != nc)
   {
     GNUNET_PEERINFO_notify_cancel (nc);
@@ -152,6 +130,11 @@ GCH_shutdown ()
     GNUNET_PEERINFO_disconnect (peerinfo);
     peerinfo = NULL;
   }
+  if (NULL != mine)
+  {
+    GNUNET_free (mine);
+    mine = NULL;
+  }
 }
 
 
@@ -166,35 +149,4 @@ GCH_get_mine (void)
   return mine;
 }
 
-
-/**
- * Get another peer's hello message.
- *
- * @param id ID of the peer whose hello message is requested.
- *
- * @return Hello message, if any (NULL possible).
- */
-const struct GNUNET_HELLO_Message *
-GCH_get (const struct GNUNET_PeerIdentity *id)
-{
-  struct CadetPeer *p;
-
-  p = GCP_get (id, GNUNET_NO);
-  if (NULL == p)
-    return NULL;
-  return GCP_get_hello (p);
-}
-
-
-/**
- * Convert a hello message to a string.
- *
- * @param h Hello message.
- */
-char *
-GCH_2s (const struct GNUNET_HELLO_Message *h)
-{
-  return "hello (TODO)";
-}
-
-
+/* end of gnunet-service-cadet-new_hello.c */
diff --git a/src/cadet/gnunet-service-cadet_hello.h b/src/cadet/gnunet-service-cadet_hello.h
index 34121e1e0..4291ae985 100644
--- a/src/cadet/gnunet-service-cadet_hello.h
+++ b/src/cadet/gnunet-service-cadet_hello.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2014 GNUnet e.V.
+     Copyright (C) 2014, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,8 +22,9 @@
  * @file cadet/gnunet-service-cadet_hello.h
  * @brief cadet service; dealing with hello messages
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  *
- * All functions in this file should use the prefix GMH (Gnunet Cadet Hello)
+ * All functions in this file should use the prefix GCH (Gnunet Cadet Hello)
  */
 
 #ifndef GNUNET_SERVICE_CADET_HELLO_H
diff --git a/src/cadet/gnunet-service-cadet_local.c b/src/cadet/gnunet-service-cadet_local.c
deleted file mode 100644
index dea6681df..000000000
--- a/src/cadet/gnunet-service-cadet_local.c
+++ /dev/null
@@ -1,1553 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#include "gnunet_statistics_service.h"
-
-#include "cadet.h"
-#include "cadet_protocol.h" /* GNUNET_CADET_Data is shared */
-
-#include "gnunet-service-cadet_local.h"
-#include "gnunet-service-cadet_channel.h"
-
-/* INFO DEBUG */
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-loc",__VA_ARGS__)
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-/**
- * Struct containing information about a client of the service
- *
- * TODO: add a list of 'waiting' ports
- */
-struct CadetClient
-{
-  /**
-   * Linked list next
-   */
-  struct CadetClient *next;
-
-  /**
-   * Linked list prev
-   */
-  struct CadetClient *prev;
-
-  /**
-   * Tunnels that belong to this client, indexed by local id
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *own_channels;
-
-  /**
-   * Tunnels this client has accepted, indexed by incoming local id
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *incoming_channels;
-
-  /**
-   * Channel ID for the next incoming channel.
-   */
-  struct GNUNET_CADET_ClientChannelNumber next_ccn;
-
-  /**
-   * Handle to communicate with the client
-   */
-  struct GNUNET_SERVER_Client *handle;
-
-  /**
-   * Ports that this client has declared interest in.
-   * Indexed by port, contains *Client.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-  /**
-   * Whether the client is active or shutting down (don't send confirmations
-   * to a client that is shutting down.
-   */
-  int shutting_down;
-
-  /**
-   * ID of the client, mainly for debug messages
-   */
-  unsigned int id;
-};
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Handle to server lib.
- */
-static struct GNUNET_SERVER_Handle *server_handle;
-
-/**
- * DLL with all the clients, head.
- */
-static struct CadetClient *clients_head;
-
-/**
- * DLL with all the clients, tail.
- */
-static struct CadetClient *clients_tail;
-
-/**
- * Next ID to assign to a client.
- */
-unsigned int next_client_id;
-
-/**
- * All ports clients of this peer have opened.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-/**
- * Notification context, to send messages to local clients.
- */
-static struct GNUNET_SERVER_NotificationContext *nc;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-/**
- * Remove client's ports from the global hashmap on disconnect.
- *
- * @param cls Closure (unused).
- * @param key Port.
- * @param value Client structure.
- *
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-client_release_ports (void *cls,
-                      const struct GNUNET_HashCode *key,
-                      void *value)
-{
-  int res;
-
-  res = GNUNET_CONTAINER_multihashmap_remove (ports, key, value);
-  if (GNUNET_YES != res)
-  {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Port %s by client %p was not registered.\n",
-         GNUNET_h2s (key), value);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Iterator for deleting each channel whose client endpoint disconnected.
- *
- * @param cls Closure (client that has disconnected).
- * @param key The local channel id (used to access the hashmap).
- * @param value The value stored at the key (channel to destroy).
- *
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-channel_destroy_iterator (void *cls,
-                          uint32_t key,
-                          void *value)
-{
-  struct CadetChannel *ch = value;
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " Channel %s destroy, due to client %s shutdown.\n",
-       GCCH_2s (ch), GML_2s (c));
-
-  GCCH_handle_local_destroy (ch,
-                             c,
-                             key < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  return GNUNET_OK;
-}
-
-
-/**
- * Unregister data and free memory for a client.
- *
- * @param c Client to destroy. No longer valid after call.
- */
-static void
-client_destroy (struct CadetClient *c)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  client destroy: %p/%u\n", c, c->id);
-  GNUNET_SERVER_client_drop (c->handle);
-  c->shutting_down = GNUNET_YES;
-
-  if (NULL != c->own_channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->own_channels,
-                                             &channel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap32_destroy (c->own_channels);
-  }
-  if (NULL != c->incoming_channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->incoming_channels,
-                                             &channel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap32_destroy (c->incoming_channels);
-  }
-  if (NULL != c->ports)
-  {
-    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
-                                           &client_release_ports, c);
-    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
-  }
-
-  GNUNET_CONTAINER_DLL_remove (clients_head, clients_tail, c);
-  GNUNET_STATISTICS_update (stats, "# clients", -1, GNUNET_NO);
-  GNUNET_SERVER_client_set_user_context (c->handle, NULL);
-  GNUNET_free (c);
-}
-
-
-/**
- * Create a client record, register data and initialize memory.
- *
- * @param client Client's handle.
- */
-static struct CadetClient *
-client_new (struct GNUNET_SERVER_Client *client)
-{
-  struct CadetClient *c;
-
-  GNUNET_SERVER_client_keep (client);
-  GNUNET_SERVER_notification_context_add (nc, client);
-
-  c = GNUNET_new (struct CadetClient);
-  c->handle = client;
-  c->id = next_client_id++; /* overflow not important: just for debug */
-
-  c->own_channels = GNUNET_CONTAINER_multihashmap32_create (32);
-  c->incoming_channels = GNUNET_CONTAINER_multihashmap32_create (32);
-
-  GNUNET_SERVER_client_set_user_context (client, c);
-  GNUNET_CONTAINER_DLL_insert (clients_head, clients_tail, c);
-  GNUNET_STATISTICS_update (stats, "# clients", +1, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  client created: %p/%u\n", c, c->id);
-
-  return c;
-}
-
-
-/******************************************************************************/
-/********************************  HANDLES  ***********************************/
-/******************************************************************************/
-
-/**
- * Handler for client connection.
- *
- * @param cls Closure (unused).
- * @param client Client handler.
- */
-static void
-handle_client_connect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Client connected: %p\n", client);
-  if (NULL == client)
-    return;
-
-  (void) client_new (client);
-}
-
-
-/**
- * Handler for client disconnection
- *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
- */
-static void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Client disconnected: %p\n", client);
-
-  c = GML_client_get (client);
-  if (NULL != c)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "matching client found (%u, %p)\n",
-                c->id, c);
-    client_destroy (c);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " disconnecting client's context NULL\n");
-  }
-  return;
-}
-
-
-/**
- * Handler for port open requests.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_port_open (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_CADET_PortMessage *pmsg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "open port requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_PortMessage) != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  pmsg = (struct GNUNET_CADET_PortMessage *) message;
-  if (NULL == c->ports)
-  {
-    c->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_NO);
-  }
-  /* store in client's hashmap */
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (c->ports, &pmsg->port, c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  /* store in global hashmap */
-  /* FIXME only allow one client to have the port open,
-   *       have a backup hashmap with waiting clients */
-  GNUNET_CONTAINER_multihashmap_put (ports, &pmsg->port, c,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for port close requests.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_port_close (void *cls, struct GNUNET_SERVER_Client *client,
-                   const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_CADET_PortMessage *pmsg;
-  int removed;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "close port requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-    /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_PortMessage) != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  pmsg = (struct GNUNET_CADET_PortMessage *) message;
-  removed = GNUNET_CONTAINER_multihashmap_remove (c->ports, &pmsg->port, c);
-  GNUNET_break_op (GNUNET_YES == removed);
-  removed = GNUNET_CONTAINER_multihashmap_remove (ports, &pmsg->port, c);
-  GNUNET_break_op (GNUNET_YES == removed);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for requests of new channels.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_channel_create (void *cls, struct GNUNET_SERVER_Client *client,
-                       const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "new channel requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_LocalChannelCreateMessage)
-      != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GCCH_handle_local_create (c,
-                                (struct GNUNET_CADET_LocalChannelCreateMessage *)
-                                message))
-  {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for requests of deleting tunnels
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_channel_destroy (void *cls, struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
-  struct CadetClient *c;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got a DESTROY CHANNEL from client!\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message sanity check */
-  if (sizeof (struct GNUNET_CADET_LocalChannelDestroyMessage)
-      != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (const struct GNUNET_CADET_LocalChannelDestroyMessage *) message;
-
-  /* Retrieve tunnel */
-  ccn = msg->ccn;
-  ch = GML_channel_get (c, ccn);
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Client %u is destroying channel %X\n",
-       c->id, ccn);
-
-  if (NULL == ch)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "  channel %X not found\n", ccn);
-    GNUNET_STATISTICS_update (stats,
-                              "# client destroy messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  GCCH_handle_local_destroy (ch,
-                             c,
-                             ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client traffic
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_data (void *cls, struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_MessageHeader *payload;
-  struct GNUNET_CADET_LocalData *msg;
-  struct CadetClient *c;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  size_t message_size;
-  size_t payload_size;
-  size_t payload_claimed_size;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got data from a client\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Sanity check for message size */
-  message_size = ntohs (message->size);
-  if (sizeof (struct GNUNET_CADET_LocalData)
-      + sizeof (struct GNUNET_MessageHeader) > message_size
-      || GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < message_size)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Sanity check for payload size */
-  payload_size = message_size - sizeof (struct GNUNET_CADET_LocalData);
-  msg = (struct GNUNET_CADET_LocalData *) message;
-  payload = (struct GNUNET_MessageHeader *) &msg[1];
-  payload_claimed_size = ntohs (payload->size);
-  if (sizeof (struct GNUNET_MessageHeader) > payload_claimed_size
-      || GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size
-      || payload_claimed_size > payload_size)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "client claims to send %u bytes in %u payload\n",
-         payload_claimed_size, payload_size);
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  %u bytes (%u payload) by client %u\n",
-       payload_size, payload_claimed_size, c->id);
-
-  /* Channel exists? */
-  fwd = ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-  ch = GML_channel_get (c, ccn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# client data messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  if (GNUNET_OK != GCCH_handle_local_data (ch, c, fwd, payload, payload_size))
-  {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "receive done OK\n");
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  return;
-}
-
-
-/**
- * Handler for client's ACKs for payload traffic.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_ack (void *cls, struct GNUNET_SERVER_Client *client,
-            const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_CADET_LocalAck *msg;
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got a local ACK\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  msg = (struct GNUNET_CADET_LocalAck *) message;
-
-  /* Channel exists? */
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  on channel %X\n",
-       ntohl (ccn.channel_of_client));
-  ch = GML_channel_get (c, ccn);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   -- ch %p\n", ch);
-  if (NULL == ch)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %X unknown.\n",
-         ntohl (ccn.channel_of_client));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  for client %u.\n", c->id);
-    GNUNET_STATISTICS_update (stats,
-                              "# client ack messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  /* If client is root, the ACK is going FWD, therefore this is "BCK ACK". */
-  /* If client is dest, the ACK is going BCK, therefore this is "FWD ACK" */
-  fwd = ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-
-  GCCH_handle_local_ack (ch, fwd);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Iterator over all peers to send a monitoring client info about each peer.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_peers_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity * peer,
-                        void *value)
-{
-  struct GNUNET_SERVER_Client *client = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_CADET_LocalInfoPeer msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  msg.destination = *peer;
-  msg.paths = htons (GCP_count_paths (p));
-  msg.tunnel = htons (NULL != GCP_get_tunnel (p));
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "sending info about peer %s\n",
-       GNUNET_i2s (peer));
-
-  GNUNET_SERVER_notification_context_unicast (nc, client,
-                                              &msg.header, GNUNET_NO);
-  return GNUNET_YES;
-}
-
-
-/**
- * Iterator over all peers to dump info for each peer.
- *
- * @param cls Closure (unused).
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-show_peer_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity * peer,
-                        void *value)
-{
-  struct CadetPeer *p = value;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p);
-  if (NULL != t)
-    GCT_debug (t, GNUNET_ERROR_TYPE_ERROR);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Iterator over all paths of a peer to build an InfoPeer message.
- *
- * Message contains blocks of peers, first not included.
- *
- * @param cls Closure (message to build).
- * @param peer Peer this path is towards.
- * @param path Path itself
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-static int
-path_info_iterator (void *cls,
-                    struct CadetPeer *peer,
-                    struct CadetPeerPath *path)
-{
-  struct GNUNET_CADET_LocalInfoPeer *resp = cls;
-  struct GNUNET_PeerIdentity *id;
-  uint16_t msg_size;
-  uint16_t path_size;
-  unsigned int i;
-
-  msg_size = ntohs (resp->header.size);
-  path_size = sizeof (struct GNUNET_PeerIdentity) * (path->length - 1);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Info Path %u\n", path->length);
-  if (msg_size + path_size > UINT16_MAX)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "path too long for info message\n");
-    return GNUNET_NO;
-  }
-
-  i = msg_size - sizeof (struct GNUNET_CADET_LocalInfoPeer);
-  i = i / sizeof (struct GNUNET_PeerIdentity);
-
-  /* Set id to the address of the first free peer slot. */
-  id = (struct GNUNET_PeerIdentity *) &resp[1];
-  id = &id[i];
-
-  /* Don't copy first peers.
-   * First peer is always the local one.
-   * Last peer is always the destination (leave as 0, EOL).
-   */
-  for (i = 0; i < path->length - 1; i++)
-  {
-    GNUNET_PEER_resolve (path->peers[i + 1], &id[i]);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " %s\n", GNUNET_i2s (&id[i]));
-  }
-
-  resp->header.size = htons (msg_size + path_size);
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO PEERS request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_peers (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader reply;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received get peers request from client %u (%p)\n",
-       c->id, client);
-
-  GCP_iterate_all (get_all_peers_iterator, client);
-  reply.size = htons (sizeof (reply));
-  reply.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  GNUNET_SERVER_notification_context_unicast (nc, client, &reply, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Get peers request from client %u completed\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client's SHOW_PEER request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_show_peer (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_CADET_LocalInfoPeer *resp;
-  struct CadetPeer *p;
-  struct CadetClient *c;
-  unsigned char cbuf[64 * 1024];
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (struct GNUNET_CADET_LocalInfo *) message;
-  resp = (struct GNUNET_CADET_LocalInfoPeer *) cbuf;
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Received peer info request from client %u for peer %s\n",
-       c->id, GNUNET_i2s_full (&msg->peer));
-
-  resp->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
-  resp->header.size = htons (sizeof (struct GNUNET_CADET_LocalInfoPeer));
-  resp->destination = msg->peer;
-  p = GCP_get (&msg->peer, GNUNET_NO);
-  if (NULL == p)
-  {
-    /* We don't know the peer */
-
-    LOG (GNUNET_ERROR_TYPE_INFO, "Peer %s unknown\n",
-         GNUNET_i2s_full (&msg->peer));
-    resp->paths = htons (0);
-    resp->tunnel = htons (NULL != GCP_get_tunnel (p));
-
-    GNUNET_SERVER_notification_context_unicast (nc, client,
-                                                &resp->header,
-                                                GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  resp->paths = htons (GCP_count_paths (p));
-  resp->tunnel = htons (NULL != GCP_get_tunnel (p));
-  GCP_iterate_paths (p, &path_info_iterator, resp);
-
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &resp->header, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Show peer from client %u completed.\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Iterator over all tunnels to send a monitoring client info about each tunnel.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Tunnel info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_tunnels_iterator (void *cls,
-                          const struct GNUNET_PeerIdentity * peer,
-                          void *value)
-{
-  struct GNUNET_SERVER_Client *client = cls;
-  struct CadetTunnel *t = value;
-  struct GNUNET_CADET_LocalInfoTunnel msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  msg.destination = *peer;
-  msg.channels = htonl (GCT_count_channels (t));
-  msg.connections = htonl (GCT_count_any_connections (t));
-  msg.cstate = htons ((uint16_t) GCT_get_cstate (t));
-  msg.estate = htons ((uint16_t) GCT_get_estate (t));
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "sending info about tunnel ->%s\n",
-       GNUNET_i2s (peer));
-
-  GNUNET_SERVER_notification_context_unicast (nc, client,
-                                              &msg.header, GNUNET_NO);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO TUNNELS request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_tunnels (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader reply;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received get tunnels request from client %u (%p)\n",
-       c->id, client);
-
-  GCT_iterate_all (get_all_tunnels_iterator, client);
-  reply.size = htons (sizeof (reply));
-  reply.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  GNUNET_SERVER_notification_context_unicast (nc, client, &reply, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Get tunnels request from client %u completed\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-static void
-iter_connection (void *cls, struct CadetConnection *c)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
-
-  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  h[msg->connections] = *(GCC_get_id (c));
-  msg->connections++;
-}
-
-static void
-iter_channel (void *cls, struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  struct GNUNET_CADET_ChannelTunnelNumber *chn = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
-
-  chn[msg->channels] = GCCH_get_id (ch);
-  msg->channels++;
-}
-
-
-/**
- * Handler for client's SHOW_TUNNEL request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_show_tunnel (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_CADET_LocalInfoTunnel *resp;
-  struct CadetClient *c;
-  struct CadetTunnel *t;
-  unsigned int ch_n;
-  unsigned int c_n;
-  size_t size;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (struct GNUNET_CADET_LocalInfo *) message;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received tunnel info request from client %u for tunnel %s\n",
-       c->id, GNUNET_i2s_full(&msg->peer));
-
-  t = GCP_get_tunnel (GCP_get (&msg->peer, GNUNET_NO));
-  if (NULL == t)
-  {
-    /* We don't know the tunnel */
-    struct GNUNET_CADET_LocalInfoTunnel warn;
-
-    LOG (GNUNET_ERROR_TYPE_INFO, "Tunnel %s unknown %u\n",
-         GNUNET_i2s_full(&msg->peer), sizeof (warn));
-    warn.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-    warn.header.size = htons (sizeof (warn));
-    warn.destination = msg->peer;
-    warn.channels = htonl (0);
-    warn.connections = htonl (0);
-    warn.cstate = htons (0);
-    warn.estate = htons (0);
-
-    GNUNET_SERVER_notification_context_unicast (nc, client,
-                                                &warn.header,
-                                                GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  /* Initialize context */
-  ch_n = GCT_count_channels (t);
-  c_n = GCT_count_any_connections (t);
-
-  size = sizeof (struct GNUNET_CADET_LocalInfoTunnel);
-  size += c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier);
-  size += ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber);
-
-  resp = GNUNET_malloc (size);
-  resp->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-  resp->header.size = htons (size);
-  resp->destination = msg->peer;
-  /* Do not interleave with iterators, iter_channel needs conn in HBO */
-  GCT_iterate_connections (t, &iter_connection, resp);
-  GCT_iterate_channels (t, &iter_channel, resp);
-  resp->connections = htonl (resp->connections);
-  resp->channels = htonl (resp->channels);
-  /* Do not interleave end */
-  resp->cstate = htons (GCT_get_cstate (t));
-  resp->estate = htons (GCT_get_estate (t));
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &resp->header, GNUNET_NO);
-  GNUNET_free (resp);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Show tunnel request from client %u completed. %u conn, %u ch\n",
-       c->id, c_n, ch_n);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client's INFO_DUMP request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_info_dump (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Received dump info request from client %u\n",
-       c->id);
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "*************************** DUMP START ***************************\n");
-
-  for (c = clients_head; NULL != c; c = c->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR, "Client %u (%p), handle: %p\n",
-         c->id, c, c->handle);
-    if (NULL != c->ports)
-      LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u ports registered\n",
-           GNUNET_CONTAINER_multihashmap_size (c->ports));
-    else
-      LOG (GNUNET_ERROR_TYPE_ERROR, "\t no ports registered\n");
-    LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u own channles\n",
-         GNUNET_CONTAINER_multihashmap32_size (c->own_channels));
-    LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u incoming channles\n",
-         GNUNET_CONTAINER_multihashmap32_size (c->incoming_channels));
-  }
-  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
-  GCP_iterate_all (&show_peer_iterator, NULL);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "**************************** DUMP END ****************************\n");
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Functions to handle messages from clients
- */
-static struct GNUNET_SERVER_MessageHandler client_handlers[] = {
-  {&handle_port_open, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
-    sizeof (struct GNUNET_CADET_PortMessage)},
-  {&handle_port_close, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
-    sizeof (struct GNUNET_CADET_PortMessage)},
-  {&handle_channel_create, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
-   sizeof (struct GNUNET_CADET_LocalChannelCreateMessage)},
-  {&handle_channel_destroy, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
-   sizeof (struct GNUNET_CADET_LocalChannelDestroyMessage)},
-  {&handle_data, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA, 0},
-  {&handle_ack, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-   sizeof (struct GNUNET_CADET_LocalAck)},
-  {&handle_get_peers, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-   sizeof (struct GNUNET_MessageHeader)},
-  {&handle_show_peer, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-   sizeof (struct GNUNET_CADET_LocalInfo)},
-  {&handle_get_tunnels, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-   sizeof (struct GNUNET_MessageHeader)},
-  {&handle_show_tunnel, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-   sizeof (struct GNUNET_CADET_LocalInfo)},
-  {&handle_info_dump, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize server subsystem.
- *
- * @param handle Server handle.
- */
-void
-GML_init (struct GNUNET_SERVER_Handle *handle)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-  server_handle = handle;
-  GNUNET_SERVER_suspend (server_handle);
-  ports = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO);
-}
-
-
-/**
- * Install server (service) handlers and start listening to clients.
- */
-void
-GML_start (void)
-{
-  GNUNET_SERVER_add_handlers (server_handle, client_handlers);
-  GNUNET_SERVER_connect_notify (server_handle,  &handle_client_connect, NULL);
-  GNUNET_SERVER_disconnect_notify (server_handle, &handle_client_disconnect,
-                                   NULL);
-  nc = GNUNET_SERVER_notification_context_create (server_handle, 1);
-
-  clients_head = NULL;
-  clients_tail = NULL;
-  next_client_id = 0;
-  GNUNET_SERVER_resume (server_handle);
-}
-
-
-/**
- * Shutdown server.
- */
-void
-GML_shutdown (void)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down local\n");
-
-  for (c = clients_head; NULL != clients_head; c = clients_head)
-    client_destroy (c);
-
-  if (nc != NULL)
-  {
-    GNUNET_SERVER_notification_context_destroy (nc);
-    nc = NULL;
-  }
-
-}
-
-
-/**
- * Get a channel from a client.
- *
- * @param c Client to check.
- * @param ccn Channel ID, must be local (> 0x800...).
- *
- * @return non-NULL if channel exists in the clients lists
- */
-struct CadetChannel *
-GML_channel_get (struct CadetClient *c,
-                 struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CONTAINER_MultiHashMap32 *map;
-
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    map = c->own_channels;
-  else
-    map = c->incoming_channels;
-
-  if (NULL == map)
-  {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Client %s does no t have a valid map for CCN %X\n",
-         GML_2s (c), ccn);
-    return NULL;
-  }
-  return GNUNET_CONTAINER_multihashmap32_get (map,
-                                              ccn.channel_of_client);
-}
-
-
-/**
- * Add a channel to a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_add (struct CadetClient *client,
-                 struct GNUNET_CADET_ClientChannelNumber ccn,
-                 struct CadetChannel *ch)
-{
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    GNUNET_CONTAINER_multihashmap32_put (client->own_channels,
-                                         ccn.channel_of_client,
-                                         ch,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
-  else
-    GNUNET_CONTAINER_multihashmap32_put (client->incoming_channels,
-                                         ccn.channel_of_client,
-                                         ch,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
-}
-
-
-/**
- * Remove a channel from a client.
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_remove (struct CadetClient *client,
-                    struct GNUNET_CADET_ClientChannelNumber ccn,
-                    struct CadetChannel *ch)
-{
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    GNUNET_CONTAINER_multihashmap32_remove (client->own_channels,
-                                            ccn.channel_of_client,
-                                            ch);
-  else
-    GNUNET_CONTAINER_multihashmap32_remove (client->incoming_channels,
-                                            ccn.channel_of_client,
-                                            ch);
-}
-
-
-/**
- * Get the tunnel's next free local channel ID.
- *
- * @param c Client.
- *
- * @return LID of a channel free to use.
- */
-struct GNUNET_CADET_ClientChannelNumber
-GML_get_next_ccn (struct CadetClient *c)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  while (NULL != GML_channel_get (c,
-                                  c->next_ccn))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %u exists...\n",
-         c->next_ccn);
-    c->next_ccn.channel_of_client
-      = htonl (1 + (ntohl (c->next_ccn.channel_of_client)));
-    if (ntohl (c->next_ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-      c->next_ccn.channel_of_client = htonl (0);
-  }
-  ccn = c->next_ccn;
-  c->next_ccn.channel_of_client
-    = htonl (1 + (ntohl (c->next_ccn.channel_of_client)));
-
-  return ccn;
-}
-
-
-/**
- * Check if client has registered with the service and has not disconnected
- *
- * @param client the client to check
- *
- * @return non-NULL if client exists in the global DLL
- */
-struct CadetClient *
-GML_client_get (struct GNUNET_SERVER_Client *client)
-{
-  if (NULL == client)
-    return NULL;
-  return GNUNET_SERVER_client_get_user_context (client,
-                                                struct CadetClient);
-}
-
-
-/**
- * Find a client that has opened a port
- *
- * @param port Port to check.
- *
- * @return non-NULL if a client has the port.
- */
-struct CadetClient *
-GML_client_get_by_port (const struct GNUNET_HashCode *port)
-{
-  return GNUNET_CONTAINER_multihashmap_get (ports, port);
-}
-
-
-/**
- * Deletes a channel from a client (either owner or destination).
- *
- * @param c Client whose tunnel to delete.
- * @param ch Channel which should be deleted.
- * @param id Channel ID.
- */
-void
-GML_client_delete_channel (struct CadetClient *c,
-                           struct CadetChannel *ch,
-                           struct GNUNET_CADET_ClientChannelNumber id)
-{
-  int res;
-
-  if (ntohl (id.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    res = GNUNET_CONTAINER_multihashmap32_remove (c->own_channels,
-                                                  id.channel_of_client,
-                                                  ch);
-    if (GNUNET_YES != res)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "client_delete_tunnel root KO\n");
-  }
-  else
-  {
-    res = GNUNET_CONTAINER_multihashmap32_remove (c->incoming_channels,
-                                                  id.channel_of_client,
-                                                  ch);
-    if (GNUNET_YES != res)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "client_delete_channel dest KO\n");
-  }
-}
-
-/**
- * Build a local ACK message and send it to a local client, if needed.
- *
- * If the client was already allowed to send data, do nothing.
- *
- * @param c Client to whom send the ACK.
- * @param ccn Channel ID to use
- */
-void
-GML_send_ack (struct CadetClient *c,
-              struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalAck msg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "send local %s ack on %X towards %p\n",
-       ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI
-       ? "FWD" : "BCK",
-       ntohl (ccn.channel_of_client),
-       c);
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              c->handle,
-                                              &msg.header,
-                                              GNUNET_NO);
-
-}
-
-
-
-/**
- * Notify the client that a new incoming channel was created.
- *
- * @param c Client to notify.
- * @param ccn Channel ID.
- * @param port Channel's destination port.
- * @param opt Options (bit array).
- * @param peer Origin peer.
- */
-void
-GML_send_channel_create (struct CadetClient *c,
-                         struct GNUNET_CADET_ClientChannelNumber ccn,
-                         const struct GNUNET_HashCode *port,
-                         uint32_t opt,
-                         const struct GNUNET_PeerIdentity *peer)
-{
-  struct GNUNET_CADET_LocalChannelCreateMessage msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  msg.ccn = ccn;
-  msg.port = *port;
-  msg.opt = htonl (opt);
-  msg.peer = *peer;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &msg.header, GNUNET_NO);
-}
-
-
-/**
- * Build a local channel NACK message and send it to a local client.
- *
- * @param c Client to whom send the NACK.
- * @param ccn Channel ID to use
- */
-void
-GML_send_channel_nack (struct CadetClient *c,
-                       struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalAck msg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "send local nack on %X towards %p\n",
-       ntohl (ccn.channel_of_client),
-       c);
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              c->handle,
-                                              &msg.header,
-                                              GNUNET_NO);
-
-}
-
-/**
- * Notify a client that a channel is no longer valid.
- *
- * @param c Client.
- * @param ccn ID of the channel that is destroyed.
- */
-void
-GML_send_channel_destroy (struct CadetClient *c,
-                          struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalChannelDestroyMessage msg;
-
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_YES == c->shutting_down)
-    return;
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &msg.header, GNUNET_NO);
-}
-
-
-/**
- * Modify the cadet message ID from global to local and send to client.
- *
- * @param c Client to send to.
- * @param msg Message to modify and send.
- * @param ccn Channel ID to use (c can be both owner and client).
- */
-void
-GML_send_data (struct CadetClient *c,
-               const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-               struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalData *copy;
-  uint16_t size = ntohs (msg->header.size) - sizeof (struct GNUNET_CADET_ChannelAppDataMessage);
-  char cbuf[size + sizeof (struct GNUNET_CADET_LocalData)];
-
-  if (size < sizeof (struct GNUNET_MessageHeader))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  copy = (struct GNUNET_CADET_LocalData *) cbuf;
-  GNUNET_memcpy (&copy[1], &msg[1], size);
-  copy->header.size = htons (sizeof (struct GNUNET_CADET_LocalData) + size);
-  copy->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  copy->ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &copy->header, GNUNET_NO);
-}
-
-
-/**
- * Get the static string to represent a client.
- *
- * @param c Client.
- *
- * @return Static string for the client.
- */
-const char *
-GML_2s (const struct CadetClient *c)
-{
-  static char buf[32];
-
-  SPRINTF (buf, "%u", c->id);
-  return buf;
-}
diff --git a/src/cadet/gnunet-service-cadet_local.h b/src/cadet/gnunet-service-cadet_local.h
deleted file mode 100644
index 113c2f489..000000000
--- a/src/cadet/gnunet-service-cadet_local.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_local.h
- * @brief cadet service; dealing with local clients
- * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GML (Gnunet Cadet Local)
- */
-
-#ifndef GNUNET_SERVICE_CADET_LOCAL_H
-#define GNUNET_SERVICE_CADET_LOCAL_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * Struct containing information about a client of the service
- */
-struct CadetClient;
-
-#include "gnunet-service-cadet_channel.h"
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize server subsystem.
- *
- * @param handle Server handle.
- */
-void
-GML_init (struct GNUNET_SERVER_Handle *handle);
-
-/**
- * Install server (service) handlers and start listening to clients.
- */
-void
-GML_start (void);
-
-/**
- * Shutdown server.
- */
-void
-GML_shutdown (void);
-
-/**
- * Get a channel from a client.
- *
- * @param c Client to check.
- * @param ccn Channel ID, must be local (> 0x800...).
- *
- * @return non-NULL if channel exists in the clients lists
- */
-struct CadetChannel *
-GML_channel_get (struct CadetClient *c,
-                 struct GNUNET_CADET_ClientChannelNumber ccn);
-
-/**
- * Add a channel to a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_add (struct CadetClient *client,
-                 struct GNUNET_CADET_ClientChannelNumber ccn,
-                 struct CadetChannel *ch);
-
-/**
- * Remove a channel from a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_remove (struct CadetClient *client,
-                    struct GNUNET_CADET_ClientChannelNumber ccn,
-                    struct CadetChannel *ch);
-
-/**
- * Get the tunnel's next free local channel ID.
- *
- * @param c Client.
- *
- * @return LID of a channel free to use.
- */
-struct GNUNET_CADET_ClientChannelNumber
-GML_get_next_ccn (struct CadetClient *c);
-
-/**
- * Check if client has registered with the service and has not disconnected
- *
- * @param client the client to check
- *
- * @return non-NULL if client exists in the global DLL
- */
-struct CadetClient *
-GML_client_get (struct GNUNET_SERVER_Client *client);
-
-/**
- * Find a client that has opened a port
- *
- * @param port Port to check.
- *
- * @return non-NULL if a client has the port.
- */
-struct CadetClient *
-GML_client_get_by_port (const struct GNUNET_HashCode *port);
-
-/**
- * Deletes a tunnel from a client (either owner or destination).
- *
- * @param c Client whose tunnel to delete.
- * @param ch Channel which should be deleted.
- * @param id Channel ID.
- */
-void
-GML_client_delete_channel (struct CadetClient *c,
-                           struct CadetChannel *ch,
-                           struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Build a local ACK message and send it to a local client, if needed.
- *
- * If the client was already allowed to send data, do nothing.
- *
- * @param c Client to whom send the ACK.
- * @param id Channel ID to use
- */
-void
-GML_send_ack (struct CadetClient *c,
-              struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Notify the appropriate client that a new incoming channel was created.
- *
- * @param c Client to notify.
- * @param id Channel ID.
- * @param port Channel's destination port.
- * @param opt Options (bit array).
- * @param peer Origin peer.
- */
-void
-GML_send_channel_create (struct CadetClient *c,
-                         struct GNUNET_CADET_ClientChannelNumber id,
-                         const struct GNUNET_HashCode *port,
-                         uint32_t opt,
-                         const struct GNUNET_PeerIdentity *peer);
-
-/**
- * Build a local channel NACK message and send it to a local client.
- *
- * @param c Client to whom send the NACK.
- * @param id Channel ID to use
- */
-void
-GML_send_channel_nack (struct CadetClient *c,
-                       struct GNUNET_CADET_ClientChannelNumber id);
-
-
-/**
- * Notify a client that a channel is no longer valid.
- *
- * @param c Client.
- * @param id ID of the channel that is destroyed.
- */
-void
-GML_send_channel_destroy (struct CadetClient *c,
-                          struct GNUNET_CADET_ClientChannelNumber id);
-
-
-/**
- * Modify the cadet message ID from global to local and send to client.
- *
- * @param c Client to send to.
- * @param msg Message to modify and send.
- * @param id Channel ID to use (c can be both owner and client).
- */
-void
-GML_send_data (struct CadetClient *c,
-               const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-               struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Get the static string to represent a client.
- *
- * @param c Client.
- *
- * @return Static string for the client.
- */
-const char *
-GML_2s (const struct CadetClient *c);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_LOCAL_H */
-#endif
-/* end of gnunet-cadet-service_LOCAL.h */
diff --git a/src/cadet/gnunet-service-cadet-new_paths.c b/src/cadet/gnunet-service-cadet_paths.c
index 685656ec3..13752643c 100644
--- a/src/cadet/gnunet-service-cadet-new_paths.c
+++ b/src/cadet/gnunet-service-cadet_paths.c
@@ -18,18 +18,16 @@
      Boston, MA 02110-1301, USA.
 */
 /**
- * @file cadet/gnunet-service-cadet-new_paths.c
+ * @file cadet/gnunet-service-cadet_paths.c
  * @brief Information we track per path.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
- *
- * TODO:
- * - path desirability score calculations are not done
  */
 #include "platform.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-pat",__VA_ARGS__)
@@ -75,8 +73,16 @@ struct CadetPeerPath
 static void
 recalculate_path_desirability (struct CadetPeerPath *path)
 {
-  /* FIXME: update path desirability! */
-  GNUNET_break (0); // not implemented
+  double result = 0.0;
+
+  for (unsigned int i=0;i<path->entries_length;i++)
+  {
+    struct CadetPeer *cp = path->entries[i]->peer;
+
+    result += GCP_get_desirability_of_path (cp,
+                                            i);
+  }
+  path->desirability = (GNUNET_CONTAINER_HeapCostType) result;
 }
 
 
@@ -147,6 +153,7 @@ GCPP_add_connection (struct CadetPeerPath *path,
   GNUNET_assert (off < path->entries_length);
   entry = path->entries[off];
   GNUNET_assert (NULL == entry->cc);
+  GNUNET_assert (NULL != cc);
   entry->cc = cc;
 }
 
@@ -191,7 +198,20 @@ path_destroy (struct CadetPeerPath *path)
        "Destroying path %s\n",
        GCPP_2s (path));
   for (unsigned int i=0;i<path->entries_length;i++)
-    GNUNET_free (path->entries[i]);
+  {
+    struct CadetPeerPathEntry *entry = path->entries[i];
+
+    if (NULL != entry->cc)
+    {
+      struct CadetTConnection *ct;
+
+      ct = GCC_get_ct (entry->cc);
+      if (NULL != ct)
+        GCT_connection_lost (ct);
+      GCC_destroy_without_tunnel (entry->cc);
+    }
+    GNUNET_free (entry);
+  }
   GNUNET_free (path->entries);
   GNUNET_free (path);
 }
@@ -208,15 +228,18 @@ void
 GCPP_release (struct CadetPeerPath *path)
 {
   struct CadetPeerPathEntry *entry;
+  int force;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Owner releases path %s\n",
        GCPP_2s (path));
   path->hn = NULL;
   entry = path->entries[path->entries_length - 1];
+  GNUNET_assert (path == entry->path);
   while (1)
   {
     /* cut 'off' end of path */
+    GNUNET_assert (NULL == entry->cc);
     GCP_path_entry_remove (entry->peer,
                            entry,
                            path->entries_length - 1);
@@ -228,12 +251,15 @@ GCPP_release (struct CadetPeerPath *path)
 
     /* see if new peer at the end likes this path any better */
     entry = path->entries[path->entries_length - 1];
+    GNUNET_assert (path == entry->path);
+    force = (NULL == entry->cc) ? GNUNET_NO : GNUNET_YES;
     path->hn = GCP_attach_path (entry->peer,
                                 path,
                                 path->entries_length - 1,
-                                GNUNET_NO);
+                                force);
     if (NULL != path->hn)
       return; /* yep, got attached, we are done. */
+    GNUNET_assert (GNUNET_NO == force);
   }
 
   /* nobody wants us, discard the path */
@@ -367,7 +393,6 @@ extend_path (struct CadetPeerPath *path,
              int force)
 {
   unsigned int old_len = path->entries_length;
-  struct GNUNET_CONTAINER_HeapNode *hn;
   int i;
 
   /* Expand path */
@@ -393,38 +418,43 @@ extend_path (struct CadetPeerPath *path,
 
   /* If we extend an existing path, detach it from the
      old owner and re-attach to the new one */
-  hn = NULL;
+  GCP_detach_path (path->entries[old_len-1]->peer,
+                   path,
+                   path->hn);
+  path->hn = NULL;
   for (i=num_peers-1;i>=0;i--)
   {
     struct CadetPeerPathEntry *entry = path->entries[old_len + i];
 
     path->entries_length = old_len + i + 1;
     recalculate_path_desirability (path);
-    hn = GCP_attach_path (peers[i],
-                          path,
-                          old_len + (unsigned int) i,
-                          GNUNET_YES);
-    if (NULL != hn)
+    path->hn = GCP_attach_path (peers[i],
+                                path,
+                                old_len + (unsigned int) i,
+                                force);
+    if (NULL != path->hn)
       break;
+    GNUNET_assert (NULL == entry->cc);
     GCP_path_entry_remove (entry->peer,
                            entry,
                            old_len + i);
     GNUNET_free (entry);
     path->entries[old_len + i] = NULL;
   }
-  if (NULL == hn)
+  if (NULL == path->hn)
   {
     /* none of the peers is interested in this path;
-       shrink path back */
+       shrink path back and re-attach. */
     GNUNET_array_grow (path->entries,
                        path->entries_length,
                        old_len);
+    path->hn = GCP_attach_path (path->entries[old_len - 1]->peer,
+                                path,
+                                old_len - 1,
+                                GNUNET_YES);
+    GNUNET_assert (NULL != path->hn);
     return;
   }
-  GCP_detach_path (path->entries[old_len-1]->peer,
-                   path,
-                   path->hn);
-  path->hn = hn;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Extended path %s\n",
        GCPP_2s (path));
@@ -454,25 +484,42 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path,
   struct CadetPeerPath *path;
   struct GNUNET_CONTAINER_HeapNode *hn;
   int i;
+  unsigned int skip;
+  unsigned int total_len;
 
   /* precompute 'cpath' so we can avoid doing the lookups lots of times */
-  for (unsigned int off=0;off<get_path_length + put_path_length;off++)
+  skip = 0;
+  memset (cpath,
+          0,
+          sizeof (cpath)); /* Just to trigger harder errors later. */
+  total_len = get_path_length + put_path_length;
+  for (unsigned int off=0;off<total_len;off++)
   {
     const struct GNUNET_PeerIdentity *pid;
 
     pid = (off < get_path_length)
       ? &get_path[get_path_length - off]
       : &put_path[get_path_length + put_path_length - off];
-    cpath[off] = GCP_get (pid,
-                          GNUNET_YES);
+    cpath[off - skip] = GCP_get (pid,
+                                 GNUNET_YES);
+    /* Check that no peer is twice on the path */
+    for (unsigned int i=0;i<off - skip;i++)
+    {
+     if (cpath[i] == cpath[off - skip])
+      {
+        skip = off - i;
+        break;
+      }
+    }
   }
+  total_len -= skip;
 
   /* First figure out if this path is a subset of an existing path, an
      extension of an existing path, or a new path. */
-  cm_ctx.cpath_length = get_path_length + put_path_length;
+  cm_ctx.cpath_length = total_len;
   cm_ctx.cpath = cpath;
   cm_ctx.match = NULL;
-  for (i=get_path_length + put_path_length-1;i>=0;i--)
+  for (i=total_len-1;i>=0;i--)
   {
     GCP_iterate_paths_at (cpath[i],
                           (unsigned int) i,
@@ -480,7 +527,7 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path,
                           &cm_ctx);
     if (NULL != cm_ctx.match)
     {
-      if (i == get_path_length + put_path_length - 1)
+      if (i == total_len - 1)
       {
         /* Existing path includes this one, nothing to do! */
         LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -494,8 +541,8 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path,
              "Trying to extend existing path %s by additional links discovered from DHT\n",
              GCPP_2s (cm_ctx.match));
         extend_path (cm_ctx.match,
-                     &cpath[i],
-                     get_path_length + put_path_length - i,
+                     &cpath[i + 1],
+                     total_len - i - 1,
                      GNUNET_NO);
         return;
       }
@@ -504,7 +551,7 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path,
 
   /* No match at all, create completely new path */
   path = GNUNET_new (struct CadetPeerPath);
-  path->entries_length = get_path_length + put_path_length;
+  path->entries_length = total_len;
   path->entries = GNUNET_new_array (path->entries_length,
                                     struct CadetPeerPathEntry *);
   for (i=path->entries_length-1;i>=0;i--)
@@ -526,7 +573,7 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path,
 
   /* Finally, try to attach it */
   hn = NULL;
-  for (i=get_path_length + put_path_length-1;i>=0;i--)
+  for (i=total_len-1;i>=0;i--)
   {
     struct CadetPeerPathEntry *entry = path->entries[i];
 
@@ -613,8 +660,8 @@ GCPP_get_path_from_route (unsigned int path_length,
              "Extending existing path %s to create inverse for incoming connection\n",
              GCPP_2s (cm_ctx.match));
         extend_path (cm_ctx.match,
-                     &cpath[i],
-                     path_length - i,
+                     &cpath[i + 1],
+                     path_length - i - 1,
                      GNUNET_YES);
         /* Check that extension was successful */
         GNUNET_assert (cm_ctx.match->entries_length == path_length);
diff --git a/src/cadet/gnunet-service-cadet-new_paths.h b/src/cadet/gnunet-service-cadet_paths.h
index 7310d75e6..6b7bef640 100644
--- a/src/cadet/gnunet-service-cadet-new_paths.h
+++ b/src/cadet/gnunet-service-cadet_paths.h
@@ -29,7 +29,7 @@
 #define GNUNET_SERVICE_CADET_PATHS_H
 
 #include "gnunet_util_lib.h"
-#include "gnunet-service-cadet-new.h"
+#include "gnunet-service-cadet.h"
 
 /**
  * Create a peer path based on the result of a DHT lookup.  If we
diff --git a/src/cadet/gnunet-service-cadet_peer.c b/src/cadet/gnunet-service-cadet_peer.c
index fa3f2be80..71c7c67d0 100644
--- a/src/cadet/gnunet-service-cadet_peer.c
+++ b/src/cadet/gnunet-service-cadet_peer.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013, 2015 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,2193 +17,1461 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
+
 /**
  * @file cadet/gnunet-service-cadet_peer.c
- * @brief GNUnet CADET service connection handling
+ * @brief Information we track per peer.
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - optimize stopping/restarting DHT search to situations
+ *   where we actually need it (i.e. not if we have a direct connection,
+ *   or if we already have plenty of good short ones, or maybe even
+ *   to take a break if we have some connections and have searched a lot (?))
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_hello_lib.h"
 #include "gnunet_signatures.h"
 #include "gnunet_transport_service.h"
 #include "gnunet_ats_service.h"
 #include "gnunet_core_service.h"
 #include "gnunet_statistics_service.h"
 #include "cadet_protocol.h"
-#include "gnunet-service-cadet_peer.h"
-#include "gnunet-service-cadet_dht.h"
 #include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "cadet_path.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
 
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-p2p",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-p2p",__VA_ARGS__)
 
+#define LOG(level, ...) GNUNET_log_from(level,"cadet-per",__VA_ARGS__)
 
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
 
 /**
- * Information about a queued message on the peer level.
+ * How long do we wait until tearing down an idle peer?
  */
-struct CadetPeerQueue {
-
-    struct CadetPeerQueue *next;
-    struct CadetPeerQueue *prev;
-
-    /**
-     * Envelope to cancel message before MQ sends it.
-     */
-    struct GNUNET_MQ_Envelope *env;
-
-    /**
-     * Peer (neighbor) this message is being sent to.
-     */
-    struct CadetPeer *peer;
-
-    /**
-     * Continuation to call to notify higher layers about message sent.
-     */
-    GCP_sent cont;
-
-    /**
-     * Closure for @a cont.
-     */
-    void *cont_cls;
-
-    /**
-     * Task to asynchronously run the drop continuation.
-     */
-    struct GNUNET_SCHEDULER_Task *drop_task;
-
-    /**
-     * Time when message was queued for sending.
-     */
-    struct GNUNET_TIME_Absolute queue_timestamp;
-
-    /**
-     * #GNUNET_YES if message was management traffic (POLL, ACK, ...).
-     */
-    int management_traffic;
-
-    /**
-     * Message type.
-     */
-    uint16_t type;
-
-    /**
-     * Message size.
-     */
-    uint16_t size;
-
-    /**
-     * Type of the message's payload, if it was encrypted data.
-     */
-    uint16_t payload_type;
-
-    /**
-     * ID of the payload (PID, ACK #, ...).
-     */
-    struct CadetEncryptedMessageIdentifier payload_id;
-
-    /**
-     * Connection this message was sent on.
-     */
-    struct CadetConnection *c;
-
-    /**
-     * Direction in @a c this message was send on (#GNUNET_YES = FWD).
-     */
-    int c_fwd;
-};
-
+#define IDLE_PEER_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
 
 /**
- * Struct containing all information regarding a given peer
+ * How long do we keep paths around if we no longer care about the peer?
  */
-struct CadetPeer
-{
-    /**
-     * ID of the peer
-     */
-    GNUNET_PEER_Id id;
-
-    struct CadetPeerQueue *q_head;
-    struct CadetPeerQueue *q_tail;
-
-    /**
-     * Last time we heard from this peer
-     */
-    struct GNUNET_TIME_Absolute last_contact;
-
-    /**
-     * Paths to reach the peer, ordered by ascending hop count
-     */
-    struct CadetPeerPath *path_head;
-
-    /**
-     * Paths to reach the peer, ordered by ascending hop count
-     */
-    struct CadetPeerPath *path_tail;
-
-    /**
-     * Handle to stop the DHT search for paths to this peer
-     */
-    struct GCD_search_handle *search_h;
-
-    /**
-     * Handle to stop the DHT search for paths to this peer
-     */
-    struct GNUNET_SCHEDULER_Task *search_delayed;
-
-    /**
-     * Tunnel to this peer, if any.
-     */
-    struct CadetTunnel *tunnel;
-
-    /**
-     * Connections that go through this peer; indexed by tid.
-     */
-    struct GNUNET_CONTAINER_MultiShortmap  *connections;
-
-    /**
-     * Handle for core transmissions.
-     */
-    struct GNUNET_MQ_Handle *core_mq;
-
-    /**
-     * How many messages are in the queue to this peer.
-     */
-    unsigned int queue_n;
-
-    /**
-     * Hello message.
-     */
-    struct GNUNET_HELLO_Message* hello;
-
-    /**
-     * Handle to us offering the HELLO to the transport.
-     */
-    struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
-
-    /**
-     * Handle to our ATS request asking ATS to suggest an address
-     * to TRANSPORT for this peer (to establish a direct link).
-     */
-    struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
+#define IDLE_PATH_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 2)
 
-};
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
 
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
 
-/**
- * Local peer own ID (short)
- */
-extern GNUNET_PEER_Id myid;
 
 /**
- * Peers known, indexed by PeerIdentity, values of type `struct CadetPeer`.
+ * Data structure used to track whom we have to notify about changes
+ * to our message queue.
  */
-static struct GNUNET_CONTAINER_MultiPeerMap *peers;
-
-/**
- * How many peers do we want to remember?
- */
-static unsigned long long max_peers;
+struct GCP_MessageQueueManager
+{
 
-/**
- * Percentage of messages that will be dropped (for test purposes only).
- */
-static unsigned long long drop_percent;
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *next;
 
-/**
- * Handle to communicate with CORE.
- */
-static struct GNUNET_CORE_Handle *core_handle;
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *prev;
 
-/**
- * Our configuration;
- */
-static const struct GNUNET_CONFIGURATION_Handle *cfg;
+  /**
+   * Function to call with updated message queue object.
+   */
+  GCP_MessageQueueNotificationCallback cb;
 
-/**
- * Handle to communicate with ATS.
- */
-static struct GNUNET_ATS_ConnectivityHandle *ats_ch;
+  /**
+   * Closure for @e cb.
+   */
+  void *cb_cls;
 
-/**
- * Shutdown falg.
- */
-static int in_shutdown;
+  /**
+   * The peer this is for.
+   */
+  struct CadetPeer *cp;
 
+  /**
+   * Envelope this manager would like to transmit once it is its turn.
+   */
+  struct GNUNET_MQ_Envelope *env;
 
-/******************************************************************************/
-/*****************************  CORE HELPERS  *********************************/
-/******************************************************************************/
+};
 
 
 /**
- * Iterator to notify all connections of a broken link. Mark connections
- * to destroy after all traffic has been sent.
- *
- * @param cls Closure (disconnected peer).
- * @param key Current key code (peer id).
- * @param value Value in the hash map (connection).
- *
- * @return #GNUNET_YES to continue to iterate.
+ * Struct containing all information regarding a given peer
  */
-static int
-notify_broken (void *cls,
-               const struct GNUNET_ShortHashCode *key,
-               void *value)
+struct CadetPeer
 {
-    struct CadetPeer *peer = cls;
-    struct CadetConnection *c = value;
+  /**
+   * ID of the peer
+   */
+  struct GNUNET_PeerIdentity pid;
+
+  /**
+   * Last time we heard from this peer (currently not used!)
+   */
+  struct GNUNET_TIME_Absolute last_contactXXX;
+
+  /**
+   * Array of DLLs of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_heads;
+
+  /**
+   * Array of DLL of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_tails;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_head;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_tail;
+
+  /**
+   * Pointer to first "ready" entry in @e mqm_head.
+   */
+  struct GCP_MessageQueueManager *mqm_ready_ptr;
+
+  /**
+   * MIN-heap of paths owned by this peer (they also end at this
+   * peer).  Ordered by desirability.
+   */
+  struct GNUNET_CONTAINER_Heap *path_heap;
+
+  /**
+   * Handle to stop the DHT search for paths to this peer
+   */
+  struct GCD_search_handle *search_h;
+
+  /**
+   * Task to clean up @e path_heap asynchronously.
+   */
+  struct GNUNET_SCHEDULER_Task *heap_cleanup_task;
+
+  /**
+   * Task to destroy this entry.
+   */
+  struct GNUNET_SCHEDULER_Task *destroy_task;
+
+  /**
+   * Tunnel to this peer, if any.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Connections that go through this peer; indexed by tid.
+   */
+  struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+  /**
+   * Handle for core transmissions.
+   */
+  struct GNUNET_MQ_Handle *core_mq;
+
+  /**
+   * Hello message of the peer.
+   */
+  struct GNUNET_HELLO_Message *hello;
+
+  /**
+   * Handle to us offering the HELLO to the transport.
+   */
+  struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
+
+  /**
+   * Handle to our ATS request asking ATS to suggest an address
+   * to TRANSPORT for this peer (to establish a direct link).
+   */
+  struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
+
+  /**
+   * How many messages are in the queue to this peer.
+   */
+  unsigned int queue_n;
+
+  /**
+   * How many paths do we have to this peer (in all @e path_heads DLLs combined).
+   */
+  unsigned int num_paths;
+
+  /**
+   * Sum over all of the offsets of all of the paths in the @a path_heads DLLs.
+   * Used to speed-up @GCP_get_desirability_of_path() calculation.
+   */
+  unsigned int off_sum;
+
+  /**
+   * Number of message queue managers of this peer that have a message in waiting.
+   *
+   * Used to quickly see if we need to bother scanning the @e msm_head DLL.
+   * TODO: could be replaced by another DLL that would then allow us to avoid
+   * the O(n)-scan of the DLL for ready entries!
+   */
+  unsigned int mqm_ready_counter;
+
+  /**
+   * Current length of the @e path_heads and @path_tails arrays.
+   * The arrays should be grown as needed.
+   */
+  unsigned int path_dll_length;
 
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Notifying %s due to %s disconnect\n",
-         GCC_2s (c), GCP_2s (peer));
-    GCC_neighbor_disconnected (c, peer);
-    return GNUNET_YES;
-}
+};
 
 
 /**
- * Remove the direct path to the peer.
- *
- * @param peer Peer to remove the direct path from.
- */
-static struct CadetPeerPath *
-pop_direct_path (struct CadetPeer *peer)
-{
-    struct CadetPeerPath *iter;
-
-    for (iter = peer->path_head; NULL != iter; iter = iter->next)
-    {
-        if (2 >= iter->length)
-        {
-            GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                         peer->path_tail,
-                                         iter);
-            return iter;
-        }
-    }
-    return NULL;
-}
-
-/**
- * Call the continuation after a message has been sent or dropped.
- *
- * This funcion removes the message from the queue.
+ * Get the static string for a peer ID.
  *
- * @param q Queue handle.
- * @param sent #GNUNET_YES if was sent to CORE, #GNUNET_NO if dropped.
+ * @param cp Peer.
+ * @return Static string for it's ID.
  */
-static void
-call_peer_cont (struct CadetPeerQueue *q, int sent);
+const char *
+GCP_2s (const struct CadetPeer *cp)
+{
+  static char buf[32];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "P(%s)",
+                   GNUNET_i2s (&cp->pid));
+  return buf;
+}
+
+
+/**
+ * Calculate how desirable a path is for @a cp if @a cp
+ * is at offset @a off.
+ *
+ * The 'desirability_table.c' program can be used to compute a list of
+ * sample outputs for different scenarios.  Basically, we score paths
+ * lower if there are many alternatives, and higher if they are
+ * shorter than average, and very high if they are much shorter than
+ * average and without many alternatives.
+ *
+ * @param cp a peer reachable via a path
+ * @param off offset of @a cp in the path
+ * @return score how useful a path is to reach @a cp,
+ *         positive scores mean path is more desirable
+ */
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off)
+{
+  unsigned int num_alts = cp->num_paths;
+  unsigned int off_sum;
+  double avg_sum;
+  double path_delta;
+  double weight_alts;
+
+  GNUNET_assert (num_alts >= 1); /* 'path' should be in there! */
+  GNUNET_assert (0 != cp->path_dll_length);
+
+  /* We maintain 'off_sum' in 'peer' and thereby
+     avoid the SLOW recalculation each time. Kept here
+     just to document what is going on. */
+#if SLOW
+  off_sum = 0;
+  for (unsigned int j=0;j<cp->path_dll_length;j++)
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[j];
+         NULL != pe;
+         pe = pe->next)
+      off_sum += j;
+  GNUNET_assert (off_sum == cp->off_sum);
+#else
+  off_sum = cp->off_sum;
+#endif
+  avg_sum = off_sum * 1.0 / cp->path_dll_length;
+  path_delta = off - avg_sum;
+  /* path_delta positiv: path off of peer above average (bad path for peer),
+     path_delta negativ: path off of peer below average (good path for peer) */
+  if (path_delta <= - 1.0)
+    weight_alts = - num_alts / path_delta; /* discount alternative paths */
+  else if (path_delta >= 1.0)
+    weight_alts = num_alts * path_delta; /* overcount alternative paths */
+  else
+    weight_alts = num_alts; /* count alternative paths normally */
 
 
-/******************************************************************************/
-/***************************** CORE CALLBACKS *********************************/
-/******************************************************************************/
+  /* off+1: long paths are generally harder to find and thus count
+     a bit more as they get longer.  However, above-average paths
+     still need to count less, hence the squaring of that factor. */
+  return (off + 1.0) / (weight_alts * weight_alts);
+}
 
 
 /**
- * Method called whenever a given peer connects.
+ * This peer is no longer be needed, clean it up now.
  *
- * @param cls Core closure (unused).
- * @param peer Peer identity this notification is about
- * @param mq Message Queue to this peer.
- *
- * @return Internal closure for handlers (CadetPeer struct).
+ * @param cls peer to clean up
  */
-static void *
-core_connect_handler (void *cls,
-                      const struct GNUNET_PeerIdentity *peer,
-                      struct GNUNET_MQ_Handle *mq)
-{
-    struct CadetPeer *neighbor;
-    struct CadetPeerPath *path;
-    char own_id[16];
-
-    GCC_check_connections ();
-    GNUNET_snprintf (own_id,
-                     sizeof (own_id),
-                     "%s",
-                     GNUNET_i2s (&my_full_id));
-
-    /* Save a path to the neighbor */
-    neighbor = GCP_get (peer, GNUNET_YES);
-    if (myid == neighbor->id)
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "CONNECTED %s (self)\n",
-             own_id);
-        path = path_new (1);
-    }
-    else
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "CONNECTED %s <= %s\n",
-             own_id,
-             GNUNET_i2s (peer));
-        path = path_new (2);
-        path->peers[1] = neighbor->id;
-        GNUNET_PEER_change_rc (neighbor->id, 1);
-        GNUNET_assert (NULL == neighbor->core_mq);
-        neighbor->core_mq = mq;
-    }
-    path->peers[0] = myid;
-    GNUNET_PEER_change_rc (myid, 1);
-    GCP_add_path (neighbor, path, GNUNET_YES);
-
-    /* Create the connections hashmap */
-    GNUNET_assert (NULL == neighbor->connections);
-    neighbor->connections = GNUNET_CONTAINER_multishortmap_create (16,
-                                                                   GNUNET_YES);
-    GNUNET_STATISTICS_update (stats,
-                              "# peers",
-                              1,
-                              GNUNET_NO);
-
-    if ( (NULL != GCP_get_tunnel (neighbor)) &&
-            (0 > GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, peer)) )
-    {
-        GCP_connect (neighbor);
-    }
-    GCC_check_connections ();
+static void
+destroy_peer (void *cls)
+{
+  struct CadetPeer *cp = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying state about peer %s\n",
+       GCP_2s (cp));
+  cp->destroy_task = NULL;
+  GNUNET_assert (NULL == cp->t);
+  GNUNET_assert (NULL == cp->core_mq);
+  GNUNET_assert (0 == cp->num_paths);
+  for (unsigned int i=0;i<cp->path_dll_length;i++)
+    GNUNET_assert (NULL == cp->path_heads[i]);
+  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (peers,
+                                                       &cp->pid,
+                                                       cp));
+  GNUNET_free_non_null (cp->path_heads);
+  GNUNET_free_non_null (cp->path_tails);
+  cp->path_dll_length = 0;
+  if (NULL != cp->search_h)
+  {
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  /* FIXME: clean up search_delayedXXX! */
 
-    return neighbor;
+  if (NULL != cp->hello_offer)
+  {
+    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
+    cp->hello_offer = NULL;
+  }
+  if (NULL != cp->connectivity_suggestion)
+  {
+    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+    cp->connectivity_suggestion = NULL;
+  }
+  GNUNET_CONTAINER_multishortmap_destroy (cp->connections);
+  if (NULL != cp->path_heap)
+  {
+    GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+    cp->path_heap = NULL;
+  }
+  if (NULL != cp->heap_cleanup_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->heap_cleanup_task);
+    cp->heap_cleanup_task = NULL;
+  }
+  GNUNET_free_non_null (cp->hello);
+  /* Peer should not be freed if paths exist; if there are no paths,
+     there ought to be no connections, and without connections, no
+     notifications. Thus we can assert that mqm_head is empty at this
+     point. */
+  GNUNET_assert (NULL == cp->mqm_head);
+  GNUNET_assert (NULL == cp->mqm_ready_ptr);
+  GNUNET_free (cp);
 }
 
 
 /**
- * Method called whenever a peer disconnects.
+ * This peer is now on more "active" duty, activate processes related to it.
  *
- * @param cls Core closure (unused).
- * @param peer Peer identity this notification is about.
- * @param internal_cls Internal closure (CadetPeer struct).
+ * @param cp the more-active peer
  */
 static void
-core_disconnect_handler (void *cls,
-                         const struct GNUNET_PeerIdentity *peer,
-                         void *internal_cls)
+consider_peer_activate (struct CadetPeer *cp)
 {
-    struct CadetPeer *p = internal_cls;
-    struct CadetPeerPath *direct_path;
-    char own_id[16];
-
-    GCC_check_connections ();
-    strncpy (own_id, GNUNET_i2s (&my_full_id), 16);
-    own_id[15] = '\0';
-    if (myid == p->id)
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "DISCONNECTED %s (self)\n",
-             own_id);
-    }
-    else
+  uint32_t strength;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Updating peer %s activation state (%u connections)%s%s\n",
+       GCP_2s (cp),
+       GNUNET_CONTAINER_multishortmap_size (cp->connections),
+       (NULL == cp->t) ? "" : " with tunnel",
+       (NULL == cp->core_mq) ? "" : " with CORE link");
+  if (NULL != cp->destroy_task)
+  {
+    /* It's active, do not destory! */
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if ( (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections)) &&
+       (NULL == cp->t) )
+  {
+    /* We're just on a path or directly connected; don't bother too much */
+    if (NULL != cp->connectivity_suggestion)
     {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "DISCONNECTED %s <= %s\n",
-             own_id, GNUNET_i2s (peer));
-        p->core_mq = NULL;
+      GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+      cp->connectivity_suggestion = NULL;
     }
-    direct_path = pop_direct_path (p);
-    if (NULL != p->connections)
+    if (NULL != cp->search_h)
     {
-      GNUNET_CONTAINER_multishortmap_iterate (p->connections,
-                                              &notify_broken,
-                                              p);
-      GNUNET_CONTAINER_multishortmap_destroy (p->connections);
-      p->connections = NULL;
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
     }
-    GNUNET_STATISTICS_update (stats,
-                              "# peers",
-                              -1,
-                              GNUNET_NO);
-    path_destroy (direct_path);
-    GCC_check_connections ();
-}
-
-
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-
-/**
- * Check if the create_connection message has the appropriate size.
- *
- * @param cls Closure (unused).
- * @param msg Message to check.
- *
- * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
- */
-static int
-check_create (void *cls, const struct GNUNET_CADET_ConnectionCreateMessage *msg)
-{
-    uint16_t size;
-
-    size = ntohs (msg->header.size);
-    if (size < sizeof (*msg))
+    return;
+  }
+  if (NULL == cp->core_mq)
+  {
+    /* Lacks direct connection, try to create one by querying the DHT */
+    if ( (NULL == cp->search_h) &&
+         (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
+      cp->search_h
+        = GCD_search (&cp->pid);
+  }
+  else
+  {
+    /* Have direct connection, stop DHT search if active */
+    if (NULL != cp->search_h)
     {
-        GNUNET_break_op (0);
-        return GNUNET_NO;
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
     }
-    return GNUNET_YES;
-}
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_create (void *cls, const struct GNUNET_CADET_ConnectionCreateMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_create (peer, msg);
-}
-
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_confirm (void *cls, const struct GNUNET_CADET_ConnectionCreateAckMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_confirm (peer, msg);
-}
-
+  }
 
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_broken (void *cls, const struct GNUNET_CADET_ConnectionBrokenMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_broken (peer, msg);
+  /* If we have a tunnel, our urge for connections is much bigger */
+  strength = (NULL != cp->t) ? 32 : 1;
+  if (NULL != cp->connectivity_suggestion)
+    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+  cp->connectivity_suggestion
+    = GNUNET_ATS_connectivity_suggest (ats_ch,
+                                       &cp->pid,
+                                       strength);
 }
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY
+ * This peer may no longer be needed, consider cleaning it up.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cp peer to clean up
  */
 static void
-handle_destroy (void *cls, const struct GNUNET_CADET_ConnectionDestroyMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_destroy (peer, msg);
-}
+consider_peer_destroy (struct CadetPeer *cp);
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK
+ * We really no longere care about a peer, stop hogging memory with paths to it.
+ * Afterwards, see if there is more to be cleaned up about this peer.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cls a `struct CadetPeer`.
  */
 static void
-handle_ack (void *cls, const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg)
+drop_paths (void *cls)
 {
-    struct CadetPeer *peer = cls;
-    GCC_handle_ack (peer, msg);
-}
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *path;
 
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_poll (void *cls, const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_poll (peer, msg);
+  cp->destroy_task = NULL;
+  while (NULL != (path = GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX
+ * This peer may no longer be needed, consider cleaning it up.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cp peer to clean up
  */
 static void
-handle_kx (void *cls, const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_kx (peer, msg);
-}
-
-
-/**
- * Check if the encrypted message has the appropriate size.
- *
- * @param cls Closure (unused).
- * @param msg Message to check.
- *
- * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
- */
-static int
-check_encrypted (void *cls, const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+consider_peer_destroy (struct CadetPeer *cp)
 {
-    uint16_t size;
-    uint16_t minimum_size;
-
-    size = ntohs (msg->header.size);
-    minimum_size = sizeof (struct GNUNET_CADET_TunnelEncryptedMessage)
-                   + sizeof (struct GNUNET_MessageHeader);
-
-    if (size < minimum_size)
-    {
-        GNUNET_break_op (0);
-        return GNUNET_NO;
-    }
-    return GNUNET_YES;
-}
+  struct GNUNET_TIME_Relative exp;
 
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED.
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_encrypted (void *cls, const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_encrypted (peer, msg);
-}
-
-
-/**
- * To be called on core init/fail.
- *
- * @param cls Closure (config)
- * @param identity The public identity of this peer.
- */
-static void
-core_init_notify (void *cls,
-                  const struct GNUNET_PeerIdentity *identity);
-
-
-static void
-connect_to_core (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-    struct GNUNET_MQ_MessageHandler core_handlers[] = {
-        GNUNET_MQ_hd_var_size (create,
-                               GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE,
-                               struct GNUNET_CADET_ConnectionCreateMessage,
-                               NULL),
-        GNUNET_MQ_hd_fixed_size (confirm,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK,
-                                 struct GNUNET_CADET_ConnectionCreateAckMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (broken,
-                                GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN,
-                                struct GNUNET_CADET_ConnectionBrokenMessage,
-                                NULL),
-        GNUNET_MQ_hd_fixed_size (destroy,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY,
-                                 struct GNUNET_CADET_ConnectionDestroyMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (ack,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK,
-                                 struct GNUNET_CADET_ConnectionEncryptedAckMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (poll,
-                                 GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL,
-                                 struct GNUNET_CADET_ConnectionHopByHopPollMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (kx,
-                                 GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX,
-                                 struct GNUNET_CADET_TunnelKeyExchangeMessage,
-                                 NULL),
-        GNUNET_MQ_hd_var_size (encrypted,
-                               GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED,
-                               struct GNUNET_CADET_TunnelEncryptedMessage,
-                               NULL),
-        GNUNET_MQ_handler_end ()
-    };
-    core_handle = GNUNET_CORE_connect (c, NULL,
-                                       &core_init_notify,
-                                       &core_connect_handler,
-                                       &core_disconnect_handler,
-                                       core_handlers);
-}
-
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-
-/**
- * To be called on core init/fail.
- *
- * @param cls Closure (config)
- * @param identity The public identity of this peer.
- */
-static void
-core_init_notify (void *cls,
-                  const struct GNUNET_PeerIdentity *core_identity)
-{
-    const struct GNUNET_CONFIGURATION_Handle *c = cls;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Core init\n");
-    if (0 != memcmp (core_identity, &my_full_id, sizeof (my_full_id)))
-    {
-        LOG (GNUNET_ERROR_TYPE_ERROR, _("Wrong CORE service\n"));
-        LOG (GNUNET_ERROR_TYPE_ERROR, " core id %s\n", GNUNET_i2s (core_identity));
-        LOG (GNUNET_ERROR_TYPE_ERROR, " my id %s\n", GNUNET_i2s (&my_full_id));
-        GNUNET_CORE_disconnect (core_handle);
-        connect_to_core (c);
-        return;
-    }
-    GML_start ();
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if (NULL != cp->t)
+    return; /* still relevant! */
+  if (NULL != cp->core_mq)
+    return; /* still relevant! */
+  if (0 != GNUNET_CONTAINER_multishortmap_size (cp->connections))
+    return; /* still relevant! */
+  if ( (NULL != cp->path_heap) &&
+       (0 < GNUNET_CONTAINER_heap_get_size (cp->path_heap)) )
+  {
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PATH_TIMEOUT,
+                                                     &drop_paths,
+                                                     cp);
+    return;
+  }
+  if (0 != cp->num_paths)
+    return; /* still relevant! */
+  if (NULL != cp->hello)
+  {
+    /* relevant only until HELLO expires */
+    exp = GNUNET_TIME_absolute_get_remaining (GNUNET_HELLO_get_last_expiration (cp->hello));
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (exp,
+                                                     &destroy_peer,
+                                                     cp);
+    return;
+  }
+  cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PEER_TIMEOUT,
+                                                   &destroy_peer,
+                                                   cp);
 }
 
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
 /**
- * Get priority for a queued message.
- *
- * @param q Queued message
- *
- * @return CORE priority to use.
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
  *
- * FIXME make static
- * FIXME use when sending
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
  */
-enum GNUNET_CORE_Priority
-get_priority (struct CadetPeerQueue *q)
-{
-    enum GNUNET_CORE_Priority low;
-    enum GNUNET_CORE_Priority high;
-
-    if (NULL == q)
-    {
-        GNUNET_break (0);
-        return GNUNET_CORE_PRIO_BACKGROUND;
-    }
-
-    /* Relayed traffic has lower priority, our own traffic has higher */
-    if (NULL == q->c || GNUNET_NO == GCC_is_origin (q->c, q->c_fwd))
+void
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message queue for peer %s is now %p\n",
+       GCP_2s (cp),
+       mq);
+  cp->core_mq = mq;
+  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head, *next;
+       NULL != mqm;
+       mqm = next)
+  {
+    /* Save next pointer in case mqm gets freed by the callback */
+    next = mqm->next;
+    if (NULL == mq)
     {
-        low = GNUNET_CORE_PRIO_BEST_EFFORT;
-        high = GNUNET_CORE_PRIO_URGENT;
+      if (NULL != mqm->env)
+      {
+        GNUNET_MQ_discard (mqm->env);
+        mqm->env = NULL;
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_SYSERR);
+      }
+      else
+      {
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_NO);
+      }
     }
     else
     {
-        low = GNUNET_CORE_PRIO_URGENT;
-        high = GNUNET_CORE_PRIO_CRITICAL_CONTROL;
+      GNUNET_assert (NULL == mqm->env);
+      mqm->cb (mqm->cb_cls,
+               GNUNET_YES);
     }
+  }
+  if ( (NULL != mq) ||
+       (NULL != cp->t) )
+    consider_peer_activate (cp);
+  else
+    consider_peer_destroy (cp);
 
-    /* Bulky payload has lower priority, control traffic has higher. */
-    if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == q->type)
-        return low;
-    return high;
-}
-
-
-/**
- * Cancel all messages queued to CORE MQ towards this peer.
- *
- * @param peer Peer towards which to cancel all messages.
- */
-static void
-cancel_queued_messages (struct CadetPeer *peer)
-{
-    while (NULL != peer->q_head)
-    {
-        struct CadetPeerQueue *q;
-
-        q = peer->q_head;
-        call_peer_cont (q, GNUNET_NO);
-        GNUNET_free (q);
-    }
-}
-
-
-/**
- * Destroy the peer_info and free any allocated resources linked to it
- *
- * @param peer The peer_info to destroy.
- * @return #GNUNET_OK on success
- */
-static int
-peer_destroy (struct CadetPeer *peer)
-{
-    struct GNUNET_PeerIdentity id;
-    struct CadetPeerPath *p;
-    struct CadetPeerPath *nextp;
-
-    GNUNET_PEER_resolve (peer->id, &id);
-    GNUNET_PEER_change_rc (peer->id, -1);
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "destroying peer %s\n",
-         GNUNET_i2s (&id));
-
-    if (GNUNET_YES !=
-            GNUNET_CONTAINER_multipeermap_remove (peers, &id, peer))
-    {
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_WARNING, " peer not in peermap!!\n");
-    }
-    GCP_stop_search (peer);
-    p = peer->path_head;
-    while (NULL != p)
-    {
-        nextp = p->next;
-        GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                     peer->path_tail,
-                                     p);
-        path_destroy (p);
-        p = nextp;
-    }
-    if (NULL != peer->tunnel)
-        GCT_destroy_empty (peer->tunnel);
-    if (NULL != peer->connections)
-    {
-        GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (peer->connections));
-        GNUNET_CONTAINER_multishortmap_destroy (peer->connections);
-        peer->connections = NULL;
-    }
-    if (NULL != peer->hello_offer)
-    {
-        GNUNET_TRANSPORT_offer_hello_cancel (peer->hello_offer);
-        peer->hello_offer = NULL;
-    }
-    if (NULL != peer->connectivity_suggestion)
-    {
-        GNUNET_ATS_connectivity_suggest_cancel (peer->connectivity_suggestion);
-        peer->connectivity_suggestion = NULL;
-    }
-    cancel_queued_messages (peer);
+  if ( (NULL != mq) &&
+       (NULL != cp->t) )
+  {
+    /* have a new, direct path to the target, notify tunnel */
+    struct CadetPeerPath *path;
 
-    GNUNET_free_non_null (peer->hello);
-    GNUNET_free (peer);
-    return GNUNET_OK;
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    GCT_consider_path (cp->t,
+                       path,
+                       0);
+  }
 }
 
 
 /**
- * Iterator over peer hash map entries to destroy the peer during in_shutdown.
+ * Debug function should NEVER return true in production code, useful to
+ * simulate losses for testcases.
  *
- * @param cls closure
- * @param key current key code
- * @param value value in the hash map
- * @return #GNUNET_YES if we should continue to iterate,
- *         #GNUNET_NO if not.
+ * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
  */
 static int
-shutdown_peer (void *cls,
-               const struct GNUNET_PeerIdentity *key,
-               void *value)
+should_I_drop (void)
 {
-    struct CadetPeer *p = value;
-    struct CadetTunnel *t = p->tunnel;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  shutting down %s\n", GCP_2s (p));
-    if (NULL != t)
-        GCT_destroy (t);
-    p->tunnel = NULL;
-    peer_destroy (p);
+  if (0 == drop_percent)
+    return GNUNET_NO;
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                101) < drop_percent)
     return GNUNET_YES;
+  return GNUNET_NO;
 }
 
 
 /**
- * Check if peer is searching for a path (either active or delayed search).
- *
- * @param peer Peer to check
- * @return #GNUNET_YES if there is a search active.
- *         #GNUNET_NO otherwise.
- */
-static int
-is_searching (const struct CadetPeer *peer)
-{
-    return ( (NULL == peer->search_h) &&
-             (NULL == peer->search_delayed) ) ?
-           GNUNET_NO : GNUNET_YES;
-}
-
-
-/**
- * @brief Start a search for a peer.
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
  *
- * @param cls Closure (Peer to search for).
+ * @param cls the `struct CadetPeeer` where we made progress
  */
 static void
-delayed_search (void *cls)
-{
-    struct CadetPeer *peer = cls;
-
-    peer->search_delayed = NULL;
-    GCC_check_connections ();
-    GCP_start_search (peer);
-    GCC_check_connections ();
-}
+mqm_send_done (void *cls);
 
 
 /**
- * Returns if peer is used (has a tunnel or is neighbor).
+ * Transmit current envelope from this @a mqm.
  *
- * @param peer Peer to check.
- * @return #GNUNET_YES if peer is in use.
+ * @param mqm mqm to transmit message for now
  */
-static int
-peer_is_used (struct CadetPeer *peer)
-{
-    struct CadetPeerPath *p;
-
-    if (NULL != peer->tunnel)
-        return GNUNET_YES;
-
-    for (p = peer->path_head; NULL != p; p = p->next)
-    {
-        if (p->length < 3)
-            return GNUNET_YES;
-    }
-    return GNUNET_NO;
-}
-
-
-/**
- * Iterator over all the peers to get the oldest timestamp.
- *
- * @param cls Closure (unsued).
- * @param key ID of the peer.
- * @param value Peer_Info of the peer.
- */
-static int
-peer_get_oldest (void *cls,
-                 const struct GNUNET_PeerIdentity *key,
-                 void *value)
+static void
+mqm_execute (struct GCP_MessageQueueManager *mqm)
 {
-    struct CadetPeer *p = value;
-    struct GNUNET_TIME_Absolute *abs = cls;
-
-    /* Don't count active peers */
-    if (GNUNET_YES == peer_is_used (p))
-        return GNUNET_YES;
+  struct CadetPeer *cp = mqm->cp;
 
-    if (abs->abs_value_us < p->last_contact.abs_value_us)
-        abs->abs_value_us = p->last_contact.abs_value_us;
-
-    return GNUNET_YES;
+  /* Move ready pointer to the next entry that might be ready. */
+  if ( (mqm == cp->mqm_ready_ptr) &&
+       (NULL != mqm->next) )
+    cp->mqm_ready_ptr = mqm->next;
+  /* Move entry to the end of the DLL, to be fair. */
+  if (mqm != cp->mqm_tail)
+  {
+    GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                                 cp->mqm_tail,
+                                 mqm);
+    GNUNET_CONTAINER_DLL_insert_tail (cp->mqm_head,
+                                      cp->mqm_tail,
+                                      mqm);
+  }
+  cp->mqm_ready_counter--;
+  if (GNUNET_YES == should_I_drop ())
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "DROPPING message to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_discard (mqm->env);
+    mqm->env = NULL;
+    mqm_send_done (cp);
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Sending to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_send (cp->core_mq,
+                    mqm->env);
+    mqm->env = NULL;
+  }
+  mqm->cb (mqm->cb_cls,
+           GNUNET_YES);
 }
 
 
 /**
- * Iterator over all the peers to remove the oldest entry.
+ * Find the next ready message in the queue (starting
+ * the search from the `cp->mqm_ready_ptr`) and if possible
+ * execute the transmission.
  *
- * @param cls Closure (unsued).
- * @param key ID of the peer.
- * @param value Peer_Info of the peer.
- */
-static int
-peer_timeout (void *cls,
-              const struct GNUNET_PeerIdentity *key,
-              void *value)
-{
-    struct CadetPeer *p = value;
-    struct GNUNET_TIME_Absolute *abs = cls;
-
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "peer %s timeout\n", GNUNET_i2s (key));
-
-    if (p->last_contact.abs_value_us == abs->abs_value_us &&
-            GNUNET_NO == peer_is_used (p))
-    {
-        peer_destroy (p);
-        return GNUNET_NO;
-    }
-    return GNUNET_YES;
-}
-
-
-/**
- * Delete oldest unused peer.
+ * @param cp peer to try to send the next ready message to
  */
 static void
-peer_delete_oldest (void)
+send_next_ready (struct CadetPeer *cp)
 {
-    struct GNUNET_TIME_Absolute abs;
-
-    abs = GNUNET_TIME_UNIT_FOREVER_ABS;
-
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &peer_get_oldest,
-                                           &abs);
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &peer_timeout,
-                                           &abs);
-}
-
+  struct GCP_MessageQueueManager *mqm;
 
-/**
- * Choose the best (yet unused) path towards a peer,
- * considering the tunnel properties.
- *
- * @param peer The destination peer.
- * @return Best current known path towards the peer, if any.
- */
-static struct CadetPeerPath *
-peer_get_best_path (const struct CadetPeer *peer)
-{
-    struct CadetPeerPath *best_p;
-    struct CadetPeerPath *p;
-    unsigned int best_cost;
-    unsigned int cost;
-
-    best_cost = UINT_MAX;
-    best_p = NULL;
-    for (p = peer->path_head; NULL != p; p = p->next)
-    {
-        if (GNUNET_NO == path_is_valid (p))
-            continue; /* Don't use invalid paths. */
-        if (GNUNET_YES == GCT_is_path_used (peer->tunnel, p))
-            continue; /* If path is already in use, skip it. */
-
-        if ((cost = GCT_get_path_cost (peer->tunnel, p)) < best_cost)
-        {
-            best_cost = cost;
-            best_p = p;
-        }
-    }
-    return best_p;
+  if (0 == cp->mqm_ready_counter)
+    return;
+  while ( (NULL != (mqm = cp->mqm_ready_ptr)) &&
+          (NULL == mqm->env) )
+    cp->mqm_ready_ptr = mqm->next;
+  if (NULL == mqm)
+    return; /* nothing to do */
+  mqm_execute (mqm);
 }
 
 
 /**
- * Function to process paths received for a new peer addition. The recorded
- * paths form the initial tunnel, which can be optimized later.
- * Called on each result obtained for the DHT search.
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
  *
- * @param cls Closure (peer towards a path has been found).
- * @param path Path created from the DHT query. Will be freed afterwards.
+ * @param cls the `struct CadetPeeer` where we made progress
  */
 static void
-search_handler (void *cls, const struct CadetPeerPath *path)
+mqm_send_done (void *cls)
 {
-    struct CadetPeer *peer = cls;
-    unsigned int connection_count;
-
-    GCC_check_connections ();
-    GCP_add_path_to_all (path, GNUNET_NO);
-
-    /* Count connections */
-    connection_count = GCT_count_connections (peer->tunnel);
+  struct CadetPeer *cp = cls;
 
-    /* If we already have our minimum (or more) connections, it's enough */
-    if (CONNECTIONS_PER_TUNNEL <= connection_count)
-    {
-        GCC_check_connections ();
-        return;
-    }
-
-    if (CADET_TUNNEL_SEARCHING == GCT_get_cstate (peer->tunnel))
-    {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " ... connect!\n");
-        GCP_connect (peer);
-    }
-    GCC_check_connections ();
-}
-
-
-/**
- * Test if a message type is connection management traffic
- * or regular payload traffic.
- *
- * @param type Message type.
- *
- * @return #GNUNET_YES if connection management, #GNUNET_NO otherwise.
- */
-static int
-is_connection_management (uint16_t type)
-{
-    return type == GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK ||
-           type == GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending to peer %s completed\n",
+       GCP_2s (cp));
+  send_next_ready (cp);
 }
 
 
 /**
- * Debug function should NEVER return true in production code, useful to
- * simulate losses for testcases.
+ * Send the message in @a env to @a cp.
  *
- * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
+ * @param mqm the message queue manager to use for transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
-static int
-should_I_drop (void)
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env)
 {
-    if (0 == drop_percent)
-        return GNUNET_NO;
+  struct CadetPeer *cp = mqm->cp;
 
-    if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 101) < drop_percent)
-        return GNUNET_YES;
-
-    return GNUNET_NO;
+  GNUNET_assert (NULL != env);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queueing message to peer %s in MQM %p\n",
+       GCP_2s (cp),
+       mqm);
+  GNUNET_assert (NULL != cp->core_mq);
+  GNUNET_assert (NULL == mqm->env);
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  mqm->env = env;
+  cp->mqm_ready_counter++;
+  if (mqm != cp->mqm_ready_ptr)
+    cp->mqm_ready_ptr = cp->mqm_head;
+  if (1 == cp->mqm_ready_counter)
+    cp->mqm_ready_ptr = mqm;
+  if (0 != GNUNET_MQ_get_length (cp->core_mq))
+    return;
+  send_next_ready (cp);
 }
 
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
 /**
- * Call the continuation after a message has been sent or dropped.
+ * Function called to destroy a peer now.
  *
- * This funcion removes the message from the queue.
- *
- * @param q Queue handle.
- * @param sent #GNUNET_YES if was sent to CORE, #GNUNET_NO if dropped.
+ * @param cls NULL
+ * @param pid identity of the peer (unused)
+ * @param value the `struct CadetPeer` to clean up
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-call_peer_cont (struct CadetPeerQueue *q, int sent)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " core mq just sent %s\n", GC_m2s (q->type));
-    if (NULL != q->cont)
-    {
-        struct GNUNET_TIME_Relative wait_time;
-
-        wait_time = GNUNET_TIME_absolute_get_duration (q->queue_timestamp);
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             " calling callback on %s after %s\n",
-             GCC_2s (q->c),
-             GNUNET_STRINGS_relative_time_to_string (wait_time, GNUNET_NO));
-        q->cont (q->cont_cls,
-                 q->c, q->c_fwd, sent,
-                 q->type,
-                 q->payload_type,
-                 q->payload_id,
-                 q->size, wait_time);
-        q->cont = NULL;
-    }
-    GNUNET_CONTAINER_DLL_remove (q->peer->q_head, q->peer->q_tail, q);
-}
-
-
-/**
- * Function called by MQ when a message is sent to CORE.
- *
- * @param cls Closure (queue handle).
- */
-static void
-mq_sent (void *cls)
+static int
+destroy_iterator_cb (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
 {
-    struct CadetPeerQueue *q = cls;
+  struct CadetPeer *cp = value;
 
-    if (GNUNET_NO == q->management_traffic)
-    {
-        q->peer->queue_n--;
-    }
-    call_peer_cont (q, GNUNET_YES);
-    GNUNET_free (q);
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  destroy_peer (cp);
+  return GNUNET_OK;
 }
 
 
 /**
- * Finish the drop operation.
- *
- * @param cls queue entry to finish drop for
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
  */
-static void
-drop_cb (void *cls)
+void
+GCP_destroy_all_peers ()
 {
-  struct CadetPeerQueue *q = cls;
-
-  GNUNET_MQ_discard (q->env);
-  call_peer_cont (q, GNUNET_YES);
-  GNUNET_free (q);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all peers now\n");
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         &destroy_iterator_cb,
+                                         NULL);
 }
 
 
 /**
- * @brief Send a message to another peer (using CORE).
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
  *
- * @param peer Peer towards which to queue the message.
- * @param message Message to send.
- * @param payload_type Type of the message's payload, for debug messages.
- *                     0 if the message is a retransmission (unknown payload).
- *                     UINT16_MAX if the message does not have payload.
- * @param payload_id ID of the payload (MID, ACK #, etc)
- * @param c Connection this message belongs to (can be NULL).
- * @param fwd Is this a message going root->dest? (FWD ACK are NOT FWD!)
- * @param cont Continuation to be called once CORE has sent the message.
- * @param cont_cls Closure for @c cont.
- *
- * @return A handle to the message in the queue or NULL (if dropped).
+ * @param cp peer to drop paths to
  */
-struct CadetPeerQueue *
-GCP_send (struct CadetPeer *peer,
-          const struct GNUNET_MessageHeader *message,
-          uint16_t payload_type,
-          struct CadetEncryptedMessageIdentifier payload_id,
-          struct CadetConnection *c,
-          int fwd,
-          GCP_sent cont,
-          void *cont_cls)
+void
+GCP_drop_owned_paths (struct CadetPeer *cp)
 {
-    struct CadetPeerQueue *q;
-    uint16_t type;
-    uint16_t size;
-
-    GCC_check_connections ();
-    type = ntohs (message->type);
-    size = ntohs (message->size);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "que %s (%s %4u) on conn %s (%p) %s towards %s (size %u)\n",
-         GC_m2s (type), GC_m2s (payload_type),
-         ntohl (payload_id.pid),
-         GCC_2s (c), c, GC_f2s (fwd), GCP_2s (peer), size);
-
-    if (NULL == peer->connections)
-    {
-        /* We are not connected to this peer, ignore request. */
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_INFO, "%s not a neighbor\n", GCP_2s (peer));
-        GNUNET_STATISTICS_update (stats, "# messages dropped due to wrong hop", 1,
-                                  GNUNET_NO);
-        return NULL;
-    }
+  struct CadetPeerPath *path;
 
-    q = GNUNET_new (struct CadetPeerQueue);
-    q->env = GNUNET_MQ_msg_copy (message);
-    q->peer = peer;
-    q->cont = cont;
-    q->cont_cls = cont_cls;
-    q->queue_timestamp = GNUNET_TIME_absolute_get ();
-    q->management_traffic = is_connection_management (type);
-    q->type = type;
-    q->size = size;
-    q->payload_type = payload_type;
-    q->payload_id = payload_id;
-    q->c = c;
-    q->c_fwd = fwd;
-    GNUNET_MQ_notify_sent (q->env, &mq_sent, q);
-    GNUNET_CONTAINER_DLL_insert (peer->q_head, peer->q_tail, q);
-
-    if (GNUNET_YES == q->management_traffic)
-    {
-        GNUNET_MQ_send (peer->core_mq, q->env);  // FIXME implement "_urgent", use
-    }
-    else
-    {
-        if (GNUNET_YES == should_I_drop ())
-        {
-            LOG (GNUNET_ERROR_TYPE_WARNING,
-                 "DD %s (%s %u) on conn %s %s (random drop for testing)\n",
-                 GC_m2s (q->type),
-                 GC_m2s (q->payload_type),
-                 ntohl (q->payload_id.pid),
-                 GCC_2s (c),
-                 GC_f2s (q->c_fwd));
-            q->drop_task = GNUNET_SCHEDULER_add_now (&drop_cb,
-                                                     q);
-            return q;
-        }
-        GNUNET_MQ_send (peer->core_mq, q->env);
-        peer->queue_n++;
-    }
-
-    GCC_check_connections ();
-    return q;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all paths to %s\n",
+       GCP_2s (cp));
+  while (NULL != (path =
+                  GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+  cp->path_heap = NULL;
 }
 
 
 /**
- * Cancel sending a message. Message must have been sent with
- * #GCP_send before.  May not be called after the notify sent
- * callback has been called.
- *
- * It DOES call the continuation given to #GCP_send.
+ * Add an entry to the DLL of all of the paths that this peer is on.
  *
- * @param q Queue handle to cancel
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_send_cancel (struct CadetPeerQueue *q)
-{
-  if (NULL != q->drop_task)
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off)
+{
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (entry->path,
+                                                off));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Discovered that peer %s is on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  if (off >= cp->path_dll_length)
   {
-    GNUNET_SCHEDULER_cancel (q->drop_task);
-    q->drop_task = NULL;
-    GNUNET_MQ_discard (q->env);
+    unsigned int len = cp->path_dll_length;
+
+    GNUNET_array_grow (cp->path_heads,
+                       len,
+                       off + 4);
+    GNUNET_array_grow (cp->path_tails,
+                       cp->path_dll_length,
+                       off + 4);
   }
-  else
+  GNUNET_CONTAINER_DLL_insert (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  cp->off_sum += off;
+  cp->num_paths++;
+
+  /* If we have a tunnel to this peer, tell the tunnel that there is a
+     new path available. */
+  if (NULL != cp->t)
+    GCT_consider_path (cp->t,
+                       entry->path,
+                       off);
+
+  if ( (NULL != cp->search_h) &&
+       (DESIRED_CONNECTIONS_PER_TUNNEL <= cp->num_paths) )
   {
-    GNUNET_MQ_send_cancel (q->env);
+    /* Now I have enough paths, stop search */
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  if (NULL != cp->destroy_task)
+  {
+    /* paths changed, this resets the destroy timeout counter
+       and aborts a destroy task that may no longer be valid
+       to have (as we now have more paths via this peer). */
+    consider_peer_destroy (cp);
   }
-  call_peer_cont (q, GNUNET_NO);
-  GNUNET_free (q);
 }
 
 
 /**
- * Initialize the peer subsystem.
+ * Remove an entry from the DLL of all of the paths that this peer is on.
  *
- * @param c Configuration.
- */
-void
-GCP_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-    cfg = c;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "GCP_init\n");
-    in_shutdown = GNUNET_NO;
-    peers = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_NO);
-    if (GNUNET_OK !=
-            GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_PEERS",
-                    &max_peers))
-    {
-        GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                                   "CADET", "MAX_PEERS", "USING DEFAULT");
-        max_peers = 1000;
-    }
-
-    if (GNUNET_OK !=
-            GNUNET_CONFIGURATION_get_value_number (c, "CADET", "DROP_PERCENT",
-                    &drop_percent))
-    {
-        drop_percent = 0;
-    }
-    else
-    {
-        LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-    }
-    ats_ch = GNUNET_ATS_connectivity_init (c);
-    connect_to_core (c);
-    if (NULL == core_handle)
-    {
-        GNUNET_break (0);
-        GNUNET_SCHEDULER_shutdown ();
-    }
-}
-
-
-/**
- * Shut down the peer subsystem.
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_shutdown (void)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Shutting down peer subsystem\n");
-    in_shutdown = GNUNET_YES;
-    if (NULL != core_handle)
-    {
-        GNUNET_CORE_disconnect (core_handle);
-        core_handle = NULL;
-    }
-    GNUNET_PEER_change_rc (myid, -1);
-    /* With MQ API, CORE calls the disconnect handler for every peer
-     * after calling GNUNET_CORE_disconnect, shutdown must occur *after* that.
-     */
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &shutdown_peer,
-                                           NULL);
-    if (NULL != ats_ch)
-    {
-        GNUNET_ATS_connectivity_done (ats_ch);
-        ats_ch = NULL;
-    }
-    GNUNET_CONTAINER_multipeermap_destroy (peers);
-    peers = NULL;
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id, int create)
-{
-    struct CadetPeer *peer;
-
-    peer = GNUNET_CONTAINER_multipeermap_get (peers, peer_id);
-    if (NULL == peer)
-    {
-        peer = GNUNET_new (struct CadetPeer);
-        if (GNUNET_CONTAINER_multipeermap_size (peers) > max_peers)
-        {
-            peer_delete_oldest ();
-        }
-        GNUNET_assert (GNUNET_OK ==
-                       GNUNET_CONTAINER_multipeermap_put (peers,
-                               peer_id,
-                               peer,
-                               GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        peer->id = GNUNET_PEER_intern (peer_id);
-    }
-    peer->last_contact = GNUNET_TIME_absolute_get ();
-
-    return peer;
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer Short identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get_short (const GNUNET_PEER_Id peer, int create)
-{
-    return GCP_get (GNUNET_PEER_resolve2 (peer), create);
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing knowledge about peer %s beging on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  GNUNET_CONTAINER_DLL_remove (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  GNUNET_assert (0 < cp->num_paths);
+  cp->off_sum -= off;
+  cp->num_paths--;
+  if ( (NULL == cp->core_mq) &&
+       (NULL != cp->t) &&
+       (NULL == cp->search_h) &&
+       (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
+    cp->search_h
+      = GCD_search (&cp->pid);
+  if (NULL == cp->destroy_task)
+  {
+    /* paths changed, we might now be ready for destruction, check again */
+    consider_peer_destroy (cp);
+  }
 }
 
 
 /**
- * Function called once #GNUNET_TRANSPORT_offer_hello() is done.
- * Marks the operation as finished.
+ * Prune down the number of paths to this peer, we seem to
+ * have way too many.
  *
- * @param cls Closure (our `struct CadetPeer`).
+ * @param cls the `struct CadetPeer` to maintain the path heap for
  */
 static void
-hello_offer_done (void *cls)
+path_heap_cleanup (void *cls)
 {
-    struct CadetPeer *peer = cls;
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *root;
 
-    peer->hello_offer = NULL;
+  cp->heap_cleanup_task = NULL;
+  while (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+         2 * DESIRED_CONNECTIONS_PER_TUNNEL)
+  {
+    /* Now we have way too many, drop least desirable UNLESS it is in use!
+       (Note that this intentionally keeps highly desireable, but currently
+       unused paths around in the hope that we might be able to switch, even
+       if the number of paths exceeds the threshold.) */
+    root = GNUNET_CONTAINER_heap_peek (cp->path_heap);
+    GNUNET_assert (NULL != root);
+    if (NULL !=
+        GCPP_get_connection (root,
+                             cp,
+                             GCPP_get_length (root) - 1))
+      break; /* can't fix */
+    /* Got plenty of paths to this destination, and this is a low-quality
+       one that we don't care about. Allow it to die. */
+    GNUNET_assert (root ==
+                   GNUNET_CONTAINER_heap_remove_root (cp->path_heap));
+    GCPP_release (root);
+  }
 }
 
 
 /**
- * Try to establish a new connection to this peer (in its tunnel).
- * If the peer doesn't have any path to it yet, try to get one.
- * If the peer already has some path, send a CREATE CONNECTION towards it.
+ * Try adding a @a path to this @a peer.  If the peer already
+ * has plenty of paths, return NULL.
  *
- * @param peer Peer to connect to.
+ * @param cp peer to which the @a path leads to
+ * @param path a path looking for an owner; may not be fully initialized yet!
+ * @param off offset of @a cp in @a path
+ * @param force force attaching the path
+ * @return NULL if this peer does not care to become a new owner,
+ *         otherwise the node in the peer's path heap for the @a path.
  */
-void
-GCP_connect (struct CadetPeer *peer)
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force)
 {
-    struct CadetTunnel *t;
-    struct CadetPeerPath *path;
-    struct CadetConnection *c;
-    int rerun_search;
-
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "peer_connect towards %s\n",
-         GCP_2s (peer));
-    /* If we have a current hello, try to connect using it. */
-    GCP_try_connect (peer);
+  GNUNET_CONTAINER_HeapCostType desirability;
+  struct CadetPeerPath *root;
+  GNUNET_CONTAINER_HeapCostType root_desirability;
+  struct GNUNET_CONTAINER_HeapNode *hn;
 
-    t = peer->tunnel;
-    c = NULL;
-    rerun_search = GNUNET_NO;
-
-    if (NULL != peer->path_head)
+  GNUNET_assert (off == GCPP_get_length (path) - 1);
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (path,
+                                                off));
+  if (NULL == cp->path_heap)
+  {
+    /* #GCP_drop_owned_paths() was already called, we cannot take new ones! */
+    GNUNET_assert (GNUNET_NO == force);
+    return NULL;
+  }
+  desirability = GCPP_get_desirability (path);
+  if (GNUNET_NO == force)
+  {
+    /* FIXME: desirability is not yet initialized; tricky! */
+    if (GNUNET_NO ==
+        GNUNET_CONTAINER_heap_peek2 (cp->path_heap,
+                                     (void **) &root,
+                                     &root_desirability))
     {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  some path exists\n");
-        path = peer_get_best_path (peer);
-        if (NULL != path)
-        {
-            char *s;
-
-            s = path_2s (path);
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  path to use: %s\n", s);
-            GNUNET_free (s);
-
-            c = GCT_use_path (t, path);
-            if (NULL == c)
-            {
-                /* This case can happen when the path includes a first hop that is
-                 * not yet known to be connected.
-                 *
-                 * This happens quite often during testing when running cadet
-                 * under valgrind: core connect notifications come very late
-                 * and the DHT result has already come and created a valid
-                 * path.  In this case, the peer->connections
-                 * hashmaps will be NULL and tunnel_use_path will not be able
-                 * to create a connection from that path.
-                 *
-                 * Re-running the DHT GET should give core time to callback.
-                 *
-                 * GCT_use_path -> GCC_new -> register_neighbors takes care of
-                 * updating statistics about this issue.
-                 */
-                rerun_search = GNUNET_YES;
-            }
-            else
-            {
-                GCC_send_create (c);
-                return;
-            }
-        }
-        else
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  but is NULL, all paths are in use\n");
-        }
+      root = NULL;
+      root_desirability = 0;
     }
 
-    if (GNUNET_YES == rerun_search)
+    if ( (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) &&
+         (desirability < root_desirability) )
     {
-        struct GNUNET_TIME_Relative delay;
-
-        GCP_stop_search (peer);
-        delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100);
-        peer->search_delayed = GNUNET_SCHEDULER_add_delayed (delay,
-                               &delayed_search,
-                               peer);
-        GCC_check_connections ();
-        return;
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Decided to not attach path %p to peer %s due to undesirability\n",
+           GCPP_2s (path),
+           GCP_2s (cp));
+      return NULL;
     }
+  }
 
-    if (GNUNET_NO == is_searching (peer))
-        GCP_start_search (peer);
-    GCC_check_connections ();
-}
-
-
-/**
- * Chech whether there is a direct (core level)  connection to peer.
- *
- * @param peer Peer to check.
- *
- * @return #GNUNET_YES if there is a direct connection.
- */
-int
-GCP_is_neighbor (const struct CadetPeer *peer)
-{
-    struct CadetPeerPath *path;
-
-    if (NULL == peer->connections)
-        return GNUNET_NO;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Attaching path %s to peer %s (%s)\n",
+       GCPP_2s (path),
+       GCP_2s (cp),
+       (GNUNET_NO == force) ? "desirable" : "forced");
 
-    for (path = peer->path_head; NULL != path; path = path->next)
-    {
-        if (3 > path->length)
-            return GNUNET_YES;
-    }
+  /* Yes, we'd like to add this path, add to our heap */
+  hn = GNUNET_CONTAINER_heap_insert (cp->path_heap,
+                                     path,
+                                     desirability);
 
-    /* Is not a neighbor but connections is not NULL, probably disconnecting */
-    return GNUNET_NO;
+  /* Consider maybe dropping other paths because of the new one */
+  if ( (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+        2 * DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (NULL != cp->heap_cleanup_task) )
+    cp->heap_cleanup_task = GNUNET_SCHEDULER_add_now (&path_heap_cleanup,
+                                                      cp);
+  return hn;
 }
 
 
 /**
- * Create and initialize a new tunnel towards a peer, in case it has none.
- * In case the peer already has a tunnel, nothing is done.
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
  *
- * Does not generate any traffic, just creates the local data structures.
- *
- * @param peer Peer towards which to create the tunnel.
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
  */
 void
-GCP_add_tunnel (struct CadetPeer *peer)
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn)
 {
-    GCC_check_connections ();
-    if (NULL != peer->tunnel)
-        return;
-    peer->tunnel = GCT_new (peer);
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Detatching path %s from peer %s\n",
+       GCPP_2s (path),
+       GCP_2s (cp));
+  GNUNET_assert (path ==
+                 GNUNET_CONTAINER_heap_remove_node (hn));
 }
 
 
 /**
- * Add a connection to a neighboring peer.
- *
- * Store that the peer is the first hop of the connection in one
- * direction and that on peer disconnect the connection must be
- * notified and destroyed, for it will no longer be valid.
+ * Add a @a connection to this @a cp.
  *
- * @param peer Peer to add connection to.
- * @param c Connection to add.
- * @param pred #GNUNET_YES if we are predecessor, #GNUNET_NO if we are successor
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
  */
 void
-GCP_add_connection (struct CadetPeer *peer,
-                    struct CadetConnection *c,
-                    int pred)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "adding connection %s\n",
-         GCC_2s (c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "to peer %s\n",
-         GCP_2s (peer));
-    GNUNET_assert (NULL != peer->connections);
-    GNUNET_assert (GNUNET_OK ==
-                   GNUNET_CONTAINER_multishortmap_put (peer->connections,
-                                                       &GCC_get_id (c)->connection_of_tunnel,
-                                                       c,
-                                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Peer %s has now %u connections.\n",
-         GCP_2s (peer),
-         GNUNET_CONTAINER_multishortmap_size (peer->connections));
-}
-
-
-/**
- * Add the path to the peer and update the path used to reach it in case this
- * is the shortest.
- *
- * @param peer Destination peer to add the path to.
- * @param path New path to add. Last peer must be @c peer.
- *             Path will be either used of freed if already known.
- * @param trusted Do we trust that this path is real?
- *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
- */
-struct CadetPeerPath *
-GCP_add_path (struct CadetPeer *peer,
-              struct CadetPeerPath *path,
-              int trusted)
-{
-    struct CadetPeerPath *aux;
-    unsigned int l;
-    unsigned int l2;
-
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "adding path [%u] to peer %s\n",
-         path->length, GCP_2s (peer));
-
-    if (NULL == peer || NULL == path
-            || path->peers[path->length - 1] != peer->id)
-    {
-        GNUNET_break (0);
-        path_destroy (path);
-        return NULL;
-    }
-
-    for (l = 1; l < path->length; l++)
-    {
-        if (path->peers[l] == myid)
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, " shortening path by %u\n", l);
-            for (l2 = 0; l2 < path->length - l; l2++)
-            {
-                path->peers[l2] = path->peers[l + l2];
-            }
-            path->length -= l;
-            l = 1;
-            path->peers = GNUNET_realloc (path->peers,
-                                          path->length * sizeof (GNUNET_PEER_Id));
-        }
-    }
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " final length: %u\n", path->length);
-
-    if (2 >= path->length && GNUNET_NO == trusted)
-    {
-        /* Only allow CORE to tell us about direct paths */
-        path_destroy (path);
-        return NULL;
-    }
-
-    l = path_get_length (path);
-    if (0 == l)
-    {
-        path_destroy (path);
-        return NULL;
-    }
-
-    GNUNET_assert (peer->id == path->peers[path->length - 1]);
-    for (aux = peer->path_head; aux != NULL; aux = aux->next)
-    {
-        l2 = path_get_length (aux);
-        if (l2 > l)
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  added\n");
-            GNUNET_CONTAINER_DLL_insert_before (peer->path_head,
-                                                peer->path_tail, aux, path);
-            goto finish;
-        }
-        else
-        {
-            if (l2 == l && memcmp (path->peers, aux->peers, l) == 0)
-            {
-                LOG (GNUNET_ERROR_TYPE_DEBUG, "  already known\n");
-                path_destroy (path);
-                return aux;
-            }
-        }
-    }
-    GNUNET_CONTAINER_DLL_insert_tail (peer->path_head,
-                                      peer->path_tail,
-                                      path);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  added last\n");
-
-finish:
-    if (NULL != peer->tunnel
-            && CONNECTIONS_PER_TUNNEL > GCT_count_connections (peer->tunnel)
-            && 2 < path->length) /* Direct paths are handled by core_connect */
-    {
-        GCP_connect (peer);
-    }
-    GCC_check_connections ();
-    return path;
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Adding connection %s to peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multishortmap_put (cp->connections,
+                                                     &GCC_get_id (cc)->connection_of_tunnel,
+                                                     cc,
+                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
 }
 
 
 /**
- * Add the path to the origin peer and update the path used to reach it in case
- * this is the shortest.
- * The path is given in peer_info -> destination, therefore we turn the path
- * upside down first.
- *
- * @param peer Peer to add the path to, being the origin of the path.
- * @param path New path to add after being inversed.
- *             Path will be either used or freed.
- * @param trusted Do we trust that this path is real?
+ * Remove a @a connection that went via this @a cp.
  *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
  */
-struct CadetPeerPath *
-GCP_add_path_to_origin (struct CadetPeer *peer,
-                        struct CadetPeerPath *path,
-                        int trusted)
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc)
 {
-    if (NULL == path)
-        return NULL;
-    path_invert (path);
-    return GCP_add_path (peer, path, trusted);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing connection %s from peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (cp->connections,
+                                                        &GCC_get_id (cc)->connection_of_tunnel,
+                                                        cc));
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Adds a path to the info of all the peers in the path
+ * Retrieve the CadetPeer stucture associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
  *
- * @param p Path to process.
- * @param confirmed Whether we know if the path works or not.
+ * @param peer_id Full identity of the peer.
+ * @param create #GNUNET_YES if a new peer should be created if unknown.
+ *               #GNUNET_NO to return NULL if peer is unknown.
+ * @return Existing or newly created peer structure.
+ *         NULL if unknown and not requested @a create
  */
-void
-GCP_add_path_to_all (const struct CadetPeerPath *p, int confirmed)
+struct CadetPeer *
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create)
 {
-    unsigned int i;
+  struct CadetPeer *cp;
 
-    /* TODO: invert and add to origin */
-    /* TODO: replace all "GCP_add_path" with this, make the other one static */
-    GCC_check_connections ();
-    for (i = 0; i < p->length && p->peers[i] != myid; i++) /* skip'em */ ;
-    for (i++; i < p->length; i++)
-    {
-        struct CadetPeer *peer;
-        struct CadetPeerPath *copy;
-
-        peer = GCP_get_short (p->peers[i], GNUNET_YES);
-        copy = path_duplicate (p);
-        copy->length = i + 1;
-        GCP_add_path (peer, copy, 3 > p->length ? GNUNET_NO : confirmed);
-    }
-    GCC_check_connections ();
+  cp = GNUNET_CONTAINER_multipeermap_get (peers,
+                                          peer_id);
+  if (NULL != cp)
+    return cp;
+  if (GNUNET_NO == create)
+    return NULL;
+  cp = GNUNET_new (struct CadetPeer);
+  cp->pid = *peer_id;
+  cp->connections = GNUNET_CONTAINER_multishortmap_create (32,
+                                                           GNUNET_YES);
+  cp->path_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_put (peers,
+                                                    &cp->pid,
+                                                    cp,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating peer %s\n",
+       GCP_2s (cp));
+  return cp;
 }
 
 
 /**
- * Remove any path to the peer that has the exact same peers as the one given.
+ * Obtain the peer identity for a `struct CadetPeer`.
  *
- * @param peer Peer to remove the path from.
- * @param path Path to remove. Is always destroyed .
+ * @param cp our peer handle
+ * @return the peer identity
  */
-void
-GCP_remove_path (struct CadetPeer *peer,
-                 struct CadetPeerPath *path)
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp)
 {
-    struct CadetPeerPath *iter;
-    struct CadetPeerPath *next;
-
-    GCC_check_connections ();
-    GNUNET_assert (myid == path->peers[0]);
-    GNUNET_assert (peer->id == path->peers[path->length - 1]);
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Removing path %p (%u) from %s\n",
-         path, path->length, GCP_2s (peer));
-
-    for (iter = peer->path_head; NULL != iter; iter = next)
-    {
-        next = iter->next;
-        if (0 == path_cmp (path, iter))
-        {
-            GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                         peer->path_tail,
-                                         iter);
-            if (iter != path)
-                path_destroy (iter);
-        }
-    }
-    path_destroy (path);
-    GCC_check_connections ();
+  return &cp->pid;
 }
 
 
 /**
- * Check that we are aware of a connection from a neighboring peer.
+ * Iterate over all known peers.
  *
- * @param peer Peer to the connection is with
- * @param c Connection that should be in the map with this peer.
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
  */
 void
-GCP_check_connection (const struct CadetPeer *peer,
-                      const struct CadetConnection *c)
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls)
 {
-  GNUNET_assert (NULL != peer);
-  GNUNET_assert (NULL != peer->connections);
-  return; // ????
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_contains_value (peer->connections,
-                                                                &GCC_get_id (c)->connection_of_tunnel,
-                                                                c));
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         iter,
+                                         cls);
 }
 
 
 /**
- * Remove a connection from a neighboring peer.
+ * Count the number of known paths toward the peer.
  *
- * @param peer Peer to remove connection from.
- * @param c Connection to remove.
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
  */
-void
-GCP_remove_connection (struct CadetPeer *peer,
-                       const struct CadetConnection *c)
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp)
 {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Removing connection %s\n",
-         GCC_2s (c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "from peer %s\n",
-         GCP_2s (peer));
-    if ( (NULL == peer) ||
-            (NULL == peer->connections) )
-        return;
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multishortmap_remove (peer->connections,
-                                                          &GCC_get_id (c)->connection_of_tunnel,
-                                                          c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Peer %s remains with %u connections.\n",
-         GCP_2s (peer),
-         GNUNET_CONTAINER_multishortmap_size (peer->connections));
+  return cp->num_paths;
 }
 
 
 /**
- * Start the DHT search for new paths towards the peer: we don't have
- * enough good connections.
+ * Iterate over the paths to a peer.
  *
- * @param peer Destination peer.
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-void
-GCP_start_search (struct CadetPeer *peer)
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls)
 {
-    const struct GNUNET_PeerIdentity *id;
-    struct CadetTunnel *t = peer->tunnel;
-
-    GCC_check_connections ();
-    if (NULL != peer->search_h)
-    {
-        GNUNET_break (0);
-        return;
-    }
-
-    if (NULL != peer->search_delayed)
-        GCP_stop_search (peer);
-
-    id = GNUNET_PEER_resolve2 (peer->id);
-    peer->search_h = GCD_search (id, &search_handler, peer);
-
-    if (NULL == t)
-    {
-        /* Why would we search for a peer with no tunnel towards it? */
-        GNUNET_break (0);
-        return;
-    }
-
-    if (CADET_TUNNEL_NEW == GCT_get_cstate (t)
-            || 0 == GCT_count_any_connections (t))
-    {
-        GCT_change_cstate (t, CADET_TUNNEL_SEARCHING);
-    }
-    GCC_check_connections ();
-}
+  unsigned int ret = 0;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Iterating over paths to peer %s%s\n",
+       GCP_2s (cp),
+       (NULL == cp->core_mq) ? "" : " including direct link");
+  if (NULL != cp->core_mq)
+  {
+    struct CadetPeerPath *path;
 
-/**
- * Stop the DHT search for new paths towards the peer: we already have
- * enough good connections.
- *
- * @param peer Destination peer.
- */
-void
-GCP_stop_search (struct CadetPeer *peer)
-{
-    GCC_check_connections ();
-    if (NULL != peer->search_h)
-    {
-        GCD_search_stop (peer->search_h);
-        peer->search_h = NULL;
-    }
-    if (NULL != peer->search_delayed)
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    ret++;
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  path,
+                  0))
+      return ret;
+  }
+  for (unsigned int i=0;i<cp->path_dll_length;i++)
+  {
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
+         NULL != pe;
+         pe = pe->next)
     {
-        GNUNET_SCHEDULER_cancel (peer->search_delayed);
-        peer->search_delayed = NULL;
+      ret++;
+      if (GNUNET_NO ==
+          callback (callback_cls,
+                    pe->path,
+                    i))
+        return ret;
     }
-    GCC_check_connections ();
-}
-
-
-/**
- * Get the Full ID of a peer.
- *
- * @param peer Peer to get from.
- *
- * @return Full ID of peer.
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (const struct CadetPeer *peer)
-{
-    return GNUNET_PEER_resolve2 (peer->id);
+  }
+  return ret;
 }
 
 
 /**
- * Get the Short ID of a peer.
- *
- * @param peer Peer to get from.
+ * Iterate over the paths to @a cp where
+ * @a cp is at distance @a dist from us.
  *
- * @return Short ID of peer.
+ * @param cp Peer to get path info.
+ * @param dist desired distance of @a cp to us on the path
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-GNUNET_PEER_Id
-GCP_get_short_id (const struct CadetPeer *peer)
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls)
 {
-    return peer->id;
-}
-
+  unsigned int ret = 0;
 
-/**
- * Set tunnel.
- *
- * If tunnel is NULL and there was a search active, stop it, as it's useless.
- *
- * @param peer Peer.
- * @param t Tunnel.
- */
-void
-GCP_set_tunnel (struct CadetPeer *peer, struct CadetTunnel *t)
-{
-    peer->tunnel = t;
-    if (NULL == t && GNUNET_YES == is_searching (peer))
-    {
-        GCP_stop_search (peer);
-    }
+  if (dist >= cp->path_dll_length)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Asked to look for paths at distance %u, but maximum for me is < %u\n",
+         dist,
+         cp->path_dll_length);
+    return 0;
+  }
+  for (struct CadetPeerPathEntry *pe = cp->path_heads[dist];
+       NULL != pe;
+       pe = pe->next)
+  {
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  pe->path,
+                  dist))
+      return ret;
+    ret++;
+  }
+  return ret;
 }
 
 
 /**
  * Get the tunnel towards a peer.
  *
- * @param peer Peer to get from.
- *
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
  * @return Tunnel towards peer.
  */
 struct CadetTunnel *
-GCP_get_tunnel (const struct CadetPeer *peer)
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create)
 {
-    if (NULL == peer)
-        return NULL;
-    return peer->tunnel;
+  if (NULL == cp)
+    return NULL;
+  if ( (NULL != cp->t) ||
+       (GNUNET_NO == create) )
+    return cp->t;
+  cp->t = GCT_create_tunnel (cp);
+  consider_peer_activate (cp);
+  return cp->t;
 }
 
 
 /**
- * Set the hello message.
+ * Hello offer was passed to the transport service. Mark it
+ * as done.
  *
- * @param peer Peer whose message to set.
- * @param hello Hello message.
+ * @param cls the `struct CadetPeer` where the offer completed
  */
-void
-GCP_set_hello (struct CadetPeer *peer,
-               const struct GNUNET_HELLO_Message *hello)
+static void
+hello_offer_done (void *cls)
 {
-    struct GNUNET_HELLO_Message *old;
-    size_t size;
+  struct CadetPeer *cp = cls;
 
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "set hello for %s\n", GCP_2s (peer));
-    if (NULL == hello)
-        return;
-
-    old = GCP_get_hello (peer);
-    if (NULL == old)
-    {
-        size = GNUNET_HELLO_size (hello);
-        peer->hello = GNUNET_malloc (size);
-        GNUNET_memcpy (peer->hello, hello, size);
-    }
-    else
-    {
-        peer->hello = GNUNET_HELLO_merge (old, hello);
-        GNUNET_free (old);
-    }
-    GCC_check_connections ();
+  cp->hello_offer = NULL;
 }
 
 
 /**
- * Get the hello message.
- *
- * @param peer Peer whose message to get.
+ * We got a HELLO for a @a peer, remember it, and possibly
+ * trigger adequate actions (like trying to connect).
  *
- * @return Hello message.
+ * @param cp the peer we got a HELLO for
+ * @param hello the HELLO to remember
  */
-struct GNUNET_HELLO_Message *
-GCP_get_hello (struct CadetPeer *peer)
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_HELLO_Message *hello)
 {
-    struct GNUNET_TIME_Absolute expiration;
-    struct GNUNET_TIME_Relative remaining;
-
-    if (NULL == peer->hello)
-        return NULL;
+  struct GNUNET_HELLO_Message *mrg;
 
-    expiration = GNUNET_HELLO_get_last_expiration (peer->hello);
-    remaining = GNUNET_TIME_absolute_get_remaining (expiration);
-    if (0 == remaining.rel_value_us)
-    {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " get - hello expired on %s\n",
-             GNUNET_STRINGS_absolute_time_to_string (expiration));
-        GNUNET_free (peer->hello);
-        peer->hello = NULL;
-    }
-    return peer->hello;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got %u byte HELLO for peer %s\n",
+       (unsigned int) GNUNET_HELLO_size (hello),
+       GCP_2s (cp));
+  if (NULL != cp->hello_offer)
+  {
+    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
+    cp->hello_offer = NULL;
+  }
+  if (NULL != cp->hello)
+  {
+    mrg = GNUNET_HELLO_merge (hello,
+                              cp->hello);
+    GNUNET_free (cp->hello);
+    cp->hello = mrg;
+  }
+  else
+  {
+    cp->hello = GNUNET_memdup (hello,
+                               GNUNET_HELLO_size (hello));
+  }
+  cp->hello_offer
+    = GNUNET_TRANSPORT_offer_hello (cfg,
+                                    GNUNET_HELLO_get_header (cp->hello) ,
+                                    &hello_offer_done,
+                                    cp);
+  /* New HELLO means cp's destruction time may change... */
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Try to connect to a peer on TRANSPORT level.
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
  *
- * @param peer Peer to whom to connect.
+ * @param cp the peer affected
+ * @param t the dead tunnel
  */
 void
-GCP_try_connect (struct CadetPeer *peer)
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t)
 {
-    struct GNUNET_HELLO_Message *hello;
-    struct GNUNET_MessageHeader *mh;
-
-    if (GNUNET_YES !=
-            GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                    "CADET",
-                    "DISABLE_TRY_CONNECT"))
-        return;
-    GCC_check_connections ();
-    if (GNUNET_YES == GCP_is_neighbor (peer))
-        return;
-    hello = GCP_get_hello (peer);
-    if (NULL == hello)
-        return;
-
-    mh = GNUNET_HELLO_get_header (hello);
-    if (NULL != peer->hello_offer)
-    {
-        GNUNET_TRANSPORT_offer_hello_cancel (peer->hello_offer);
-        peer->hello_offer = NULL;
-    }
-    peer->hello_offer = GNUNET_TRANSPORT_offer_hello (cfg,
-                        mh,
-                        &hello_offer_done,
-                        peer);
-    if (NULL == peer->connectivity_suggestion)
-        peer->connectivity_suggestion
-            = GNUNET_ATS_connectivity_suggest (ats_ch,
-                                               GCP_get_id (peer),
-                                               1);  /* strength */
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Dropping tunnel %s to peer %s\n",
+       GCT_2s (t),
+       GCP_2s (cp));
+  GNUNET_assert (cp->t == t);
+  cp->t = NULL;
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Notify a peer that a link between two other peers is broken. If any path
- * used that link, eliminate it.
+ * Test if @a cp has a core-level connection
  *
- * @param peer Peer affected by the change.
- * @param peer1 Peer whose link is broken.
- * @param peer2 Peer whose link is broken.
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
  */
-void
-GCP_notify_broken_link (struct CadetPeer *peer,
-                        const struct GNUNET_PeerIdentity *peer1,
-                        const struct GNUNET_PeerIdentity *peer2)
+int
+GCP_has_core_connection (struct CadetPeer *cp)
 {
-    struct CadetPeerPath *iter;
-    struct CadetPeerPath *next;
-    unsigned int i;
-    GNUNET_PEER_Id p1;
-    GNUNET_PEER_Id p2;
-
-    GCC_check_connections ();
-    p1 = GNUNET_PEER_search (peer1);
-    p2 = GNUNET_PEER_search (peer2);
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Link %u-%u broken\n", p1, p2);
-    if (0 == p1 || 0 == p2)
-    {
-        /* We don't even know them */
-        return;
-    }
-
-    for (iter = peer->path_head; NULL != iter; iter = next)
-    {
-        next = iter->next;
-        for (i = 0; i < iter->length - 1; i++)
-        {
-            if ((iter->peers[i] == p1 && iter->peers[i + 1] == p2)
-                    || (iter->peers[i] == p2 && iter->peers[i + 1] == p1))
-            {
-                char *s;
-
-                s = path_2s (iter);
-                LOG (GNUNET_ERROR_TYPE_DEBUG, " - invalidating %s\n", s);
-                GNUNET_free (s);
-
-                path_invalidate (iter);
-            }
-        }
-    }
-    GCC_check_connections ();
+  return (NULL != cp->core_mq) ? GNUNET_YES : GNUNET_NO;
 }
 
 
 /**
- * Count the number of known paths toward the peer.
- *
- * @param peer Peer to get path info.
+ * Start message queue change notifications.
  *
- * @return Number of known paths.
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
  */
-unsigned int
-GCP_count_paths (const struct CadetPeer *peer)
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls)
 {
-    struct CadetPeerPath *iter;
-    unsigned int i;
+  struct GCP_MessageQueueManager *mqm;
 
-    for (iter = peer->path_head, i = 0; NULL != iter; iter = iter->next)
-        i++;
-
-    return i;
+  mqm = GNUNET_new (struct GCP_MessageQueueManager);
+  mqm->cb = cb;
+  mqm->cb_cls = cb_cls;
+  mqm->cp = cp;
+  GNUNET_CONTAINER_DLL_insert (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating MQM %p for peer %s\n",
+       mqm,
+       GCP_2s (cp));
+  if (NULL != cp->core_mq)
+    cb (cb_cls,
+        GNUNET_YES);
+  return mqm;
 }
 
 
 /**
- * Iterate over the paths to a peer.
+ * Stops message queue change notifications.
  *
- * @param peer Peer to get path info.
- * @param callback Function to call for every path.
- * @param cls Closure for @a callback.
- *
- * @return Number of iterated paths.
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
  */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *peer,
-                   GCP_path_iterator callback,
-                   void *cls)
-{
-    struct CadetPeerPath *iter;
-    unsigned int i;
-
-    for (iter = peer->path_head, i = 0; NULL != iter; iter = iter->next)
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env)
+{
+  struct CadetPeer *cp = mqm->cp;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying MQM %p for peer %s%s\n",
+       mqm,
+       GCP_2s (cp),
+       (NULL == last_env) ? "" : " with last ditch transmission");
+  if (NULL != mqm->env)
+    GNUNET_MQ_discard (mqm->env);
+  if (NULL != last_env)
+  {
+    if (NULL != cp->core_mq)
     {
-        i++;
-        if (GNUNET_YES != callback (cls, peer, iter))
-            break;
+      GNUNET_MQ_notify_sent (last_env,
+                             &mqm_send_done,
+                             cp);
+      GNUNET_MQ_send (cp->core_mq,
+                      last_env);
     }
-
-    return i;
+    else
+    {
+      GNUNET_MQ_discard (last_env);
+    }
+  }
+  if (cp->mqm_ready_ptr == mqm)
+    cp->mqm_ready_ptr = mqm->next;
+  GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  GNUNET_free (mqm);
 }
 
 
 /**
- * Iterate all known peers.
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
  *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
  */
 void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls)
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env)
 {
-    GCC_check_connections ();
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           iter,
-                                           cls);
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message to %s out of management\n",
+       GCP_2s (cp));
+  if (NULL == cp->core_mq)
+  {
+    GNUNET_MQ_discard (env);
+    return;
+  }
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  GNUNET_MQ_send (cp->core_mq,
+                  env);
 }
 
 
-/**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
- *
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *peer)
-{
-    if (NULL == peer)
-        return "(NULL)";
-    return GNUNET_i2s (GNUNET_PEER_resolve2 (peer->id));
-}
 
 
-/* end of gnunet-service-cadet_peer.c */
+/* end of gnunet-service-cadet-new_peer.c */
diff --git a/src/cadet/gnunet-service-cadet_peer.h b/src/cadet/gnunet-service-cadet_peer.h
index 1e206e10f..baa87ea87 100644
--- a/src/cadet/gnunet-service-cadet_peer.h
+++ b/src/cadet/gnunet-service-cadet_peer.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,464 +20,374 @@
 
 /**
  * @file cadet/gnunet-service-cadet_peer.h
- * @brief cadet service; dealing with remote peers
+ * @brief Information we track per peer.
  * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMP (Gnunet Cadet Peer)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_PEER_H
 #define GNUNET_SERVICE_CADET_PEER_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet_path.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet_hello_lib.h"
 
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetPeer;
-
-/**
- * Handle to queued messages on a peer level.
- */
-struct CadetPeerQueue;
-
-#include "gnunet-service-cadet_connection.h"
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param fwd Was this a FWD going message?
- * @param sent Was it really sent? (Could have been canceled)
- * @param type Type of message sent.
- * @param payload_type Type of payload, if applicable.
- * @param pid Message ID, or 0 if not applicable (create, destroy, etc).
- * @param size Size of the message.
- * @param wait Time spent waiting for core (only the time for THIS message)
- */
-typedef void
-(*GCP_sent) (void *cls,
-             struct CadetConnection *c,
-             int fwd,
-             int sent,
-             uint16_t type,
-             uint16_t payload_type,
-             struct CadetEncryptedMessageIdentifier pid,
-             size_t size,
-             struct GNUNET_TIME_Relative wait);
 
 /**
- * Peer path iterator.
+ * Get the static string for a peer ID.
  *
- * @param cls Closure.
- * @param peer Peer this path is towards.
- * @param path Path itself
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-typedef int
-(*GCP_path_iterator) (void *cls,
-                      struct CadetPeer *peer,
-                      struct CadetPeerPath *path);
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize peer subsystem.
+ * @param peer Peer.
  *
- * @param c Configuration.
- */
-void
-GCP_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-/**
- * Shut down the peer subsystem.
+ * @return Static string for it's ID.
  */
-void
-GCP_shutdown (void);
+const char *
+GCP_2s (const struct CadetPeer *peer);
 
 
 /**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
+ * Retrieve the CadetPeer stucture associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
  *
  * @param peer_id Full identity of the peer.
  * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
+ *               #GNUNET_NO to return NULL if peer is unknown.
  * @return Existing or newly created peer structure.
  *         NULL if unknown and not requested @a create
  */
 struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id, int create);
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create);
 
 
 /**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
- *
- * @param peer Short identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
+ * Calculate how desirable a path is for @a cp if
+ * @a cp is at offset @a off in the path.
  *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
+ * @param cp a peer reachable via a path
+ * @param off offset of @a cp in a path
+ * @return score how useful a path is to reach @a cp,
+ *         positive scores mean path is more desirable
  */
-struct CadetPeer *
-GCP_get_short (const GNUNET_PEER_Id peer, int create);
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off);
 
 
 /**
- * Try to establish a new connection to this peer (in its tunnel).
- * If the peer doesn't have any path to it yet, try to get one.
- * If the peer already has some path, send a CREATE CONNECTION towards it.
- *
- * @param peer Peer to connect to.
- */
-void
-GCP_connect (struct CadetPeer *peer);
-
-/**
- * @brief Send a message to another peer (using CORE).
+ * Obtain the peer identity for a `struct CadetPeer`.
  *
- * @param peer Peer towards which to queue the message.
- * @param message Message to send.
- * @param payload_type Type of the message's payload, for debug messages.
- *                     0 if the message is a retransmission (unknown payload).
- *                     UINT16_MAX if the message does not have payload.
- * @param payload_id ID of the payload (MID, ACK #, etc)
- * @param c Connection this message belongs to (can be NULL).
- * @param fwd Is this a message going root->dest? (FWD ACK are NOT FWD!)
- * @param cont Continuation to be called once CORE has sent the message.
- * @param cont_cls Closure for @c cont.
+ * @param cp our peer handle
+ * @return the peer identity
  */
-struct CadetPeerQueue *
-GCP_send (struct CadetPeer *peer,
-          const struct GNUNET_MessageHeader *message,
-          uint16_t payload_type,
-          struct CadetEncryptedMessageIdentifier payload_id,
-          struct CadetConnection *c,
-          int fwd,
-          GCP_sent cont,
-          void *cont_cls);
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp);
 
-/**
- * Cancel sending a message. Message must have been sent with
- * #GCP_send before.  May not be called after the notify sent
- * callback has been called.
- *
- * It does NOT call the continuation given to #GCP_send.
- *
- * @param q Queue handle to cancel
- */
-void
-GCP_send_cancel (struct CadetPeerQueue *q);
 
 /**
- * Set tunnel.
+ * Iterate over all known peers.
  *
- * @param peer Peer.
- * @param t Tunnel.
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
  */
 void
-GCP_set_tunnel (struct CadetPeer *peer, struct CadetTunnel *t);
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls);
 
 
 /**
- * Check whether there is a direct (core level)  connection to peer.
- *
- * @param peer Peer to check.
+ * Count the number of known paths toward the peer.
  *
- * @return #GNUNET_YES if there is a direct connection.
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
  */
-int
-GCP_is_neighbor (const struct CadetPeer *peer);
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp);
 
 
 /**
- * Create and initialize a new tunnel towards a peer, in case it has none.
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
  *
- * Does not generate any traffic, just creates the local data structures.
- *
- * @param peer Peer towards which to create the tunnel.
+ * @param cp peer to drop paths to
  */
 void
-GCP_add_tunnel (struct CadetPeer *peer);
+GCP_drop_owned_paths (struct CadetPeer *cp);
 
 
 /**
- * Add a connection to a neighboring peer.
- *
- * Store that the peer is the first hop of the connection in one
- * direction and that on peer disconnect the connection must be
- * notified and destroyed, for it will no longer be valid.
+ * Peer path iterator.
  *
- * @param peer Peer to add connection to.
- * @param c Connection to add.
- * @param pred #GNUNET_YES if we are predecessor, #GNUNET_NO if we are successor
+ * @param cls Closure.
+ * @param path Path itself
+ * @param off offset of the target peer in @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
  */
-void
-GCP_add_connection (struct CadetPeer *peer,
-                    struct CadetConnection *c,
-                    int pred);
+typedef int
+(*GCP_PathIterator) (void *cls,
+                     struct CadetPeerPath *path,
+                     unsigned int off);
 
 
 /**
- * Add the path to the peer and update the path used to reach it in case this
- * is the shortest.
- *
- * @param peer Destination peer to add the path to.
- * @param path New path to add. Last peer must be the peer in arg 1.
- *             Path will be either used of freed if already known.
- * @param trusted Do we trust that this path is real?
+ * Iterate over the paths to a peer.
  *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-struct CadetPeerPath *
-GCP_add_path (struct CadetPeer *peer,
-              struct CadetPeerPath *p,
-              int trusted);
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls);
 
 
 /**
- * Add the path to the origin peer and update the path used to reach it in case
- * this is the shortest.
- * The path is given in peer_info -> destination, therefore we turn the path
- * upside down first.
- *
- * @param peer Peer to add the path to, being the origin of the path.
- * @param path New path to add after being inversed.
- *             Path will be either used or freed.
- * @param trusted Do we trust that this path is real?
+ * Iterate over the paths to @a peer where
+ * @a peer is at distance @a dist from us.
  *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp Peer to get path info.
+ * @param dist desired distance of @a peer to us on the path
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-struct CadetPeerPath *
-GCP_add_path_to_origin (struct CadetPeer *peer,
-                        struct CadetPeerPath *path,
-                        int trusted);
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls);
+
 
 /**
- * Adds a path to the info of all the peers in the path
+ * Remove an entry from the DLL of all of the paths that this peer is on.
  *
- * @param p Path to process.
- * @param confirmed Whether we know if the path works or not.
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_add_path_to_all (const struct CadetPeerPath *p, int confirmed);
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off);
 
 
 /**
- * Remove any path to the peer that has the extact same peers as the one given.
+ * Add an entry to the DLL of all of the paths that this peer is on.
  *
- * @param peer Peer to remove the path from.
- * @param path Path to remove. Is always destroyed .
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_remove_path (struct CadetPeer *peer,
-                 struct CadetPeerPath *path);
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off);
 
 
 /**
- * Check that we are aware of a connection from a neighboring peer.
+ * Get the tunnel towards a peer.
  *
- * @param peer Peer to the connection is with
- * @param c Connection that should be in the map with this peer.
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
+ * @return Tunnel towards peer.
  */
-void
-GCP_check_connection (const struct CadetPeer *peer,
-                      const struct CadetConnection *c);
+struct CadetTunnel *
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create);
 
 
 /**
- * Remove a connection from a neighboring peer.
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
  *
- * @param peer Peer to remove connection from.
- * @param c Connection to remove.
+ * @param cp the peer affected
+ * @param t the dead tunnel
  */
 void
-GCP_remove_connection (struct CadetPeer *peer,
-                       const struct CadetConnection *c);
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t);
 
 
 /**
- * Start the DHT search for new paths towards the peer: we don't have
- * enough good connections.
+ * Try adding a @a path to this @a cp.  If the peer already
+ * has plenty of paths, return NULL.
  *
- * @param peer Destination peer.
+ * @param cp peer to which the @a path leads to
+ * @param path a path looking for an owner; may not be fully initialized yet!
+ * @param off offset of @a cp in @a path
+ * @param force for attaching the path
+ * @return NULL if this peer does not care to become a new owner,
+ *         otherwise the node in the peer's path heap for the @a path.
  */
-void
-GCP_start_search (struct CadetPeer *peer);
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force);
 
 
 /**
- * Stop the DHT search for new paths towards the peer: we already have
- * enough good connections.
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
  *
- * @param peer Destination peer.
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
  */
 void
-GCP_stop_search (struct CadetPeer *peer);
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn);
 
 
 /**
- * Get the Full ID of a peer.
+ * Add a @a connection to this @a cp.
  *
- * @param peer Peer to get from.
- *
- * @return Full ID of peer.
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
  */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (const struct CadetPeer *peer);
+void
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc);
 
 
 /**
- * Get the Short ID of a peer.
- *
- * @param peer Peer to get from.
+ * Remove a @a connection that went via this @a cp.
  *
- * @return Short ID of peer.
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
  */
-GNUNET_PEER_Id
-GCP_get_short_id (const struct CadetPeer *peer);
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc);
 
 
 /**
- * Get the tunnel towards a peer.
+ * We got a HELLO for a @a cp, remember it, and possibly
+ * trigger adequate actions (like trying to connect).
  *
- * @param peer Peer to get from.
- *
- * @return Tunnel towards peer.
+ * @param cp the peer we got a HELLO for
+ * @param hello the HELLO to remember
  */
-struct CadetTunnel *
-GCP_get_tunnel (const struct CadetPeer *peer);
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_HELLO_Message *hello);
 
 
 /**
- * Set the hello message.
- *
- * @param peer Peer whose message to set.
- * @param hello Hello message.
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
  */
 void
-GCP_set_hello (struct CadetPeer *peer,
-               const struct GNUNET_HELLO_Message *hello);
+GCP_destroy_all_peers (void);
 
 
 /**
- * Get the hello message.
- *
- * @param peer Peer whose message to get.
+ * Data structure used to track whom we have to notify about changes
+ * in our ability to transmit to a given peer.
  *
- * @return Hello message.
+ * All queue managers will be given equal chance for sending messages
+ * to @a cp.  This construct this guarantees fairness for access to @a
+ * cp among the different message queues.  Each connection or route
+ * will have its respective message queue managers for each direction.
  */
-struct GNUNET_HELLO_Message *
-GCP_get_hello (struct CadetPeer *peer);
+struct GCP_MessageQueueManager;
 
 
 /**
- * Try to connect to a peer on TRANSPORT level.
+ * Function to call with updated message queue object.
  *
- * @param peer Peer to whom to connect.
+ * @param cls closure
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
  */
-void
-GCP_try_connect (struct CadetPeer *peer);
+typedef void
+(*GCP_MessageQueueNotificationCallback)(void *cls,
+                                        int available);
+
 
 /**
- * Notify a peer that a link between two other peers is broken. If any path
- * used that link, eliminate it.
+ * Start message queue change notifications.  Will create a new slot
+ * to manage the message queue to the given @a cp.
  *
- * @param peer Peer affected by the change.
- * @param peer1 Peer whose link is broken.
- * @param peer2 Peer whose link is broken.
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
  */
-void
-GCP_notify_broken_link (struct CadetPeer *peer,
-                        const struct GNUNET_PeerIdentity *peer1,
-                        const struct GNUNET_PeerIdentity *peer2);
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls);
 
 
 /**
- * Count the number of known paths toward the peer.
+ * Test if @a cp has a core-level connection
  *
- * @param peer Peer to get path info.
- *
- * @return Number of known paths.
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
  */
-unsigned int
-GCP_count_paths (const struct CadetPeer *peer);
+int
+GCP_has_core_connection (struct CadetPeer *cp);
+
 
 /**
- * Iterate over the paths to a peer.
+ * Send the message in @a env via a @a mqm.  Must only be called at
+ * most once after the respective
+ * #GCP_MessageQueueNotificationCallback was called with `available`
+ * set to #GNUNET_YES, and not after the callback was called with
+ * `available` set to #GNUNET_NO or #GNUNET_SYSERR.
  *
- * @param peer Peer to get path info.
- * @param callback Function to call for every path.
- * @param cls Closure for @a callback.
- *
- * @return Number of iterated paths.
+ * @param mqm message queue manager for the transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *peer,
-                   GCP_path_iterator callback,
-                   void *cls);
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env);
 
 
 /**
- * Iterate all known peers.
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
  *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
  */
 void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls);
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env);
 
 
 /**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
+ * Stops message queue change notifications and sends a last message.
+ * In practice, this is implemented by sending that @a last_env
+ * message immediately (if any), ignoring queue order.
  *
- * @return Static string for it's ID.
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
  */
-const char *
-GCP_2s (const struct CadetPeer *peer);
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env);
 
 
 /**
- * Log all kinds of info about a peer.
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
  *
- * @param peer Peer.
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
  */
 void
-GCP_debug (const struct CadetPeer *p,
-           enum GNUNET_ErrorType level);
-
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq);
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
 
-/* ifndef GNUNET_CADET_SERVICE_PEER_H */
 #endif
-/* end of gnunet-cadet-service_peer.h */
diff --git a/src/cadet/gnunet-service-cadet_tunnel.c b/src/cadet/gnunet-service-cadet_tunnel.c
deleted file mode 100644
index 3b21f4107..000000000
--- a/src/cadet/gnunet-service-cadet_tunnel.c
+++ /dev/null
@@ -1,3501 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet_tunnel.c
- * @brief logical links between CADET clients
- * @author Bartlomiej Polot
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_signatures.h"
-#include "gnunet_statistics_service.h"
-#include "cadet_protocol.h"
-#include "cadet_path.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-tun",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-tun",__VA_ARGS__)
-
-#define REKEY_WAIT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 5)
-
-#if !defined(GNUNET_CULL_LOGGING)
-  #define DUMP_KEYS_TO_STDERR GNUNET_YES
-#else
-  #define DUMP_KEYS_TO_STDERR GNUNET_NO
-#endif
-
-#define MIN_TUNNEL_BUFFER       8
-#define MAX_TUNNEL_BUFFER       64
-#define MAX_SKIPPED_KEYS        64
-#define MAX_KEY_GAP             256
-#define AX_HEADER_SIZE (sizeof (uint32_t) * 2\
-                        + sizeof (struct GNUNET_CRYPTO_EcdhePublicKey))
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-struct CadetTChannel
-{
-  struct CadetTChannel *next;
-  struct CadetTChannel *prev;
-  struct CadetChannel *ch;
-};
-
-
-/**
- * Entry in list of connections used by tunnel, with metadata.
- */
-struct CadetTConnection
-{
-  /**
-   * Next in DLL.
-   */
-  struct CadetTConnection *next;
-
-  /**
-   * Prev in DLL.
-   */
-  struct CadetTConnection *prev;
-
-  /**
-   * Connection handle.
-   */
-  struct CadetConnection *c;
-
-  /**
-   * Creation time, to keep oldest connection alive.
-   */
-  struct GNUNET_TIME_Absolute created;
-
-  /**
-   * Connection throughput, to keep fastest connection alive.
-   */
-  uint32_t throughput;
-};
-
-
-/**
- * Struct to old keys for skipped messages while advancing the Axolotl ratchet.
- */
-struct CadetTunnelSkippedKey
-{
-  /**
-   * DLL next.
-   */
-  struct CadetTunnelSkippedKey *next;
-
-  /**
-   * DLL prev.
-   */
-  struct CadetTunnelSkippedKey *prev;
-
-  /**
-   * When was this key stored (for timeout).
-   */
-  struct GNUNET_TIME_Absolute timestamp;
-
-  /**
-   * Header key.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HK;
-
-  /**
-   * Message key.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-
-  /**
-   * Key number for a given HK.
-   */
-  unsigned int Kn;
-};
-
-
-/**
- * Axolotl data, according to https://github.com/trevp/axolotl/wiki .
- */
-struct CadetTunnelAxolotl
-{
-  /**
-   * A (double linked) list of stored message keys and associated header keys
-   * for "skipped" messages, i.e. messages that have not been
-   * received despite the reception of more recent messages, (head).
-   */
-  struct CadetTunnelSkippedKey *skipped_head;
-
-  /**
-   * Skipped messages' keys DLL, tail.
-   */
-  struct CadetTunnelSkippedKey *skipped_tail;
-
-  /**
-   * Elements in @a skipped_head <-> @a skipped_tail.
-   */
-  unsigned int skipped;
-
-  /**
-   * 32-byte root key which gets updated by DH ratchet.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey RK;
-
-  /**
-   * 32-byte header key (send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HKs;
-
-  /**
-   * 32-byte header key (recv)
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HKr;
-
-  /**
-   * 32-byte next header key (send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey NHKs;
-
-  /**
-   * 32-byte next header key (recv).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey NHKr;
-
-  /**
-   * 32-byte chain keys (used for forward-secrecy updating, send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey CKs;
-
-  /**
-   * 32-byte chain keys (used for forward-secrecy updating, recv).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey CKr;
-
-  /**
-   * ECDH for key exchange (A0 / B0).
-   */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *kx_0;
-
-  /**
-   * ECDH Ratchet key (send).
-   */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *DHRs;
-
-  /**
-   * ECDH Ratchet key (recv).
-   */
-  struct GNUNET_CRYPTO_EcdhePublicKey DHRr;
-
-  /**
-   * Message number (reset to 0 with each new ratchet, next message to send).
-   */
-  uint32_t Ns;
-
-  /**
-   * Message number (reset to 0 with each new ratchet, next message to recv).
-   */
-  uint32_t Nr;
-
-  /**
-   * Previous message numbers (# of msgs sent under prev ratchet)
-   */
-  uint32_t PNs;
-
-  /**
-   * True (#GNUNET_YES) if we have to send a new ratchet key in next msg.
-   */
-  int ratchet_flag;
-
-  /**
-   * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
-   */
-  unsigned int ratchet_allowed;
-
-  /**
-   * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
-   */
-  unsigned int ratchet_counter;
-
-  /**
-   * When does this ratchet expire and a new one is triggered.
-   */
-  struct GNUNET_TIME_Absolute ratchet_expiration;
-};
-
-
-/**
- * Struct containing all information regarding a tunnel to a peer.
- */
-struct CadetTunnel
-{
-  /**
-   * Endpoint of the tunnel.
-   */
-  struct CadetPeer *peer;
-
-  /**
-   * Axolotl info.
-   */
-  struct CadetTunnelAxolotl *ax;
-
-  /**
-   * State of the tunnel connectivity.
-   */
-  enum CadetTunnelCState cstate;
-
-  /**
-   * State of the tunnel encryption.
-   */
-  enum CadetTunnelEState estate;
-
-  /**
-   * Peer's ephemeral key, to recreate @c e_key and @c d_key when own ephemeral
-   * key changes.
-   */
-  struct GNUNET_CRYPTO_EcdhePublicKey peers_ephemeral_key;
-
-  /**
-   * Encryption ("our") key. It is only "confirmed" if kx_ctx is NULL.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey e_key;
-
-  /**
-   * Decryption ("their") key. It is only "confirmed" if kx_ctx is NULL.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey d_key;
-
-  /**
-   * Task to start the rekey process.
-   */
-  struct GNUNET_SCHEDULER_Task *rekey_task;
-
-  /**
-   * Paths that are actively used to reach the destination peer.
-   */
-  struct CadetTConnection *connection_head;
-  struct CadetTConnection *connection_tail;
-
-  /**
-   * Next connection number.
-   */
-  uint32_t next_cid;
-
-  /**
-   * Channels inside this tunnel.
-   */
-  struct CadetTChannel *channel_head;
-  struct CadetTChannel *channel_tail;
-
-  /**
-   * Channel ID for the next created channel.
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber next_ctn;
-
-  /**
-   * Destroy flag: if true, destroy on last message.
-   */
-  struct GNUNET_SCHEDULER_Task * destroy_task;
-
-  /**
-   * Queued messages, to transmit once tunnel gets connected.
-   */
-  struct CadetTunnelDelayed *tq_head;
-  struct CadetTunnelDelayed *tq_tail;
-
-  /**
-   * Task to trim connections if too many are present.
-   */
-  struct GNUNET_SCHEDULER_Task * trim_connections_task;
-
-  /**
-   * Ephemeral message in the queue (to avoid queueing more than one).
-   */
-  struct CadetConnectionQueue *ephm_h;
-
-  /**
-   * Pong message in the queue.
-   */
-  struct CadetConnectionQueue *pong_h;
-};
-
-
-/**
- * Struct used to save messages in a non-ready tunnel to send once connected.
- */
-struct CadetTunnelDelayed
-{
-  /**
-   * DLL
-   */
-  struct CadetTunnelDelayed *next;
-  struct CadetTunnelDelayed *prev;
-
-  /**
-   * Tunnel.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Tunnel queue given to the channel to cancel request. Update on send_queued.
-   */
-  struct CadetTunnelQueue *tq;
-
-  /**
-   * Message to send.
-   */
-  /* struct GNUNET_MessageHeader *msg; */
-};
-
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetTunnelQueue
-{
-  /**
-   * Connection queue handle, to cancel if necessary.
-   */
-  struct CadetConnectionQueue *cq;
-
-  /**
-   * Handle in case message hasn't been given to a connection yet.
-   */
-  struct CadetTunnelDelayed *tqd;
-
-  /**
-   * Continuation to call once sent.
-   */
-  GCT_sent cont;
-
-  /**
-   * Closure for @c cont.
-   */
-  void *cont_cls;
-};
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-
-/**
- * Don't try to recover tunnels if shutting down.
- */
-extern int shutting_down;
-
-
-/**
- * Set of all tunnels, in order to trigger a new exchange on rekey.
- * Indexed by peer's ID.
- */
-static struct GNUNET_CONTAINER_MultiPeerMap *tunnels;
-
-/**
- * Own Peer ID private key.
- */
-const static struct GNUNET_CRYPTO_EddsaPrivateKey *id_key;
-
-
-/********************************  AXOLOTL ************************************/
-
-/**
- * How many messages are needed to trigger a ratchet advance.
- */
-static unsigned long long ratchet_messages;
-
-/**
- * How long until we trigger a ratched advance.
- */
-static struct GNUNET_TIME_Relative ratchet_time;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-/**
- * Get string description for tunnel connectivity state.
- *
- * @param cs Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-cstate2s (enum CadetTunnelCState cs)
-{
-  static char buf[32];
-
-  switch (cs)
-  {
-    case CADET_TUNNEL_NEW:
-      return "CADET_TUNNEL_NEW";
-    case CADET_TUNNEL_SEARCHING:
-      return "CADET_TUNNEL_SEARCHING";
-    case CADET_TUNNEL_WAITING:
-      return "CADET_TUNNEL_WAITING";
-    case CADET_TUNNEL_READY:
-      return "CADET_TUNNEL_READY";
-    case CADET_TUNNEL_SHUTDOWN:
-      return "CADET_TUNNEL_SHUTDOWN";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", cs);
-      return buf;
-  }
-  return "";
-}
-
-
-/**
- * Get string description for tunnel encryption state.
- *
- * @param es Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-estate2s (enum CadetTunnelEState es)
-{
-  static char buf[32];
-
-  switch (es)
-  {
-    case CADET_TUNNEL_KEY_UNINITIALIZED:
-      return "CADET_TUNNEL_KEY_UNINITIALIZED";
-    case CADET_TUNNEL_KEY_SENT:
-      return "CADET_TUNNEL_KEY_SENT";
-    case CADET_TUNNEL_KEY_PING:
-      return "CADET_TUNNEL_KEY_PING";
-    case CADET_TUNNEL_KEY_OK:
-      return "CADET_TUNNEL_KEY_OK";
-    case CADET_TUNNEL_KEY_REKEY:
-      return "CADET_TUNNEL_KEY_REKEY";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", es);
-      return buf;
-  }
-  return "";
-}
-
-
-/**
- * @brief Check if tunnel is ready to send traffic.
- *
- * Tunnel must be connected and with encryption correctly set up.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if ready, #GNUNET_NO otherwise
- */
-static int
-is_ready (struct CadetTunnel *t)
-{
-  int ready;
-  int conn_ok;
-  int enc_ok;
-
-  conn_ok = CADET_TUNNEL_READY == t->cstate;
-  enc_ok = CADET_TUNNEL_KEY_OK == t->estate
-           || CADET_TUNNEL_KEY_REKEY == t->estate
-           || CADET_TUNNEL_KEY_PING == t->estate;
-  ready = conn_ok && enc_ok;
-  ready = ready || GCT_is_loopback (t);
-  return ready;
-}
-
-
-/**
- * Get the channel's buffer. ONLY FOR NON-LOOPBACK CHANNELS!!
- *
- * @param tch Tunnel's channel handle.
- *
- * @return Amount of messages the channel can still buffer towards the client.
- */
-static unsigned int
-get_channel_buffer (const struct CadetTChannel *tch)
-{
-  int fwd;
-
-  /* If channel is incoming, is terminal in the FWD direction and fwd is YES */
-  fwd = GCCH_is_terminal (tch->ch, GNUNET_YES);
-
-  return GCCH_get_buffer (tch->ch, fwd);
-}
-
-
-/**
- * Get the channel's allowance status.
- *
- * @param tch Tunnel's channel handle.
- *
- * @return #GNUNET_YES if we allowed the client to send data to us.
- */
-static int
-get_channel_allowed (const struct CadetTChannel *tch)
-{
-  int fwd;
-
-  /* If channel is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCCH_is_origin (tch->ch, GNUNET_YES);
-
-  return GCCH_get_allowed (tch->ch, fwd);
-}
-
-
-/**
- * Get the connection's buffer.
- *
- * @param tc Tunnel's connection handle.
- *
- * @return Amount of messages the connection can still buffer.
- */
-static unsigned int
-get_connection_buffer (const struct CadetTConnection *tc)
-{
-  int fwd;
-
-  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCC_is_origin (tc->c, GNUNET_YES);
-
-  return GCC_get_buffer (tc->c, fwd);
-}
-
-
-/**
- * Get the connection's allowance.
- *
- * @param tc Tunnel's connection handle.
- *
- * @return Amount of messages we have allowed the next peer to send us.
- */
-static unsigned int
-get_connection_allowed (const struct CadetTConnection *tc)
-{
-  int fwd;
-
-  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCC_is_origin (tc->c, GNUNET_YES);
-
-  return GCC_get_allowed (tc->c, fwd);
-}
-
-
-/**
- * Create a new Axolotl ephemeral (ratchet) key.
- *
- * @param t Tunnel.
- */
-static void
-new_ephemeral (struct CadetTunnel *t)
-{
-  GNUNET_free_non_null (t->ax->DHRs);
-  t->ax->DHRs = GNUNET_CRYPTO_ecdhe_key_create();
-  #if DUMP_KEYS_TO_STDERR
-  {
-    struct GNUNET_CRYPTO_EcdhePublicKey pub;
-    GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &pub);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  new DHRs generated: pub  %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &pub));
-  }
-  #endif
-}
-
-
-/**
- * Calculate HMAC.
- *
- * @param plaintext Content to HMAC.
- * @param size Size of @c plaintext.
- * @param iv Initialization vector for the message.
- * @param key Key to use.
- * @param hmac[out] Destination to store the HMAC.
- */
-static void
-t_hmac (const void *plaintext, size_t size,
-        uint32_t iv, const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-        struct GNUNET_ShortHashCode *hmac)
-{
-  static const char ctx[] = "cadet authentication key";
-  struct GNUNET_CRYPTO_AuthKey auth_key;
-  struct GNUNET_HashCode hash;
-
-#if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  HMAC %u bytes with key %s\n", size,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) key));
-#endif
-  GNUNET_CRYPTO_hmac_derive_key (&auth_key, key,
-                                 &iv, sizeof (iv),
-                                 key, sizeof (*key),
-                                 ctx, sizeof (ctx),
-                                 NULL);
-  /* Two step: CADET_Hash is only 256 bits, HashCode is 512. */
-  GNUNET_CRYPTO_hmac (&auth_key, plaintext, size, &hash);
-  GNUNET_memcpy (hmac, &hash, sizeof (*hmac));
-}
-
-
-/**
- * Perform a HMAC.
- *
- * @param key Key to use.
- * @param hash[out] Resulting HMAC.
- * @param source Source key material (data to HMAC).
- * @param len Length of @a source.
- */
-static void
-t_ax_hmac_hash (struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-                struct GNUNET_HashCode *hash,
-                void *source, unsigned int len)
-{
-  static const char ctx[] = "axolotl HMAC-HASH";
-  struct GNUNET_CRYPTO_AuthKey auth_key;
-
-  GNUNET_CRYPTO_hmac_derive_key (&auth_key, key,
-                                 ctx, sizeof (ctx),
-                                 NULL);
-  GNUNET_CRYPTO_hmac (&auth_key, source, len, hash);
-}
-
-
-/**
- * Derive a key from a HMAC-HASH.
- *
- * @param key Key to use for the HMAC.
- * @param out Key to generate.
- * @param source Source key material (data to HMAC).
- * @param len Length of @a source.
- */
-static void
-t_hmac_derive_key (struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-                   struct GNUNET_CRYPTO_SymmetricSessionKey *out,
-                   void *source, unsigned int len)
-{
-  static const char ctx[] = "axolotl derive key";
-  struct GNUNET_HashCode h;
-
-  t_ax_hmac_hash (key, &h, source, len);
-  GNUNET_CRYPTO_kdf (out, sizeof (*out), ctx, sizeof (ctx),
-                     &h, sizeof (h), NULL);
-}
-
-
-/**
- * Encrypt data with the axolotl tunnel key.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the encrypted data.
- * @param src Source of the plaintext. Can overlap with @c dst.
- * @param size Size of the plaintext.
- *
- * @return Size of the encrypted data.
- */
-static int
-t_ax_encrypt (struct CadetTunnel *t, void *dst, const void *src, size_t size)
-{
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-  ax->ratchet_counter++;
-  if (GNUNET_YES == ax->ratchet_allowed
-      && (ratchet_messages <= ax->ratchet_counter
-          || 0 == GNUNET_TIME_absolute_get_remaining (ax->ratchet_expiration).rel_value_us))
-  {
-    ax->ratchet_flag = GNUNET_YES;
-  }
-
-  if (GNUNET_YES == ax->ratchet_flag)
-  {
-    /* Advance ratchet */
-    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3];
-    struct GNUNET_HashCode dh;
-    struct GNUNET_HashCode hmac;
-    static const char ctx[] = "axolotl ratchet";
-
-    new_ephemeral (t);
-    ax->HKs = ax->NHKs;
-
-    /* RK, NHKs, CKs = KDF( HMAC-HASH(RK, DH(DHRs, DHRr)) ) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs, &ax->DHRr, &dh);
-    t_ax_hmac_hash (&ax->RK, &hmac, &dh, sizeof (dh));
-    GNUNET_CRYPTO_kdf (keys, sizeof (keys), ctx, sizeof (ctx),
-                       &hmac, sizeof (hmac), NULL);
-    ax->RK = keys[0];
-    ax->NHKs = keys[1];
-    ax->CKs = keys[2];
-
-    ax->PNs = ax->Ns;
-    ax->Ns = 0;
-    ax->ratchet_flag = GNUNET_NO;
-    ax->ratchet_allowed = GNUNET_NO;
-    ax->ratchet_counter = 0;
-    ax->ratchet_expiration =
-      GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(), ratchet_time);
-  }
-
-  t_hmac_derive_key (&ax->CKs, &MK, "0", 1);
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &MK, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  CKs: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKs));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_ENC with key %u: %s\n", ax->Ns,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &MK));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_encrypt (src, size, &MK, &iv, dst);
-  t_hmac_derive_key (&ax->CKs, &ax->CKs, "1", 1);
-
-  CADET_TIMING_END;
-
-  return out_size;
-}
-
-
-/**
- * Decrypt data with the axolotl tunnel key.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the decrypted data.
- * @param src Source of the ciphertext. Can overlap with @c dst.
- * @param size Size of the ciphertext.
- *
- * @return Size of the decrypted data.
- */
-static int
-t_ax_decrypt (struct CadetTunnel *t, void *dst, const void *src, size_t size)
-{
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-
-  t_hmac_derive_key (&ax->CKr, &MK, "0", 1);
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &MK, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  CKr: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKr));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC with key %u: %s\n", ax->Nr,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &MK));
-  #endif
-
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  out_size = GNUNET_CRYPTO_symmetric_decrypt (src, size, &MK, &iv, dst);
-  GNUNET_assert (out_size == size);
-
-  t_hmac_derive_key (&ax->CKr, &ax->CKr, "1", 1);
-
-  CADET_TIMING_END;
-
-  return out_size;
-}
-
-
-/**
- * Encrypt header with the axolotl header key.
- *
- * @param t Tunnel whose key to use.
- * @param msg Message whose header to encrypt.
- */
-static void
-t_h_encrypt (struct CadetTunnel *t, struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-  ax = t->ax;
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &ax->HKs, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_ENC_H with key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKs));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_encrypt (&msg->Ns, AX_HEADER_SIZE,
-                                              &ax->HKs, &iv, &msg->Ns);
-
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
-  CADET_TIMING_END;
-}
-
-
-/**
- * Decrypt header with the current axolotl header key.
- *
- * @param t Tunnel whose current ax HK to use.
- * @param src Message whose header to decrypt.
- * @param dst Where to decrypt header to.
- */
-static void
-t_h_decrypt (struct CadetTunnel *t, const struct GNUNET_CADET_TunnelEncryptedMessage *src,
-             struct GNUNET_CADET_TunnelEncryptedMessage *dst)
-{
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &ax->HKr, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC_H with key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKr));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns, AX_HEADER_SIZE,
-                                              &ax->HKr, &iv, &dst->Ns);
-
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
-
-  CADET_TIMING_END;
-}
-
-
-/**
- * Decrypt and verify data with the appropriate tunnel key and verify that the
- * data has not been altered since it was sent by the remote peer.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the plaintext.
- * @param src Source of the message. Can overlap with @c dst.
- * @param size Size of the message.
- *
- * @return Size of the decrypted data, -1 if an error was encountered.
- */
-static int
-try_old_ax_keys (struct CadetTunnel *t, void *dst,
-                 const struct GNUNET_CADET_TunnelEncryptedMessage *src, size_t size)
-{
-  struct CadetTunnelSkippedKey *key;
-  struct GNUNET_ShortHashCode *hmac;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
-  struct GNUNET_CRYPTO_SymmetricSessionKey *valid_HK;
-  size_t esize;
-  size_t res;
-  size_t len;
-  unsigned int N;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Trying old keys\n");
-  hmac = &plaintext_header.hmac;
-  esize = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-
-  /* Find a correct Header Key */
-  for (key = t->ax->skipped_head; NULL != key; key = key->next)
-  {
-    #if DUMP_KEYS_TO_STDERR
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Trying hmac with key %s\n",
-         GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->HK));
-    #endif
-    t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &key->HK, hmac);
-    if (0 == memcmp (hmac, &src->hmac, sizeof (*hmac)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  hmac correct\n");
-      valid_HK = &key->HK;
-      break;
-    }
-  }
-  if (NULL == key)
-    return -1;
-
-  /* Should've been checked in -cadet_connection.c handle_cadet_encrypted. */
-  GNUNET_assert (size > sizeof (struct GNUNET_CADET_TunnelEncryptedMessage));
-  len = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  GNUNET_assert (len >= sizeof (struct GNUNET_MessageHeader));
-
-  /* Decrypt header */
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &key->HK, NULL, 0, NULL);
-  res = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns, AX_HEADER_SIZE,
-                                         &key->HK, &iv, &plaintext_header.Ns);
-  GNUNET_assert (AX_HEADER_SIZE == res);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Message %u, previous: %u\n",
-       ntohl (plaintext_header.Ns), ntohl (plaintext_header.PNs));
-
-  /* Find the correct Message Key */
-  N = ntohl (plaintext_header.Ns);
-  while (NULL != key && N != key->Kn)
-    key = key->next;
-  if (NULL == key || 0 != memcmp (&key->HK, valid_HK, sizeof (*valid_HK)))
-    return -1;
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC_H with skipped key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->HK));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC with skipped key %u: %s\n",
-       key->Kn, GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->MK));
-  #endif
-
-  /* Decrypt payload */
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &key->MK, NULL, 0, NULL);
-  res = GNUNET_CRYPTO_symmetric_decrypt (&src[1], len, &key->MK, &iv, dst);
-
-  /* Remove key */
-  GNUNET_CONTAINER_DLL_remove (t->ax->skipped_head, t->ax->skipped_tail, key);
-  t->ax->skipped--;
-  GNUNET_free (key); /* GNUNET_free overwrites memory with 0xbaadf00d */
-
-  return res;
-}
-
-
-/**
- * Delete a key from the list of skipped keys.
- *
- * @param t Tunnel to delete from.
- * @param HKr Header Key to use.
- */
-static void
-store_skipped_key (struct CadetTunnel *t,
-                   const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr)
-{
-  struct CadetTunnelSkippedKey *key;
-
-  key = GNUNET_new (struct CadetTunnelSkippedKey);
-  key->timestamp = GNUNET_TIME_absolute_get ();
-  key->Kn = t->ax->Nr;
-  key->HK = t->ax->HKr;
-  t_hmac_derive_key (&t->ax->CKr, &key->MK, "0", 1);
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    storing MK for Nr %u: %s\n",
-       key->Kn, GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->MK));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    for CKr: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &t->ax->CKr));
-  #endif
-  t_hmac_derive_key (&t->ax->CKr, &t->ax->CKr, "1", 1);
-  GNUNET_CONTAINER_DLL_insert (t->ax->skipped_head, t->ax->skipped_tail, key);
-  t->ax->Nr++;
-  t->ax->skipped++;
-}
-
-
-/**
- * Delete a key from the list of skipped keys.
- *
- * @param t Tunnel to delete from.
- * @param key Key to delete.
- */
-static void
-delete_skipped_key (struct CadetTunnel *t, struct CadetTunnelSkippedKey *key)
-{
-  GNUNET_CONTAINER_DLL_remove (t->ax->skipped_head, t->ax->skipped_tail, key);
-  GNUNET_free (key);
-  t->ax->skipped--;
-}
-
-
-/**
- * Stage skipped AX keys and calculate the message key.
- *
- * Stores each HK and MK for skipped messages.
- *
- * @param t Tunnel where to stage the keys.
- * @param HKr Header key.
- * @param Np Received meesage number.
- *
- * @return GNUNET_OK if keys were stored.
- *         GNUNET_SYSERR if an error ocurred (Np not expected).
- */
-static int
-store_ax_keys (struct CadetTunnel *t,
-               const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr,
-               uint32_t Np)
-{
-  int gap;
-
-
-  gap = Np - t->ax->Nr;
-  LOG (GNUNET_ERROR_TYPE_INFO, "Storing keys [%u, %u)\n", t->ax->Nr, Np);
-  if (MAX_KEY_GAP < gap)
-  {
-    /* Avoid DoS (forcing peer to do 2*33 chain HMAC operations) */
-    /* TODO: start new key exchange on return */
-    GNUNET_break_op (0);
-    LOG (GNUNET_ERROR_TYPE_WARNING, "Got message %u, expected %u+\n",
-         Np, t->ax->Nr);
-    return GNUNET_SYSERR;
-  }
-  if (0 > gap)
-  {
-    /* Delayed message: don't store keys, flag to try old keys. */
-    return GNUNET_SYSERR;
-  }
-
-  while (t->ax->Nr < Np)
-    store_skipped_key (t, HKr);
-
-  while (t->ax->skipped > MAX_SKIPPED_KEYS)
-    delete_skipped_key (t, t->ax->skipped_tail);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Decrypt and verify data with the appropriate tunnel key and verify that the
- * data has not been altered since it was sent by the remote peer.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the plaintext.
- * @param src Source of the message. Can overlap with @c dst.
- * @param size Size of the message.
- *
- * @return Size of the decrypted data, -1 if an error was encountered.
- */
-static int
-t_ax_decrypt_and_validate (struct CadetTunnel *t, void *dst,
-                           const struct GNUNET_CADET_TunnelEncryptedMessage *src,
-                           size_t size)
-{
-  struct CadetTunnelAxolotl *ax;
-  struct GNUNET_ShortHashCode msg_hmac;
-  struct GNUNET_HashCode hmac;
-  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
-  uint32_t Np;
-  uint32_t PNp;
-  size_t esize; /* Size of encryped payload */
-  size_t osize; /* Size of output (decrypted payload) */
-
-  esize = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  ax = t->ax;
-  if (NULL == ax)
-    return -1;
-
-  /* Try current HK */
-  t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &ax->HKr, &msg_hmac);
-  if (0 != memcmp (&msg_hmac, &src->hmac, sizeof (msg_hmac)))
-  {
-    static const char ctx[] = "axolotl ratchet";
-    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3]; /* RKp, NHKp, CKp */
-    struct GNUNET_CRYPTO_SymmetricSessionKey HK;
-    struct GNUNET_HashCode dh;
-    struct GNUNET_CRYPTO_EcdhePublicKey *DHRp;
-
-    /* Try Next HK */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  trying next HK\n");
-    t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &ax->NHKr, &msg_hmac);
-    if (0 != memcmp (&msg_hmac, &src->hmac, sizeof (msg_hmac)))
-    {
-      /* Try the skipped keys, if that fails, we're out of luck. */
-      return try_old_ax_keys (t, dst, src, size);
-    }
-    LOG (GNUNET_ERROR_TYPE_INFO, "next HK worked\n");
-
-    HK = ax->HKr;
-    ax->HKr = ax->NHKr;
-    t_h_decrypt (t, src, &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
-    DHRp = &plaintext_header.DHRs;
-    store_ax_keys (t, &HK, PNp);
-
-    /* RKp, NHKp, CKp = KDF (HMAC-HASH (RK, DH (DHRp, DHRs))) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs, DHRp, &dh);
-    t_ax_hmac_hash (&ax->RK, &hmac, &dh, sizeof (dh));
-    GNUNET_CRYPTO_kdf (keys, sizeof (keys), ctx, sizeof (ctx),
-                       &hmac, sizeof (hmac), NULL);
-
-    /* Commit "purported" keys */
-    ax->RK = keys[0];
-    ax->NHKr = keys[1];
-    ax->CKr = keys[2];
-    ax->DHRr = *DHRp;
-    ax->Nr = 0;
-    ax->ratchet_allowed = GNUNET_YES;
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "current HK\n");
-    t_h_decrypt (t, src, &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO, "  got AX Nr %u\n", Np);
-  if (Np != ax->Nr)
-    if (GNUNET_OK != store_ax_keys (t, &ax->HKr, Np))
-      /* Try the skipped keys, if that fails, we're out of luck. */
-      return try_old_ax_keys (t, dst, src, size);
-
-  osize = t_ax_decrypt (t, dst, &src[1], esize);
-  ax->Nr = Np + 1;
-
-  if (osize != esize)
-  {
-    GNUNET_break_op (0);
-    return -1;
-  }
-
-  return osize;
-}
-
-
-/**
- * Pick a connection on which send the next data message.
- *
- * @param t Tunnel on which to send the message.
- *
- * @return The connection on which to send the next message.
- */
-static struct CadetConnection *
-tunnel_get_connection (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  struct CadetConnection *best;
-  unsigned int qn;
-  unsigned int lowest_q;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "tunnel_get_connection %s\n", GCT_2s (t));
-  best = NULL;
-  lowest_q = UINT_MAX;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  connection %s: %u\n",
-         GCC_2s (iter->c), GCC_get_state (iter->c));
-    if (CADET_CONNECTION_READY == GCC_get_state (iter->c))
-    {
-      qn = GCC_get_qn (iter->c, GCC_is_origin (iter->c, GNUNET_YES));
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "    q_n %u, \n", qn);
-      if (qn < lowest_q)
-      {
-        best = iter->c;
-        lowest_q = qn;
-      }
-    }
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " selected: connection %s\n", GCC_2s (best));
-  return best;
-}
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * Calculates the average time and connection packet tracking.
- *
- * @param cls Closure (TunnelQueue handle).
- * @param c Connection this message was on.
- * @param q Connection queue handle (unused).
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
- */
-static void
-tun_message_sent (void *cls,
-              struct CadetConnection *c,
-              struct CadetConnectionQueue *q,
-              uint16_t type, int fwd, size_t size)
-{
-  struct CadetTunnelQueue *qt = cls;
-  struct CadetTunnel *t;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "tun_message_sent\n");
-
-  GNUNET_assert (NULL != qt->cont);
-  t = NULL == c ? NULL : GCC_get_tunnel (c);
-  qt->cont (qt->cont_cls, t, qt, type, size);
-  GNUNET_free (qt);
-}
-
-
-static unsigned int
-count_queued_data (const struct CadetTunnel *t)
-{
-  struct CadetTunnelDelayed *iter;
-  unsigned int count;
-
-  for (count = 0, iter = t->tq_head; iter != NULL; iter = iter->next)
-    count++;
-
-  return count;
-}
-
-/**
- * Delete a queued message: either was sent or the channel was destroyed
- * before the tunnel's key exchange had a chance to finish.
- *
- * @param tqd Delayed queue handle.
- */
-static void
-unqueue_data (struct CadetTunnelDelayed *tqd)
-{
-  GNUNET_CONTAINER_DLL_remove (tqd->t->tq_head, tqd->t->tq_tail, tqd);
-  GNUNET_free (tqd);
-}
-
-
-/**
- * Cache a message to be sent once tunnel is online.
- *
- * @param t Tunnel to hold the message.
- * @param msg Message itself (copy will be made).
- */
-static struct CadetTunnelDelayed *
-queue_data (struct CadetTunnel *t, const struct GNUNET_MessageHeader *msg)
-{
-  struct CadetTunnelDelayed *tqd;
-  uint16_t size = ntohs (msg->size);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "queue data on Tunnel %s\n", GCT_2s (t));
-
-  GNUNET_assert (GNUNET_NO == is_ready (t));
-
-  tqd = GNUNET_malloc (sizeof (struct CadetTunnelDelayed) + size);
-
-  tqd->t = t;
-  GNUNET_memcpy (&tqd[1], msg, size);
-  GNUNET_CONTAINER_DLL_insert_tail (t->tq_head, t->tq_tail, tqd);
-  return tqd;
-}
-
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- * @param existing_q In case this a transmission of previously queued data,
- *                   this should be TunnelQueue given to the client.
- *                   Otherwise, NULL.
- * @return Handle to cancel message.
- *         NULL if @c cont is NULL or an error happens and message is dropped.
- */
-static struct CadetTunnelQueue *
-send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                       struct CadetTunnel *t,
-                       struct CadetConnection *c,
-                       int force,
-                       GCT_sent cont,
-                       void *cont_cls,
-                       struct CadetTunnelQueue *existing_q)
-{
-  struct GNUNET_MessageHeader *msg;
-  struct GNUNET_CADET_TunnelEncryptedMessage *ax_msg;
-  struct CadetTunnelQueue *tq;
-  size_t size = ntohs (message->size);
-  char cbuf[sizeof (struct GNUNET_CADET_TunnelEncryptedMessage) + size] GNUNET_ALIGN;
-  size_t esize;
-  uint16_t type;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT Send on Tunnel %s\n", GCT_2s (t));
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    struct CadetTunnelDelayed *tqd;
-    /* A non null existing_q indicates sending of queued data.
-     * Should only happen after tunnel becomes ready.
-     */
-    GNUNET_assert (NULL == existing_q);
-    tqd = queue_data (t, message);
-    if (NULL == cont)
-      return NULL;
-    tq = GNUNET_new (struct CadetTunnelQueue);
-    tq->tqd = tqd;
-    tqd->tq = tq;
-    tq->cont = cont;
-    tq->cont_cls = cont_cls;
-    return tq;
-  }
-
-  GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
-
-  ax_msg = (struct GNUNET_CADET_TunnelEncryptedMessage *) cbuf;
-  msg = &ax_msg->header;
-  msg->size = htons (sizeof (struct GNUNET_CADET_TunnelEncryptedMessage) + size);
-  msg->type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED);
-  esize = t_ax_encrypt (t, &ax_msg[1], message, size);
-  ax_msg->Ns = htonl (t->ax->Ns++);
-  ax_msg->PNs = htonl (t->ax->PNs);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &ax_msg->DHRs);
-  t_h_encrypt (t, ax_msg);
-  t_hmac (&ax_msg->Ns, AX_HEADER_SIZE + esize, 0, &t->ax->HKs, &ax_msg->hmac);
-  GNUNET_assert (esize == size);
-
-  if (NULL == c)
-    c = tunnel_get_connection (t);
-  if (NULL == c)
-  {
-    /* Why is tunnel 'ready'? Should have been queued! */
-    if (NULL != t->destroy_task)
-    {
-      GNUNET_break (0);
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    }
-    return NULL; /* Drop... */
-  }
-  fwd = GCC_is_origin (c, GNUNET_YES);
-  ax_msg->cid = *GCC_get_id (c);
-  ax_msg->cemi = GCC_get_pid (c, fwd);
-
-  type = htons (message->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message of type %s with CEMI %u and CID %s\n",
-       GC_m2s (type),
-       htonl (ax_msg->cemi.pid),
-       GNUNET_sh2s (&ax_msg->cid.connection_of_tunnel));
-
-  if (NULL == cont)
-  {
-    (void) GCC_send_prebuilt_message (msg,
-                                      type,
-                                      ax_msg->cemi,
-                                      c,
-                                      fwd,
-                                      force, NULL, NULL);
-    return NULL;
-  }
-  if (NULL == existing_q)
-  {
-    tq = GNUNET_new (struct CadetTunnelQueue); /* FIXME valgrind: leak*/
-  }
-  else
-  {
-    tq = existing_q;
-    tq->tqd = NULL;
-  }
-  tq->cont = cont;
-  tq->cont_cls = cont_cls;
-  tq->cq = GCC_send_prebuilt_message (msg,
-                                      type,
-                                      ax_msg->cemi,
-                                      c,
-                                      fwd,
-                                      force,
-                                      &tun_message_sent, tq);
-  GNUNET_assert (NULL != tq->cq);
-
-  return tq;
-}
-
-
-/**
- * Send all cached messages that we can, tunnel is online.
- *
- * @param t Tunnel that holds the messages. Cannot be loopback.
- */
-static void
-send_queued_data (struct CadetTunnel *t)
-{
-  struct CadetTunnelDelayed *tqd;
-  struct CadetTunnelDelayed *next;
-  unsigned int room;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Send queued data, tunnel %s\n", GCT_2s (t));
-
-  if (GCT_is_loopback (t))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "  not ready yet: %s/%s\n",
-         estate2s (t->estate), cstate2s (t->cstate));
-    return;
-  }
-
-  room = GCT_get_connections_buffer (t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer space: %u\n", room);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  tq head: %p\n", t->tq_head);
-  for (tqd = t->tq_head; NULL != tqd && room > 0; tqd = next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " sending queued data\n");
-    next = tqd->next;
-    room--;
-    send_prebuilt_message ((struct GNUNET_MessageHeader *) &tqd[1],
-                           tqd->t, NULL, GNUNET_YES,
-                           NULL != tqd->tq ? tqd->tq->cont : NULL,
-                           NULL != tqd->tq ? tqd->tq->cont_cls : NULL,
-                           tqd->tq);
-    unqueue_data (tqd);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCT_send_queued_data end\n", GCP_2s (t->peer));
-}
-
-
-/**
- * @brief Resend the KX until we complete the handshake.
- *
- * @param cls Closure (tunnel).
- */
-static void
-kx_resend (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->rekey_task = NULL;
-  if (CADET_TUNNEL_KEY_OK == t->estate)
-  {
-    /* Should have been canceled on estate change */
-    GNUNET_break (0);
-    return;
-  }
-
-  GCT_send_kx (t, CADET_TUNNEL_KEY_SENT >= t->estate);
-}
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
- */
-static void
-ephm_sent (void *cls,
-           struct CadetConnection *c,
-           struct CadetConnectionQueue *q,
-           uint16_t type, int fwd, size_t size)
-{
-  struct CadetTunnel *t = cls;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "ephemeral sent %s\n", GC_m2s (type));
-
-  t->ephm_h = NULL;
-
-  if (CADET_TUNNEL_KEY_OK == t->estate)
-    return;
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_break (0);
-    GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-  }
-  t->rekey_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                                &kx_resend, t);
-
-}
-
-
-/**
- * Called only on shutdown, destroy every tunnel.
- *
- * @param cls Closure (unused).
- * @param key Current public key.
- * @param value Value in the hash map (tunnel).
- *
- * @return #GNUNET_YES, so we should continue to iterate,
- */
-static int
-destroy_iterator (void *cls,
-                const struct GNUNET_PeerIdentity *key,
-                void *value)
-{
-  struct CadetTunnel *t = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GCT_shutdown destroying tunnel at %p\n", t);
-  GCT_destroy (t);
-  return GNUNET_YES;
-}
-
-
-/**
- * Notify remote peer that we don't know a channel he is talking about,
- * probably CHANNEL_DESTROY was missed.
- *
- * @param t Tunnel on which to notify.
- * @param gid ID of the channel.
- */
-static void
-send_channel_destroy (struct CadetTunnel *t,
-                      struct GNUNET_CADET_ChannelTunnelNumber gid)
-{
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = gid;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "WARNING destroying unknown channel %u on tunnel %s\n",
-       ntohl (gid.cn),
-       GCT_2s (t));
-  send_prebuilt_message (&msg.header, t, NULL, GNUNET_YES, NULL, NULL, NULL);
-}
-
-
-/**
- * Demultiplex data per channel and call appropriate channel handler.
- *
- * @param t Tunnel on which the data came.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_data (struct CadetTunnel *t,
-             const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-             int fwd)
-{
-  struct CadetChannel *ch;
-  char buf[128];
-  size_t size;
-  uint16_t type;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size <
-      sizeof (struct GNUNET_CADET_ChannelAppDataMessage) +
-      sizeof (struct GNUNET_MessageHeader))
-  {
-    GNUNET_break (0);
-    return;
-  }
-  type = ntohs (msg[1].header.type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " payload of type %s\n", GC_m2s (type));
-  SPRINTF (buf, "# received payload of type %hu", type);
-  GNUNET_STATISTICS_update (stats, buf, 1, GNUNET_NO);
-
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# data on unknown channel",
-                              1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "channel 0x%X unknown\n",
-         ntohl (msg->ctn.cn));
-    send_channel_destroy (t, msg->ctn);
-    return;
-  }
-
-  GCCH_handle_data (ch, msg, fwd);
-}
-
-
-/**
- * Demultiplex data ACKs per channel and update appropriate channel buffer info.
- *
- * @param t Tunnel on which the DATA ACK came.
- * @param msg DATA ACK message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_data_ack (struct CadetTunnel *t,
-                 const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                 int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelDataAckMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats, "# data ack on unknown channel",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_data_ack (ch, msg, fwd);
-}
-
-
-/**
- * Handle channel create.
- *
- * @param t Tunnel on which the message came.
- * @param msg ChannelCreate message.
- */
-static void
-handle_ch_create (struct CadetTunnel *t,
-                  const struct GNUNET_CADET_ChannelOpenMessage *msg)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelOpenMessage))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL != ch && ! GCT_is_loopback (t))
-  {
-    /* Probably a retransmission, safe to ignore */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   already exists...\n");
-  }
-  ch = GCCH_handle_create (t, msg);
-  if (NULL != ch)
-    GCT_add_channel (t, ch);
-}
-
-
-
-/**
- * Handle channel NACK: check correctness and call channel handler for NACKs.
- *
- * @param t Tunnel on which the NACK came.
- * @param msg NACK message.
- */
-static void
-handle_ch_nack (struct CadetTunnel *t,
-                const struct GNUNET_CADET_ChannelManageMessage *msg)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats, "# channel NACK on unknown channel",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_nack (ch);
-}
-
-
-/**
- * Handle a CHANNEL ACK (SYNACK/ACK).
- *
- * @param t Tunnel on which the CHANNEL ACK came.
- * @param msg CHANNEL ACK message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_ch_ack (struct CadetTunnel *t,
-               const struct GNUNET_CADET_ChannelManageMessage *msg,
-               int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# channel ack on unknown channel",
-                              1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_ack (ch, msg, fwd);
-}
-
-
-/**
- * Handle a channel destruction message.
- *
- * @param t Tunnel on which the message came.
- * @param msg Channel destroy message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_ch_destroy (struct CadetTunnel *t,
-                   const struct GNUNET_CADET_ChannelManageMessage *msg,
-                   int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    /* Probably a retransmission, safe to ignore */
-    return;
-  }
-
-  GCCH_handle_destroy (ch, msg, fwd);
-}
-
-
-/**
- * Free Axolotl data.
- *
- * @param t Tunnel.
- */
-static void
-destroy_ax (struct CadetTunnel *t)
-{
-  if (NULL == t->ax)
-    return;
-
-  GNUNET_free_non_null (t->ax->DHRs);
-  GNUNET_free_non_null (t->ax->kx_0);
-  while (NULL != t->ax->skipped_head)
-    delete_skipped_key (t, t->ax->skipped_head);
-  GNUNET_assert (0 == t->ax->skipped);
-
-  GNUNET_free (t->ax);
-  t->ax = NULL;
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-    t->rekey_task = NULL;
-  }
-  if (NULL != t->ephm_h)
-  {
-    GCC_cancel (t->ephm_h);
-    t->ephm_h = NULL;
-  }
-}
-
-
-/**
- * Demultiplex by message type and call appropriate handler for a message
- * towards a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msgh Message header.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_decrypted (struct CadetTunnel *t,
-                  const struct GNUNET_MessageHeader *msgh,
-                  int fwd)
-{
-  uint16_t type;
-  char buf[256];
-
-  type = ntohs (msgh->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "<-- %s on %s\n", GC_m2s (type), GCT_2s (t));
-  SPRINTF (buf, "# received encrypted of type %hu (%s)", type, GC_m2s (type));
-  GNUNET_STATISTICS_update (stats, buf, 1, GNUNET_NO);
-
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE:
-      /* Do nothing, connection aleady got updated. */
-      GNUNET_STATISTICS_update (stats, "# keepalives received", 1, GNUNET_NO);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      /* Don't send hop ACK, wait for client to ACK */
-      handle_data (t, (struct GNUNET_CADET_ChannelAppDataMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      handle_data_ack (t, (struct GNUNET_CADET_ChannelDataAckMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      handle_ch_create (t, (struct GNUNET_CADET_ChannelOpenMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      handle_ch_nack (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      handle_ch_ack (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      handle_ch_destroy (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh, fwd);
-      break;
-
-    default:
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "end-to-end message not known (%u)\n",
-           ntohs (msgh->type));
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-  }
-}
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Decrypt and process an encrypted message.
- *
- * Calls the appropriate handler for a message in a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msg Message header.
- */
-void
-GCT_handle_encrypted (struct CadetTunnel *t,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  uint16_t size = ntohs (msg->header.size);
-  char cbuf [size];
-  int decrypted_size;
-  const struct GNUNET_MessageHeader *msgh;
-  unsigned int off;
-
-  GNUNET_STATISTICS_update (stats, "# received encrypted", 1, GNUNET_NO);
-
-  decrypted_size = t_ax_decrypt_and_validate (t, cbuf, msg, size);
-
-  if (-1 == decrypted_size)
-  {
-    GNUNET_STATISTICS_update (stats, "# unable to decrypt", 1, GNUNET_NO);
-    if (CADET_TUNNEL_KEY_PING <= t->estate)
-    {
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING, "Wrong crypto, tunnel %s\n", GCT_2s (t));
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    }
-    return;
-  }
-  GCT_change_estate (t, CADET_TUNNEL_KEY_OK);
-
-  /* FIXME: this is bad, as the structs returned from
-     this loop may be unaligned, see util's MST for
-     how to do this right. */
-  off = 0;
-  while (off + sizeof (struct GNUNET_MessageHeader) <= decrypted_size)
-  {
-    uint16_t msize;
-
-    msgh = (const struct GNUNET_MessageHeader *) &cbuf[off];
-    msize = ntohs (msgh->size);
-    if (msize < sizeof (struct GNUNET_MessageHeader))
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    if (off + msize < decrypted_size)
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    handle_decrypted (t, msgh, GNUNET_SYSERR);
-    off += msize;
-  }
-}
-
-
-/**
- * Handle a Key eXchange message.
- *
- * @param t Tunnel on which the message came.
- * @param msg KX message itself.
- */
-void
-GCT_handle_kx (struct CadetTunnel *t,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-  struct CadetTunnelAxolotl *ax;
-  struct GNUNET_HashCode key_material[3];
-  struct GNUNET_CRYPTO_SymmetricSessionKey keys[5];
-  const char salt[] = "CADET Axolotl salt";
-  const struct GNUNET_PeerIdentity *pid;
-  int am_I_alice;
-
-  CADET_TIMING_START;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "<== {        KX} on %s\n", GCT_2s (t));
-
-  if (NULL == t->ax)
-  {
-    /* Something is wrong if ax is NULL. Whose fault it is? */
-    return;
-  }
-  ax = t->ax;
-
-  pid = GCT_get_destination (t);
-  if (0 > GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, pid))
-    am_I_alice = GNUNET_YES;
-  else if (0 < GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, pid))
-    am_I_alice = GNUNET_NO;
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  if (0 != (GNUNET_CADET_KX_FLAG_FORCE_REPLY & ntohl (msg->flags)))
-  {
-    if (NULL != t->rekey_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->rekey_task);
-      t->rekey_task = NULL;
-    }
-    GCT_send_kx (t, GNUNET_NO);
-  }
-
-  if (0 == memcmp (&ax->DHRr, &msg->ratchet_key, sizeof(msg->ratchet_key)))
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, " known ratchet key, exit\n");
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO, " is Alice? %s\n", am_I_alice ? "YES" : "NO");
-
-  ax->DHRr = msg->ratchet_key;
-
-  /* ECDH A B0 */
-  if (GNUNET_YES == am_I_alice)
-  {
-    GNUNET_CRYPTO_eddsa_ecdh (id_key,              /* A */
-                              &msg->ephemeral_key, /* B0 */
-                              &key_material[0]);
-  }
-  else
-  {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* B0 */
-                              &pid->public_key,    /* A */
-                              &key_material[0]);
-  }
-
-  /* ECDH A0 B */
-  if (GNUNET_YES == am_I_alice)
-  {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* A0 */
-                              &pid->public_key,    /* B */
-                              &key_material[1]);
-  }
-  else
-  {
-    GNUNET_CRYPTO_eddsa_ecdh (id_key,              /* A */
-                              &msg->ephemeral_key, /* B0 */
-                              &key_material[1]);
-
-
-  }
-
-  /* ECDH A0 B0 */
-  /* (This is the triple-DH, we could probably safely skip this,
-     as A0/B0 are already in the key material.) */
-  GNUNET_CRYPTO_ecc_ecdh (ax->kx_0,             /* A0 or B0 */
-                          &msg->ephemeral_key,  /* B0 or A0 */
-                          &key_material[2]);
-
-  #if DUMP_KEYS_TO_STDERR
-  {
-    unsigned int i;
-    for (i = 0; i < 3; i++)
-      LOG (GNUNET_ERROR_TYPE_INFO, "km[%u]: %s\n",
-           i, GNUNET_h2s (&key_material[i]));
-  }
-  #endif
-
-  /* KDF */
-  GNUNET_CRYPTO_kdf (keys, sizeof (keys),
-                     salt, sizeof (salt),
-                     &key_material, sizeof (key_material), NULL);
-
-  if (0 == memcmp (&ax->RK, &keys[0], sizeof(ax->RK)))
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, " known handshake key, exit\n");
-    return;
-  }
-  ax->RK = keys[0];
-  if (GNUNET_YES == am_I_alice)
-  {
-    ax->HKr = keys[1];
-    ax->NHKs = keys[2];
-    ax->NHKr = keys[3];
-    ax->CKr = keys[4];
-    ax->ratchet_flag = GNUNET_YES;
-  }
-  else
-  {
-    ax->HKs = keys[1];
-    ax->NHKr = keys[2];
-    ax->NHKs = keys[3];
-    ax->CKs = keys[4];
-    ax->ratchet_flag = GNUNET_NO;
-    ax->ratchet_allowed = GNUNET_NO;
-    ax->ratchet_counter = 0;
-    ax->ratchet_expiration =
-      GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(), ratchet_time);
-  }
-  ax->PNs = 0;
-  ax->Nr = 0;
-  ax->Ns = 0;
-
-  GCT_change_estate (t, CADET_TUNNEL_KEY_PING);
-  send_queued_data (t);
-
-  CADET_TIMING_END;
-}
-
-/**
- * Initialize the tunnel subsystem.
- *
- * @param c Configuration handle.
- * @param key ECC private key, to derive all other keys and do crypto.
- */
-void
-GCT_init (const struct GNUNET_CONFIGURATION_Handle *c,
-          const struct GNUNET_CRYPTO_EddsaPrivateKey *key)
-{
-  unsigned int expected_overhead;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-
-  expected_overhead = 0;
-  expected_overhead += sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  expected_overhead += sizeof (struct GNUNET_CADET_ChannelAppDataMessage);
-  expected_overhead += sizeof (struct GNUNET_CADET_ConnectionEncryptedAckMessage);
-  GNUNET_assert (GNUNET_CONSTANTS_CADET_P2P_OVERHEAD == expected_overhead);
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "RATCHET_MESSAGES",
-                                             &ratchet_messages))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_MESSAGES",
-                               "USING DEFAULT");
-    ratchet_messages = 64;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "RATCHET_TIME",
-                                           &ratchet_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET", "RATCHET_TIME", "USING DEFAULT");
-    ratchet_time = GNUNET_TIME_UNIT_HOURS;
-  }
-
-
-  id_key = key;
-  tunnels = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_YES);
-}
-
-
-/**
- * Shut down the tunnel subsystem.
- */
-void
-GCT_shutdown (void)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down tunnels\n");
-  GNUNET_CONTAINER_multipeermap_iterate (tunnels, &destroy_iterator, NULL);
-  GNUNET_CONTAINER_multipeermap_destroy (tunnels);
-}
-
-
-/**
- * Create a tunnel.
- *
- * @param destination Peer this tunnel is towards.
- */
-struct CadetTunnel *
-GCT_new (struct CadetPeer *destination)
-{
-  struct CadetTunnel *t;
-
-  t = GNUNET_new (struct CadetTunnel);
-  t->next_ctn.cn = 0;
-  t->peer = destination;
-
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multipeermap_put (tunnels, GCP_get_id (destination), t,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
-  {
-    GNUNET_break (0);
-    GNUNET_free (t);
-    return NULL;
-  }
-  t->ax = GNUNET_new (struct CadetTunnelAxolotl);
-  new_ephemeral (t);
-  t->ax->kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
-  return t;
-}
-
-
-/**
- * Change the tunnel's connection state.
- *
- * @param t Tunnel whose connection state to change.
- * @param cstate New connection state.
- */
-void
-GCT_change_cstate (struct CadetTunnel* t, enum CadetTunnelCState cstate)
-{
-  if (NULL == t)
-    return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s cstate %s => %s\n",
-       GCP_2s (t->peer), cstate2s (t->cstate), cstate2s (cstate));
-  if (myid != GCP_get_short_id (t->peer) &&
-      CADET_TUNNEL_READY != t->cstate &&
-      CADET_TUNNEL_READY == cstate)
-  {
-    t->cstate = cstate;
-    if (CADET_TUNNEL_KEY_OK == t->estate)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered send queued data\n");
-      send_queued_data (t);
-    }
-    else if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered KX\n");
-      GCT_send_kx (t, GNUNET_NO);
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "estate %s\n", estate2s (t->estate));
-    }
-  }
-  t->cstate = cstate;
-
-  if (CADET_TUNNEL_READY == cstate
-      && CONNECTIONS_PER_TUNNEL <= GCT_count_connections (t))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered stop dht\n");
-    GCP_stop_search (t->peer);
-  }
-}
-
-
-/**
- * Change the tunnel encryption state.
- *
- * If the encryption state changes to OK, stop the rekey task.
- *
- * @param t Tunnel whose encryption state to change, or NULL.
- * @param state New encryption state.
- */
-void
-GCT_change_estate (struct CadetTunnel* t, enum CadetTunnelEState state)
-{
-  enum CadetTunnelEState old;
-
-  if (NULL == t)
-    return;
-
-  old = t->estate;
-  t->estate = state;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s estate was %s\n",
-       GCP_2s (t->peer), estate2s (old));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s estate is now %s\n",
-       GCP_2s (t->peer), estate2s (t->estate));
-
-  if (CADET_TUNNEL_KEY_OK != old && CADET_TUNNEL_KEY_OK == t->estate)
-  {
-    if (NULL != t->rekey_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->rekey_task);
-      t->rekey_task = NULL;
-    }
-    /* Send queued data if tunnel is not loopback */
-    if (myid != GCP_get_short_id (t->peer))
-      send_queued_data (t);
-  }
-}
-
-
-/**
- * @brief Check if tunnel has too many connections, and remove one if necessary.
- *
- * Currently this means the newest connection, unless it is a direct one.
- * Implemented as a task to avoid freeing a connection that is in the middle
- * of being created/processed.
- *
- * @param cls Closure (Tunnel to check).
- */
-static void
-trim_connections (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->trim_connections_task = NULL;
-  if (GCT_count_connections (t) > 2 * CONNECTIONS_PER_TUNNEL)
-  {
-    struct CadetTConnection *iter;
-    struct CadetTConnection *c;
-
-    for (c = iter = t->connection_head; NULL != iter; iter = iter->next)
-    {
-      if ((iter->created.abs_value_us > c->created.abs_value_us)
-          && GNUNET_NO == GCC_is_direct (iter->c))
-      {
-        c = iter;
-      }
-    }
-    if (NULL != c)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Too many connections on tunnel %s\n",
-           GCT_2s (t));
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Destroying connection %s\n",
-           GCC_2s (c->c));
-      GCC_destroy (c->c);
-    }
-    else
-    {
-      GNUNET_break (0);
-    }
-  }
-}
-
-
-/**
- * Add a connection to a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_add_connection (struct CadetTunnel *t, struct CadetConnection *c)
-{
-  struct CadetTConnection *aux;
-
-  GNUNET_assert (NULL != c);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add connection %s\n", GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " to tunnel %s\n", GCT_2s (t));
-  for (aux = t->connection_head; aux != NULL; aux = aux->next)
-    if (aux->c == c)
-      return;
-
-  aux = GNUNET_new (struct CadetTConnection);
-  aux->c = c;
-  aux->created = GNUNET_TIME_absolute_get ();
-
-  GNUNET_CONTAINER_DLL_insert (t->connection_head, t->connection_tail, aux);
-
-  if (CADET_TUNNEL_SEARCHING == t->cstate)
-    GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-
-  if (NULL != t->trim_connections_task)
-    t->trim_connections_task = GNUNET_SCHEDULER_add_now (&trim_connections, t);
-}
-
-
-/**
- * Remove a connection from a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_remove_connection (struct CadetTunnel *t,
-                       struct CadetConnection *c)
-{
-  struct CadetTConnection *aux;
-  struct CadetTConnection *next;
-  unsigned int conns;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Removing connection %s from tunnel %s\n",
-       GCC_2s (c), GCT_2s (t));
-  for (aux = t->connection_head; aux != NULL; aux = next)
-  {
-    next = aux->next;
-    if (aux->c == c)
-    {
-      GNUNET_CONTAINER_DLL_remove (t->connection_head, t->connection_tail, aux);
-      GNUNET_free (aux);
-    }
-  }
-
-  conns = GCT_count_connections (t);
-  if (0 == conns
-      && NULL == t->destroy_task
-      && CADET_TUNNEL_SHUTDOWN != t->cstate
-      && GNUNET_NO == shutting_down)
-  {
-    if (0 == GCT_count_any_connections (t))
-      GCT_change_cstate (t, CADET_TUNNEL_SEARCHING);
-    else
-      GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-  }
-
-  /* Start new connections if needed */
-  if (CONNECTIONS_PER_TUNNEL > conns
-      && CADET_TUNNEL_SHUTDOWN != t->cstate
-      && GNUNET_NO == shutting_down)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  too few connections, getting new ones\n");
-    GCP_connect (t->peer); /* Will change cstate to WAITING when possible */
-    return;
-  }
-
-  /* If not marked as ready, no change is needed */
-  if (CADET_TUNNEL_READY != t->cstate)
-    return;
-
-  /* Check if any connection is ready to maintain cstate */
-  for (aux = t->connection_head; aux != NULL; aux = aux->next)
-    if (CADET_CONNECTION_READY == GCC_get_state (aux->c))
-      return;
-}
-
-
-/**
- * Add a channel to a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_add_channel (struct CadetTunnel *t,
-                 struct CadetChannel *ch)
-{
-  struct CadetTChannel *aux;
-
-  GNUNET_assert (NULL != ch);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Adding channel %p to tunnel %p\n", ch, t);
-
-  for (aux = t->channel_head; aux != NULL; aux = aux->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  already there %p\n", aux->ch);
-    if (aux->ch == ch)
-      return;
-  }
-
-  aux = GNUNET_new (struct CadetTChannel);
-  aux->ch = ch;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " adding %p to %p\n", aux, t->channel_head);
-  GNUNET_CONTAINER_DLL_insert_tail (t->channel_head,
-                                    t->channel_tail,
-                                    aux);
-
-  if (NULL != t->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->destroy_task);
-    t->destroy_task = NULL;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " undo destroy!\n");
-  }
-}
-
-
-/**
- * Remove a channel from a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_remove_channel (struct CadetTunnel *t, struct CadetChannel *ch)
-{
-  struct CadetTChannel *aux;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Removing channel %p from tunnel %p\n", ch, t);
-  for (aux = t->channel_head; aux != NULL; aux = aux->next)
-  {
-    if (aux->ch == ch)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " found! %s\n", GCCH_2s (ch));
-      GNUNET_CONTAINER_DLL_remove (t->channel_head,
-                                   t->channel_tail,
-                                   aux);
-      GNUNET_free (aux);
-      return;
-    }
-  }
-}
-
-
-/**
- * Search for a channel by global ID.
- *
- * @param t Tunnel containing the channel.
- * @param ctn Public channel number.
- *
- * @return channel handler, NULL if doesn't exist
- */
-struct CadetChannel *
-GCT_get_channel (struct CadetTunnel *t,
-                 struct GNUNET_CADET_ChannelTunnelNumber ctn)
-{
-  struct CadetTChannel *iter;
-
-  if (NULL == t)
-    return NULL;
-
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    if (GCCH_get_id (iter->ch).cn == ctn.cn)
-      break;
-  }
-
-  return NULL == iter ? NULL : iter->ch;
-}
-
-
-/**
- * @brief Destroy a tunnel and free all resources.
- *
- * Should only be called a while after the tunnel has been marked as destroyed,
- * in case there is a new channel added to the same peer shortly after marking
- * the tunnel. This way we avoid a new public key handshake.
- *
- * @param cls Closure (tunnel to destroy).
- */
-static void
-delayed_destroy (void *cls)
-{
-  struct CadetTunnel *t = cls;
-  struct CadetTConnection *iter;
-
-  t->destroy_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "delayed destroying tunnel %p\n",
-       t);
-  t->cstate = CADET_TUNNEL_SHUTDOWN;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    GCC_send_destroy (iter->c);
-  }
-  GCT_destroy (t);
-}
-
-
-/**
- * Tunnel is empty: destroy it.
- *
- * Notifies all connections about the destruction.
- *
- * @param t Tunnel to destroy.
- */
-void
-GCT_destroy_empty (struct CadetTunnel *t)
-{
-  if (GNUNET_YES == shutting_down)
-    return; /* Will be destroyed immediately anyway */
-
-  if (NULL != t->destroy_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Tunnel %s is already scheduled for destruction. Tunnel debug dump:\n",
-         GCT_2s (t));
-    GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    GNUNET_break (0);
-    /* should never happen, tunnel can only become empty once, and the
-     * task identifier should be NO_TASK (cleaned when the tunnel was created
-     * or became un-empty)
-     */
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s empty: scheduling destruction\n",
-       GCT_2s (t));
-
-  // FIXME make delay a config option
-  t->destroy_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                                  &delayed_destroy, t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Scheduled destroy of %p as %p\n",
-       t, t->destroy_task);
-}
-
-
-/**
- * Destroy tunnel if empty (no more channels).
- *
- * @param t Tunnel to destroy if empty.
- */
-void
-GCT_destroy_if_empty (struct CadetTunnel *t)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s destroy if empty\n", GCT_2s (t));
-  if (0 < GCT_count_channels (t))
-    return;
-
-  GCT_destroy_empty (t);
-}
-
-
-/**
- * Destroy the tunnel.
- *
- * This function does not generate any warning traffic to clients or peers.
- *
- * Tasks:
- * Cancel messages belonging to this tunnel queued to neighbors.
- * Free any allocated resources linked to the tunnel.
- *
- * @param t The tunnel to destroy.
- */
-void
-GCT_destroy (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter_c;
-  struct CadetTConnection *next_c;
-  struct CadetTChannel *iter_ch;
-  struct CadetTChannel *next_ch;
-  unsigned int keepalives_queued;
-
-  if (NULL == t)
-    return;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying tunnel %s\n",
-       GCP_2s (t->peer));
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multipeermap_remove (tunnels,
-                                                      GCP_get_id (t->peer), t));
-
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = next_c)
-  {
-    next_c = iter_c->next;
-    GCC_destroy (iter_c->c);
-  }
-  for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = next_ch)
-  {
-    next_ch = iter_ch->next;
-    GCCH_destroy (iter_ch->ch);
-    /* Should only happen on shutdown, but it's ok. */
-  }
-  keepalives_queued = 0;
-  while (NULL != t->tq_head)
-  {
-    /* Should have been cleaned by destuction of channel. */
-    struct GNUNET_MessageHeader *mh;
-    uint16_t type;
-
-    mh = (struct GNUNET_MessageHeader *) &t->tq_head[1];
-    type = ntohs (mh->type);
-    if (0 == keepalives_queued && GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE == type)
-    {
-      keepalives_queued = 1;
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "one keepalive left behind on tunnel shutdown\n");
-    }
-    else if (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY == type)
-    {
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "tunnel destroyed before a CHANNEL_DESTROY was sent to peer\n");
-    }
-    else
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           "message left behind on tunnel shutdown: %s\n",
-           GC_m2s (type));
-    }
-    unqueue_data (t->tq_head);
-  }
-
-
-  if (NULL != t->destroy_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "cancelling dest: %p\n",
-         t->destroy_task);
-    GNUNET_SCHEDULER_cancel (t->destroy_task);
-    t->destroy_task = NULL;
-  }
-
-  if (NULL != t->trim_connections_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "cancelling trim: %p\n",
-         t->trim_connections_task);
-    GNUNET_SCHEDULER_cancel (t->trim_connections_task);
-    t->trim_connections_task = NULL;
-  }
-
-  GNUNET_STATISTICS_update (stats, "# tunnels", -1, GNUNET_NO);
-  GCP_set_tunnel (t->peer, NULL);
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-    t->rekey_task = NULL;
-  }
-  if (NULL != t->ax)
-    destroy_ax (t);
-
-  GNUNET_free (t);
-}
-
-
-/**
- * @brief Use the given path for the tunnel.
- * Update the next and prev hops (and RCs).
- * (Re)start the path refresh in case the tunnel is locally owned.
- *
- * @param t Tunnel to update.
- * @param p Path to use.
- *
- * @return Connection created.
- */
-struct CadetConnection *
-GCT_use_path (struct CadetTunnel *t, struct CadetPeerPath *path)
-{
-  struct CadetConnection *c;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-  unsigned int own_pos;
-
-  if (NULL == t || NULL == path)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  if (CADET_TUNNEL_SHUTDOWN == t->cstate)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  for (own_pos = 0; own_pos < path->length; own_pos++)
-  {
-    if (path->peers[own_pos] == myid)
-      break;
-  }
-  if (own_pos >= path->length)
-  {
-    GNUNET_break_op (0);
-    return NULL;
-  }
-
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE, &cid, sizeof (cid));
-  c = GCC_new (&cid, t, path, own_pos);
-  if (NULL == c)
-  {
-    /* Path was flawed */
-    return NULL;
-  }
-  GCT_add_connection (t, c);
-  return c;
-}
-
-
-/**
- * Count all created connections of a tunnel. Not necessarily ready connections!
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections created, either being established or ready.
- */
-unsigned int
-GCT_count_any_connections (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int count;
-
-  if (NULL == t)
-    return 0;
-
-  for (count = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-    count++;
-
-  return count;
-}
-
-
-/**
- * Count established (ready) connections of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections.
- */
-unsigned int
-GCT_count_connections (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int count;
-
-  if (NULL == t)
-    return 0;
-
-  for (count = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-    if (CADET_CONNECTION_READY == GCC_get_state (iter->c))
-      count++;
-
-  return count;
-}
-
-
-/**
- * Count channels of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of channels.
- */
-unsigned int
-GCT_count_channels (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int count;
-
-  for (count = 0, iter = t->channel_head;
-       NULL != iter;
-       iter = iter->next, count++) /* skip */;
-
-  return count;
-}
-
-
-/**
- * Get the connectivity state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's connectivity state.
- */
-enum CadetTunnelCState
-GCT_get_cstate (struct CadetTunnel *t)
-{
-  if (NULL == t)
-  {
-    GNUNET_assert (0);
-    return (enum CadetTunnelCState) -1;
-  }
-  return t->cstate;
-}
-
-
-/**
- * Get the encryption state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's encryption state.
- */
-enum CadetTunnelEState
-GCT_get_estate (struct CadetTunnel *t)
-{
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    return (enum CadetTunnelEState) -1;
-  }
-  return t->estate;
-}
-
-/**
- * Get the maximum buffer space for a tunnel towards a local client.
- *
- * @param t Tunnel.
- *
- * @return Biggest buffer space offered by any channel in the tunnel.
- */
-unsigned int
-GCT_get_channels_buffer (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int buffer;
-  unsigned int ch_buf;
-
-  if (NULL == t->channel_head)
-  {
-    /* Probably getting buffer for a channel create/handshake. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  no channels, allow max\n");
-    return MIN_TUNNEL_BUFFER;
-  }
-
-  buffer = 0;
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    ch_buf = get_channel_buffer (iter);
-    if (ch_buf > buffer)
-      buffer = ch_buf;
-  }
-  if (MIN_TUNNEL_BUFFER > buffer)
-    return MIN_TUNNEL_BUFFER;
-
-  if (MAX_TUNNEL_BUFFER < buffer)
-  {
-    GNUNET_break (0);
-    return MAX_TUNNEL_BUFFER;
-  }
-  return buffer;
-}
-
-
-/**
- * Get the total buffer space for a tunnel for P2P traffic.
- *
- * @param t Tunnel.
- *
- * @return Buffer space offered by all connections in the tunnel.
- */
-unsigned int
-GCT_get_connections_buffer (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int buffer;
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    if (count_queued_data (t) >= 3)
-      return 0;
-    else
-      return 1;
-  }
-
-  buffer = 0;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    if (GCC_get_state (iter->c) != CADET_CONNECTION_READY)
-    {
-      continue;
-    }
-    buffer += get_connection_buffer (iter);
-  }
-
-  return buffer;
-}
-
-
-/**
- * Get the tunnel's destination.
- *
- * @param t Tunnel.
- *
- * @return ID of the destination peer.
- */
-const struct GNUNET_PeerIdentity *
-GCT_get_destination (struct CadetTunnel *t)
-{
-  return GCP_get_id (t->peer);
-}
-
-
-/**
- * Get the tunnel's next free global channel ID.
- *
- * @param t Tunnel.
- *
- * @return GID of a channel free to use.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCT_get_next_ctn (struct CadetTunnel *t)
-{
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-  struct GNUNET_CADET_ChannelTunnelNumber mask;
-  int result;
-
-  /* Set bit 30 depending on the ID relationship. Bit 31 is always 0 for GID.
-   * If our ID is bigger or loopback tunnel, start at 0, bit 30 = 0
-   * If peer's ID is bigger, start at 0x4... bit 30 = 1
-   */
-  result = GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, GCP_get_id (t->peer));
-  if (0 > result)
-    mask.cn = htonl (0x40000000);
-  else
-    mask.cn = 0x0;
-  t->next_ctn.cn |= mask.cn;
-
-  while (NULL != GCT_get_channel (t, t->next_ctn))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %u exists...\n",
-         t->next_ctn.cn);
-    t->next_ctn.cn = htonl ((ntohl (t->next_ctn.cn) + 1) & ~GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-    t->next_ctn.cn |= mask.cn;
-  }
-  ctn = t->next_ctn;
-  t->next_ctn.cn = (t->next_ctn.cn + 1) & ~GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-  t->next_ctn.cn |= mask.cn;
-
-  return ctn;
-}
-
-
-/**
- * Send ACK on one or more channels due to buffer in connections.
- *
- * @param t Channel which has some free buffer space.
- */
-void
-GCT_unchoke_channels (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int buffer;
-  unsigned int channels = GCT_count_channels (t);
-  unsigned int choked_n;
-  struct CadetChannel *choked[channels];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCT_unchoke_channels on %s\n", GCT_2s (t));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " head: %p\n", t->channel_head);
-  if (NULL != t->channel_head)
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " head ch: %p\n", t->channel_head->ch);
-
-  if (NULL != t->tq_head)
-    send_queued_data (t);
-
-  /* Get buffer space */
-  buffer = GCT_get_connections_buffer (t);
-  if (0 == buffer)
-  {
-    return;
-  }
-
-  /* Count and remember choked channels */
-  choked_n = 0;
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    if (GNUNET_NO == get_channel_allowed (iter))
-    {
-      choked[choked_n++] = iter->ch;
-    }
-  }
-
-  /* Unchoke random channels */
-  while (0 < buffer && 0 < choked_n)
-  {
-    unsigned int r = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                               choked_n);
-    GCCH_allow_client (choked[r], GCCH_is_origin (choked[r], GNUNET_YES));
-    choked_n--;
-    buffer--;
-    choked[r] = choked[choked_n];
-  }
-}
-
-
-/**
- * Send ACK on one or more connections due to buffer space to the client.
- *
- * Iterates all connections of the tunnel and sends ACKs appropriately.
- *
- * @param t Tunnel.
- */
-void
-GCT_send_connection_acks (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  uint32_t allowed;
-  uint32_t to_allow;
-  uint32_t allow_per_connection;
-  unsigned int cs;
-  unsigned int buffer;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel send connection ACKs on %s\n",
-       GCT_2s (t));
-
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (CADET_TUNNEL_READY != t->cstate)
-    return;
-
-  buffer = GCT_get_channels_buffer (t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer %u\n", buffer);
-
-  /* Count connections, how many messages are already allowed */
-  cs = GCT_count_connections (t);
-  for (allowed = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    allowed += get_connection_allowed (iter);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  allowed %u\n", allowed);
-
-  /* Make sure there is no overflow */
-  if (allowed > buffer)
-    return;
-
-  /* Authorize connections to send more data */
-  to_allow = buffer - allowed;
-
-  for (iter = t->connection_head;
-       NULL != iter && to_allow > 0;
-       iter = iter->next)
-  {
-    if (CADET_CONNECTION_READY != GCC_get_state (iter->c)
-        || get_connection_allowed (iter) > 64 / 3)
-    {
-      continue;
-    }
-    GNUNET_assert(cs != 0);
-    allow_per_connection = to_allow/cs;
-    to_allow -= allow_per_connection;
-    cs--;
-    GCC_allow (iter->c, allow_per_connection,
-               GCC_is_origin (iter->c, GNUNET_NO));
-  }
-
-  if (0 != to_allow)
-  {
-    /* Since we don't allow if it's allowed to send 64/3, this can happen. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  reminding to_allow: %u\n", to_allow);
-  }
-}
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCT_cancel (struct CadetTunnelQueue *q)
-{
-  if (NULL != q->cq)
-  {
-    GNUNET_assert (NULL == q->tqd);
-    GCC_cancel (q->cq);
-    /* tun_message_sent() will be called and free q */
-  }
-  else if (NULL != q->tqd)
-  {
-    unqueue_data (q->tqd);
-    q->tqd = NULL;
-    if (NULL != q->cont)
-      q->cont (q->cont_cls, NULL, q, 0, 0);
-    GNUNET_free (q);
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
-}
-
-
-/**
- * Check if the tunnel has queued traffic.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if there is queued traffic
- *         #GNUNET_NO otherwise
- */
-int
-GCT_has_queued_traffic (struct CadetTunnel *t)
-{
-  return (NULL != t->tq_head) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection if not provided.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel message. NULL if @c cont is NULL.
- */
-struct CadetTunnelQueue *
-GCT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           struct CadetTunnel *t,
-                           struct CadetConnection *c,
-                           int force, GCT_sent cont, void *cont_cls)
-{
-  return send_prebuilt_message (message, t, c, force, cont, cont_cls, NULL);
-}
-
-
-/**
- * Send a KX message.
- *
- * @param t Tunnel on which to send it.
- * @param force_reply Force the other peer to reply with a KX message.
- */
-void
-GCT_send_kx (struct CadetTunnel *t, int force_reply)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  struct GNUNET_CADET_TunnelKeyExchangeMessage msg;
-  enum GNUNET_CADET_KX_Flags flags;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "==> {        KX} on %s\n", GCT_2s (t));
-  if (NULL != t->ephm_h)
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, "     already queued, nop\n");
-    return;
-  }
-  GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
-
-  c = tunnel_get_connection (t);
-  if (NULL == c)
-  {
-    if (NULL == t->destroy_task && CADET_TUNNEL_READY == t->cstate)
-    {
-      GNUNET_break (0);
-      GCT_debug (t, GNUNET_ERROR_TYPE_ERROR);
-    }
-    return;
-  }
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX);
-  flags = GNUNET_CADET_KX_FLAG_NONE;
-  if (GNUNET_YES == force_reply)
-    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
-  msg.flags = htonl (flags);
-  msg.cid = *GCC_get_id (c);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->kx_0, &msg.ephemeral_key);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &msg.ratchet_key);
-
-  t->ephm_h = GCC_send_prebuilt_message (&msg.header,
-                                         UINT16_MAX,
-                                         zero,
-                                         c,
-                                         GCC_is_origin (c, GNUNET_YES),
-                                         GNUNET_YES, &ephm_sent, t);
-  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
-    GCT_change_estate (t, CADET_TUNNEL_KEY_SENT);
-}
-
-
-/**
- * Is the tunnel directed towards the local peer?
- *
- * @param t Tunnel.
- *
- * @return #GNUNET_YES if it is loopback.
- */
-int
-GCT_is_loopback (const struct CadetTunnel *t)
-{
-  return (myid == GCP_get_short_id (t->peer));
-}
-
-
-/**
- * Is the tunnel this path already?
- *
- * @param t Tunnel.
- * @param p Path.
- *
- * @return #GNUNET_YES a connection uses this path.
- */
-int
-GCT_is_path_used (const struct CadetTunnel *t, const struct CadetPeerPath *p)
-{
-  struct CadetTConnection *iter;
-
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-    if (path_equivalent (GCC_get_path (iter->c), p))
-      return GNUNET_YES;
-
-  return GNUNET_NO;
-}
-
-
-/**
- * Get a cost of a path for a tunnel considering existing connections.
- *
- * @param t Tunnel.
- * @param path Candidate path.
- *
- * @return Cost of the path (path length + number of overlapping nodes)
- */
-unsigned int
-GCT_get_path_cost (const struct CadetTunnel *t,
-                   const struct CadetPeerPath *path)
-{
-  struct CadetTConnection *iter;
-  const struct CadetPeerPath *aux;
-  unsigned int overlap;
-  unsigned int i;
-  unsigned int j;
-
-  if (NULL == path)
-    return 0;
-
-  overlap = 0;
-  GNUNET_assert (NULL != t);
-
-  for (i = 0; i < path->length; i++)
-  {
-    for (iter = t->connection_head; NULL != iter; iter = iter->next)
-    {
-      aux = GCC_get_path (iter->c);
-      if (NULL == aux)
-        continue;
-
-      for (j = 0; j < aux->length; j++)
-      {
-        if (path->peers[i] == aux->peers[j])
-        {
-          overlap++;
-          break;
-        }
-      }
-    }
-  }
-  return path->length + overlap;
-}
-
-
-/**
- * Get the static string for the peer this tunnel is directed.
- *
- * @param t Tunnel.
- *
- * @return Static string the destination peer's ID.
- */
-const char *
-GCT_2s (const struct CadetTunnel *t)
-{
-  if (NULL == t)
-    return "(NULL)";
-
-  return GCP_2s (t->peer);
-}
-
-
-/******************************************************************************/
-/*****************************    INFO/DEBUG    *******************************/
-/******************************************************************************/
-
-static void
-ax_debug (const struct CadetTunnelAxolotl *ax, enum GNUNET_ErrorType level)
-{
-  struct GNUNET_CRYPTO_EcdhePublicKey pub;
-  struct CadetTunnelSkippedKey *iter;
-
-  LOG2 (level, "TTT  RK  \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->RK));
-
-  LOG2 (level, "TTT  HKs \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKs));
-  LOG2 (level, "TTT  HKr \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKr));
-  LOG2 (level, "TTT  NHKs\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->NHKs));
-  LOG2 (level, "TTT  NHKr\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->NHKr));
-
-  LOG2 (level, "TTT  CKs \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKs));
-  LOG2 (level, "TTT  CKr \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKr));
-
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->DHRs, &pub);
-  LOG2 (level, "TTT  DHRs\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &pub));
-  LOG2 (level, "TTT  DHRr\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->DHRr));
-
-  LOG2 (level, "TTT  Nr\t %u\tNs\t%u\n", ax->Nr, ax->Ns);
-  LOG2 (level, "TTT  PNs\t %u\tSkipped\t%u\n", ax->PNs, ax->skipped);
-  LOG2 (level, "TTT  Ratchet\t%u\n", ax->ratchet_flag);
-
-  for (iter = ax->skipped_head; NULL != iter; iter = iter->next)
-  {
-    LOG2 (level, "TTT    HK\t %s\n",
-          GNUNET_i2s ((struct GNUNET_PeerIdentity *) &iter->HK));
-    LOG2 (level, "TTT    MK\t %s\n",
-          GNUNET_i2s ((struct GNUNET_PeerIdentity *) &iter->MK));
-  }
-}
-
-/**
- * Log all possible info about the tunnel state.
- *
- * @param t Tunnel to debug.
- * @param level Debug level to use.
- */
-void
-GCT_debug (const struct CadetTunnel *t, enum GNUNET_ErrorType level)
-{
-  struct CadetTChannel *iter_ch;
-  struct CadetTConnection *iter_c;
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-tun",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  LOG2 (level, "TTT DEBUG TUNNEL TOWARDS %s\n", GCT_2s (t));
-  LOG2 (level, "TTT  cstate %s, estate %s\n",
-       cstate2s (t->cstate), estate2s (t->estate));
-#if DUMP_KEYS_TO_STDERR
-  ax_debug (t->ax, level);
-#endif
-  LOG2 (level, "TTT  tq_head %p, tq_tail %p\n", t->tq_head, t->tq_tail);
-  LOG2 (level, "TTT  destroy %p\n", t->destroy_task);
-  LOG2 (level, "TTT  channels:\n");
-  for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = iter_ch->next)
-  {
-    GCCH_debug (iter_ch->ch, level);
-  }
-
-  LOG2 (level, "TTT  connections:\n");
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = iter_c->next)
-  {
-    GCC_debug (iter_c->c, level);
-  }
-
-  LOG2 (level, "TTT DEBUG TUNNEL END\n");
-}
-
-
-/**
- * Iterate all tunnels.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter, void *cls)
-{
-  GNUNET_CONTAINER_multipeermap_iterate (tunnels, iter, cls);
-}
-
-
-/**
- * Count all tunnels.
- *
- * @return Number of tunnels to remote peers kept by this peer.
- */
-unsigned int
-GCT_count_all (void)
-{
-  return GNUNET_CONTAINER_multipeermap_size (tunnels);
-}
-
-
-/**
- * Iterate all connections of a tunnel.
- *
- * @param t Tunnel whose connections to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_connections (struct CadetTunnel *t, GCT_conn_iter iter, void *cls)
-{
-  struct CadetTConnection *ct;
-
-  for (ct = t->connection_head; NULL != ct; ct = ct->next)
-    iter (cls, ct->c);
-}
-
-
-/**
- * Iterate all channels of a tunnel.
- *
- * @param t Tunnel whose channels to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_channels (struct CadetTunnel *t, GCT_chan_iter iter, void *cls)
-{
-  struct CadetTChannel *cht;
-
-  for (cht = t->channel_head; NULL != cht; cht = cht->next)
-    iter (cls, cht->ch);
-}
diff --git a/src/cadet/gnunet-service-cadet_tunnel.h b/src/cadet/gnunet-service-cadet_tunnel.h
deleted file mode 100644
index 0abdc02ce..000000000
--- a/src/cadet/gnunet-service-cadet_tunnel.h
+++ /dev/null
@@ -1,616 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_tunnel.h
- * @brief cadet service; dealing with tunnels and crypto
- * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMT (Gnunet Cadet Tunnel)
- */
-
-#ifndef GNUNET_SERVICE_CADET_TUNNEL_H
-#define GNUNET_SERVICE_CADET_TUNNEL_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define CONNECTIONS_PER_TUNNEL 3
-
-/**
- * All the connectivity states a tunnel can be in.
- */
-enum CadetTunnelCState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_TUNNEL_NEW,
-
-  /**
-   * No path to the peer known yet.
-   */
-  CADET_TUNNEL_SEARCHING,
-
-  /**
-   * Request sent, not yet answered.
-   */
-  CADET_TUNNEL_WAITING,
-
-  /**
-   * Peer connected and ready to accept data.
-   */
-  CADET_TUNNEL_READY,
-
-  /**
-   * Tunnel being shut down, don't try to keep it alive.
-   */
-  CADET_TUNNEL_SHUTDOWN
-};
-
-
-/**
- * All the encryption states a tunnel can be in.
- */
-enum CadetTunnelEState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_TUNNEL_KEY_UNINITIALIZED,
-
-  /**
-   * Ephemeral key sent, waiting for peer's key.
-   */
-  CADET_TUNNEL_KEY_SENT,
-
-  /**
-   * In OTR: New ephemeral key and ping sent, waiting for pong.
-   *
-   * This means that we DO have the peer's ephemeral key, otherwise the
-   * state would be KEY_SENT. We DO NOT have a valid session key (either no
-   * previous key or previous key expired).
-   *
-   *
-   * In Axolotl: Key sent and received but no deciphered traffic yet.
-   *
-   * This means that we can send traffic (otherwise we would never complete
-   * the handshake), but we don't have complete confirmation. Since the first
-   * traffic MUST be a complete channel creation 3-way handshake, no payload
-   * will be sent before confirmation.
-   */
-  CADET_TUNNEL_KEY_PING,
-
-  /**
-   * Handshake completed: session key available.
-   */
-  CADET_TUNNEL_KEY_OK,
-
-  /**
-   * New ephemeral key and ping sent, waiting for pong. Unlike KEY_PING,
-   * we still have a valid session key and therefore we *can* still send
-   * traffic on the tunnel.
-   */
-  CADET_TUNNEL_KEY_REKEY
-};
-
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetTunnel;
-
-
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_peer.h"
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetTunnelQueue;
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param t Tunnel this message was on.
- * @param type Type of message sent.
- * @param size Size of the message.
- */
-typedef void
-(*GCT_sent) (void *cls,
-             struct CadetTunnel *t,
-             struct CadetTunnelQueue *q,
-             uint16_t type, size_t size);
-
-typedef void
-(*GCT_conn_iter) (void *cls, struct CadetConnection *c);
-
-
-typedef void
-(*GCT_chan_iter) (void *cls, struct CadetChannel *ch);
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize tunnel subsystem.
- *
- * @param c Configuration handle.
- * @param key ECC private key, to derive all other keys and do crypto.
- */
-void
-GCT_init (const struct GNUNET_CONFIGURATION_Handle *c,
-          const struct GNUNET_CRYPTO_EddsaPrivateKey *key);
-
-
-/**
- * Shut down the tunnel subsystem.
- */
-void
-GCT_shutdown (void);
-
-
-/**
- * Create a tunnel.
- *
- * @param destination Peer this tunnel is towards.
- */
-struct CadetTunnel *
-GCT_new (struct CadetPeer *destination);
-
-
-/**
- * Tunnel is empty: destroy it.
- *
- * Notifies all connections about the destruction.
- *
- * @param t Tunnel to destroy.
- */
-void
-GCT_destroy_empty (struct CadetTunnel *t);
-
-
-/**
- * Destroy tunnel if empty (no more channels).
- *
- * @param t Tunnel to destroy if empty.
- */
-void
-GCT_destroy_if_empty (struct CadetTunnel *t);
-
-
-/**
- * Destroy the tunnel.
- *
- * This function does not generate any warning traffic to clients or peers.
- *
- * Tasks:
- * Cancel messages belonging to this tunnel queued to neighbors.
- * Free any allocated resources linked to the tunnel.
- *
- * @param t The tunnel to destroy.
- */
-void
-GCT_destroy (struct CadetTunnel *t);
-
-
-/**
- * Change the tunnel's connection state.
- *
- * @param t Tunnel whose connection state to change.
- * @param cstate New connection state.
- */
-void
-GCT_change_cstate (struct CadetTunnel* t, enum CadetTunnelCState cstate);
-
-
-/**
- * Change the tunnel encryption state.
- *
- * @param t Tunnel whose encryption state to change.
- * @param state New encryption state.
- */
-void
-GCT_change_estate (struct CadetTunnel* t, enum CadetTunnelEState state);
-
-
-/**
- * Add a connection to a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_add_connection (struct CadetTunnel *t, struct CadetConnection *c);
-
-
-/**
- * Remove a connection from a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_remove_connection (struct CadetTunnel *t, struct CadetConnection *c);
-
-
-/**
- * Add a channel to a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_add_channel (struct CadetTunnel *t, struct CadetChannel *ch);
-
-
-/**
- * Remove a channel from a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_remove_channel (struct CadetTunnel *t, struct CadetChannel *ch);
-
-
-/**
- * Search for a channel by global ID.
- *
- * @param t Tunnel containing the channel.
- * @param ctn Public channel number.
- *
- * @return channel handler, NULL if doesn't exist
- */
-struct CadetChannel *
-GCT_get_channel (struct CadetTunnel *t, struct GNUNET_CADET_ChannelTunnelNumber ctn);
-
-
-/**
- * Decrypt and process an encrypted message.
- *
- * Calls the appropriate handler for a message in a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msg Message header.
- */
-void
-GCT_handle_encrypted (struct CadetTunnel *t,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
-
-
-/**
- * Handle a Key eXchange message.
- *
- * @param t Tunnel on which the message came.
- * @param msg KX message itself.
- */
-void
-GCT_handle_kx (struct CadetTunnel *t,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
-
-
-/**
- * @brief Use the given path for the tunnel.
- * Update the next and prev hops (and RCs).
- * (Re)start the path refresh in case the tunnel is locally owned.
- *
- * @param t Tunnel to update.
- * @param p Path to use.
- *
- * @return Connection created.
- */
-struct CadetConnection *
-GCT_use_path (struct CadetTunnel *t, struct CadetPeerPath *p);
-
-
-/**
- * Count all created connections of a tunnel. Not necessarily ready connections!
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections created, either being established or ready.
- */
-unsigned int
-GCT_count_any_connections (struct CadetTunnel *t);
-
-
-/**
- * Count established (ready) connections of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections.
- */
-unsigned int
-GCT_count_connections (struct CadetTunnel *t);
-
-
-/**
- * Count channels of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of channels.
- */
-unsigned int
-GCT_count_channels (struct CadetTunnel *t);
-
-
-/**
- * Get the connectivity state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's connectivity state.
- */
-enum CadetTunnelCState
-GCT_get_cstate (struct CadetTunnel *t);
-
-
-/**
- * Get the encryption state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's encryption state.
- */
-enum CadetTunnelEState
-GCT_get_estate (struct CadetTunnel *t);
-
-
-/**
- * Get the maximum buffer space for a tunnel towards a local client.
- *
- * @param t Tunnel.
- *
- * @return Biggest buffer space offered by any channel in the tunnel.
- */
-unsigned int
-GCT_get_channels_buffer (struct CadetTunnel *t);
-
-
-/**
- * Get the total buffer space for a tunnel for P2P traffic.
- *
- * @param t Tunnel.
- *
- * @return Buffer space offered by all connections in the tunnel.
- */
-unsigned int
-GCT_get_connections_buffer (struct CadetTunnel *t);
-
-
-/**
- * Get the tunnel's destination.
- *
- * @param t Tunnel.
- *
- * @return ID of the destination peer.
- */
-const struct GNUNET_PeerIdentity *
-GCT_get_destination (struct CadetTunnel *t);
-
-
-/**
- * Get the tunnel's next free Channel ID.
- *
- * @param t Tunnel.
- *
- * @return ID of a channel free to use.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCT_get_next_ctn (struct CadetTunnel *t);
-
-
-/**
- * Send ACK on one or more channels due to buffer in connections.
- *
- * @param t Channel which has some free buffer space.
- */
-void
-GCT_unchoke_channels (struct CadetTunnel *t);
-
-
-/**
- * Send ACK on one or more connections due to buffer space to the client.
- *
- * Iterates all connections of the tunnel and sends ACKs appropriately.
- *
- * @param t Tunnel which has some free buffer space.
- */
-void
-GCT_send_connection_acks (struct CadetTunnel *t);
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCT_cancel (struct CadetTunnelQueue *q);
-
-
-/**
- * Check if the tunnel has queued traffic.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if there is queued traffic
- *         #GNUNET_NO otherwise
- */
-int
-GCT_has_queued_traffic (struct CadetTunnel *t);
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel message. NULL if @c cont is NULL.
- */
-struct CadetTunnelQueue *
-GCT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           struct CadetTunnel *t, struct CadetConnection *c,
-                           int force, GCT_sent cont, void *cont_cls);
-
-
-/**
- * Send a KX message.
- *
- * @param t Tunnel on which to send it.
- * @param force_reply Force the other peer to reply with a KX message.
- */
-void
-GCT_send_kx (struct CadetTunnel *t, int force_reply);
-
-
-/**
- * Is the tunnel directed towards the local peer?
- *
- * @param t Tunnel.
- *
- * @return #GNUNET_YES if it is loopback.
- */
-int
-GCT_is_loopback (const struct CadetTunnel *t);
-
-
-/**
- * Is the tunnel using this path already?
- *
- * @param t Tunnel.
- * @param p Path.
- *
- * @return #GNUNET_YES a connection uses this path.
- */
-int
-GCT_is_path_used (const struct CadetTunnel *t, const struct CadetPeerPath *p);
-
-
-/**
- * Get a cost of a path for a tunnel considering existing connections.
- *
- * @param t Tunnel.
- * @param path Candidate path.
- *
- * @return Cost of the path (path length + number of overlapping nodes)
- */
-unsigned int
-GCT_get_path_cost (const struct CadetTunnel *t,
-                   const struct CadetPeerPath *path);
-
-
-/**
- * Get the static string for the peer this tunnel is directed.
- *
- * @param t Tunnel.
- *
- * @return Static string the destination peer's ID.
- */
-const char *
-GCT_2s (const struct CadetTunnel *t);
-
-
-/**
- * Log all possible info about the tunnel state.
- *
- * @param t Tunnel to debug.
- * @param level Debug level to use.
- */
-void
-GCT_debug (const struct CadetTunnel *t, enum GNUNET_ErrorType level);
-
-
-/**
- * Iterate all tunnels.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter, void *cls);
-
-
-/**
- * Count all tunnels.
- *
- * @return Number of tunnels to remote peers kept by this peer.
- */
-unsigned int
-GCT_count_all (void);
-
-
-/**
- * Iterate all connections of a tunnel.
- *
- * @param t Tunnel whose connections to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_connections (struct CadetTunnel *t, GCT_conn_iter iter, void *cls);
-
-
-/**
- * Iterate all channels of a tunnel.
- *
- * @param t Tunnel whose channels to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_channels (struct CadetTunnel *t,
-                      GCT_chan_iter iter,
-                      void *cls);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_TUNNEL_H */
-#endif
-/* end of gnunet-cadet-service_tunnel.h */
diff --git a/src/cadet/gnunet-service-cadet-new_tunnels.c b/src/cadet/gnunet-service-cadet_tunnels.c
index fd8335486..28004debc 100644
--- a/src/cadet/gnunet-service-cadet-new_tunnels.c
+++ b/src/cadet/gnunet-service-cadet_tunnels.c
@@ -18,35 +18,38 @@
      Boston, MA 02110-1301, USA.
 */
 /**
- * @file cadet/gnunet-service-cadet-new_tunnels.c
+ * @file cadet/gnunet-service-cadet_tunnels.c
  * @brief Information we track per tunnel.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
  *
  * FIXME:
- * - check KX estate machine -- make sure it is never stuck!
- * - clean up KX logic, including adding sender authentication
- * - implement connection management (evaluate, kill old ones,
- *   search for new ones)
- * - when managing connections, distinguish those that
- *   have (recently) had traffic from those that were
- *   never ready (or not recently)
+ * - proper connection evaluation during connection management:
+ *   + consider quality (or quality spread?) of current connection set
+ *     when deciding how often to do maintenance
+ *   + interact with PEER to drive DHT GET/PUT operations based
+ *     on how much we like our connections
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet_signatures.h"
-#include "gnunet-service-cadet-new.h"
 #include "cadet_protocol.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-tun",__VA_ARGS__)
 
+/**
+ * How often do we try to decrypt payload with unverified key
+ * material?  Used to limit CPU increase upon receiving bogus
+ * KX.
+ */
+#define MAX_UNVERIFIED_ATTEMPTS 16
 
 /**
  * How long do we wait until tearing down an idle tunnel?
@@ -54,12 +57,10 @@
 #define IDLE_DESTROY_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
 
 /**
- * Yuck, replace by 'offsetof' expression?
- * FIXME.
+ * How long do we wait initially before retransmitting the KX?
+ * TODO: replace by 2 RTT if/once we have connection-level RTT data!
  */
-#define AX_HEADER_SIZE (sizeof (uint32_t) * 2\
-                        + sizeof (struct GNUNET_CRYPTO_EcdhePublicKey))
-
+#define INITIAL_KX_RETRY_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
 
 /**
  * Maximum number of skipped keys we keep in memory per tunnel.
@@ -133,52 +134,61 @@ struct CadetTunnelAxolotl
   struct GNUNET_CRYPTO_SymmetricSessionKey RK;
 
   /**
-   * 32-byte header key (send).
+   * 32-byte header key (currently used for sending).
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey HKs;
 
   /**
-   * 32-byte header key (recv)
+   * 32-byte header key (currently used for receiving)
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey HKr;
 
   /**
-   * 32-byte next header key (send).
+   * 32-byte next header key (for sending), used once the
+   * ratchet advances.  We are sure that the sender has this
+   * key as well only after @e ratchet_allowed is #GNUNET_YES.
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey NHKs;
 
   /**
-   * 32-byte next header key (recv).
+   * 32-byte next header key (for receiving).  To be tried
+   * when decrypting with @e HKr fails and thus the sender
+   * may have advanced the ratchet.
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey NHKr;
 
   /**
-   * 32-byte chain keys (used for forward-secrecy updating, send).
+   * 32-byte chain keys (used for forward-secrecy) for
+   * sending messages. Updated for every message.
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey CKs;
 
   /**
-   * 32-byte chain keys (used for forward-secrecy updating, recv).
+   * 32-byte chain keys (used for forward-secrecy) for
+   * receiving messages. Updated for every message. If
+   * messages are skipped, the respective derived MKs
+   * (and the current @HKr) are kept in the @e skipped_head DLL.
    */
   struct GNUNET_CRYPTO_SymmetricSessionKey CKr;
 
   /**
    * ECDH for key exchange (A0 / B0).
    */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *kx_0;
+  struct GNUNET_CRYPTO_EcdhePrivateKey kx_0;
 
   /**
-   * ECDH Ratchet key (send).
+   * ECDH Ratchet key (our private key in the current DH).
    */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *DHRs;
+  struct GNUNET_CRYPTO_EcdhePrivateKey DHRs;
 
   /**
-   * ECDH Ratchet key (recv).
+   * ECDH Ratchet key (other peer's public key in the current DH).
    */
   struct GNUNET_CRYPTO_EcdhePublicKey DHRr;
 
   /**
-   * When does this ratchet expire and a new one is triggered.
+   * Time when the current ratchet expires and a new one is triggered
+   * (if @e ratchet_allowed is #GNUNET_YES).
    */
   struct GNUNET_TIME_Absolute ratchet_expiration;
 
@@ -208,16 +218,28 @@ struct CadetTunnelAxolotl
   int ratchet_flag;
 
   /**
-   * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
+   * True (#GNUNET_YES) if we have received a message from the
+   * other peer that uses the keys from our last ratchet step.
+   * This implies that we are again allowed to advance the ratchet,
+   * otherwise we have to wait until the other peer sees our current
+   * ephemeral key and advances first.
+   *
+   * #GNUNET_NO if we have advanced the ratched but lack any evidence
+   * that the other peer has noticed this.
    */
-  unsigned int ratchet_allowed;
+  int ratchet_allowed;
 
   /**
    * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
+   *
+   * If this counter = 0, we cannot send a new ratchet key in the next
+   * message.
+   *
+   * If this counter > 0, we could (but don't have to) send a new key.
+   *
+   * Once the @e ratchet_counter is larger than
+   * #ratchet_messages (or @e ratchet_expiration time has past), and
+   * @e ratchet_allowed is #GNUNET_YES, we advance the ratchet.
    */
   unsigned int ratchet_counter;
 
@@ -247,7 +269,7 @@ struct CadetTunnelQueueEntry
   /**
    * Continuation to call once sent (on the channel layer).
    */
-  GNUNET_SCHEDULER_TaskCallback cont;
+  GCT_SendContinuation cont;
 
   /**
    * Closure for @c cont.
@@ -299,6 +321,15 @@ struct CadetTunnel
   struct CadetTunnelAxolotl ax;
 
   /**
+   * Unverified Axolotl info, used only if we got a fresh KX (not a
+   * KX_AUTH) while our end of the tunnel was still up.  In this case,
+   * we keep the fresh KX around but do not put it into action until
+   * we got encrypted payload that assures us of the authenticity of
+   * the KX.
+   */
+  struct CadetTunnelAxolotl *unverified_ax;
+
+  /**
    * Task scheduled if there are no more channels using the tunnel.
    */
   struct GNUNET_SCHEDULER_Task *destroy_task;
@@ -329,14 +360,24 @@ struct CadetTunnel
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * DLL of connections that are actively used to reach the destination peer.
+   * DLL of ready connections that are actively used to reach the destination peer.
    */
-  struct CadetTConnection *connection_head;
+  struct CadetTConnection *connection_ready_head;
 
   /**
-   * DLL of connections that are actively used to reach the destination peer.
+   * DLL of ready connections that are actively used to reach the destination peer.
    */
-  struct CadetTConnection *connection_tail;
+  struct CadetTConnection *connection_ready_tail;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_head;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_tail;
 
   /**
    * Channels inside this tunnel. Maps
@@ -359,16 +400,12 @@ struct CadetTunnel
    */
   struct CadetTunnelQueueEntry *tq_tail;
 
-
-  /**
-   * Ephemeral message in the queue (to avoid queueing more than one).
-   */
-  struct CadetConnectionQueue *ephm_hKILL;
-
   /**
-   * Pong message in the queue.
+   * Identification of the connection from which we are currently processing
+   * a message. Only valid (non-NULL) during #handle_decrypted() and the
+   * handle-*()-functions called from our @e mq during that function.
    */
-  struct CadetConnectionQueue *pong_hKILL;
+  struct CadetTConnection *current_ct;
 
   /**
    * How long do we wait until we retry the KX?
@@ -381,9 +418,21 @@ struct CadetTunnel
   struct GNUNET_TIME_Absolute next_kx_attempt;
 
   /**
-   * Number of connections in the @e connection_head DLL.
+   * Number of connections in the @e connection_ready_head DLL.
    */
-  unsigned int num_connections;
+  unsigned int num_ready_connections;
+
+  /**
+   * Number of connections in the @e connection_busy_head DLL.
+   */
+  unsigned int num_busy_connections;
+
+  /**
+   * How often have we tried and failed to decrypt a message using
+   * the unverified KX material from @e unverified_ax?  Used to
+   * stop trying after #MAX_UNVERIFIED_ATTEMPTS.
+   */
+  unsigned int unverified_attempts;
 
   /**
    * Number of entries in the @e tq_head DLL.
@@ -395,10 +444,40 @@ struct CadetTunnel
    */
   enum CadetTunnelEState estate;
 
+  /**
+   * Force triggering KX_AUTH independent of @e estate.
+   */
+  int kx_auth_requested;
+
 };
 
 
 /**
+ * Connection @a ct is now unready, clear it's ready flag
+ * and move it from the ready DLL to the busy DLL.
+ *
+ * @param ct connection to move to unready status
+ */
+static void
+mark_connection_unready (struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = ct->t;
+
+  GNUNET_assert (GNUNET_YES == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                               t->connection_ready_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_ready_connections);
+  t->num_ready_connections--;
+  ct->is_ready = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  t->num_busy_connections++;
+}
+
+
+/**
  * Get the static string for the peer this tunnel is directed.
  *
  * @param t Tunnel.
@@ -434,19 +513,24 @@ estate2s (enum CadetTunnelEState es)
 
   switch (es)
   {
-    case CADET_TUNNEL_KEY_UNINITIALIZED:
-      return "CADET_TUNNEL_KEY_UNINITIALIZED";
-    case CADET_TUNNEL_KEY_SENT:
-      return "CADET_TUNNEL_KEY_SENT";
-    case CADET_TUNNEL_KEY_PING:
-      return "CADET_TUNNEL_KEY_PING";
-    case CADET_TUNNEL_KEY_OK:
-      return "CADET_TUNNEL_KEY_OK";
-    case CADET_TUNNEL_KEY_REKEY:
-      return "CADET_TUNNEL_KEY_REKEY";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", es);
-      return buf;
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    return "CADET_TUNNEL_KEY_UNINITIALIZED";
+  case CADET_TUNNEL_KEY_AX_RECV:
+    return "CADET_TUNNEL_KEY_AX_RECV";
+  case CADET_TUNNEL_KEY_AX_SENT:
+    return "CADET_TUNNEL_KEY_AX_SENT";
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    return "CADET_TUNNEL_KEY_AX_SENT_AND_RECV";
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    return "CADET_TUNNEL_KEY_AX_AUTH_SENT";
+  case CADET_TUNNEL_KEY_OK:
+    return "CADET_TUNNEL_KEY_OK";
+  default:
+    GNUNET_snprintf (buf,
+                     sizeof (buf),
+                     "%u (UNKNOWN STATE)",
+                     es);
+    return buf;
   }
 }
 
@@ -502,9 +586,9 @@ lookup_channel (struct CadetTunnel *t,
  * @return Number of connections created, either being established or ready.
  */
 unsigned int
-GCT_count_any_connections (struct CadetTunnel *t)
+GCT_count_any_connections (const struct CadetTunnel *t)
 {
-  return t->num_connections;
+  return t->num_ready_connections + t->num_busy_connections;
 }
 
 
@@ -518,25 +602,11 @@ GCT_count_any_connections (struct CadetTunnel *t)
 static struct CadetTConnection *
 get_ready_connection (struct CadetTunnel *t)
 {
-  for (struct CadetTConnection *pos = t->connection_head;
-       NULL != pos;
-       pos = pos->next)
-    if (GNUNET_YES == pos->is_ready)
-    {
-      if (pos != t->connection_tail)
-      {
-        /* move 'pos' to the end, so we try other ready connections
-           first next time (round-robin, modulo availability) */
-        GNUNET_CONTAINER_DLL_remove (t->connection_head,
-                                     t->connection_tail,
-                                     pos);
-        GNUNET_CONTAINER_DLL_insert_tail (t->connection_head,
-                                          t->connection_tail,
-                                          pos);
-      }
-      return pos;
-    }
-  return NULL;
+  struct CadetTConnection *hd = t->connection_ready_head;
+
+  GNUNET_assert ( (NULL == hd) ||
+                  (GNUNET_YES == hd->is_ready) );
+  return hd;
 }
 
 
@@ -555,20 +625,6 @@ GCT_get_estate (struct CadetTunnel *t)
 
 
 /**
- * Create a new Axolotl ephemeral (ratchet) key.
- *
- * @param t Tunnel.
- */
-static void
-new_ephemeral (struct CadetTunnel *t)
-{
-  GNUNET_free_non_null (t->ax.DHRs);
-  t->ax.DHRs = GNUNET_CRYPTO_ecdhe_key_create ();
-}
-
-
-
-/**
  * Called when either we have a new connection, or a new message in the
  * queue, or some existing connection has transmission capacity.  Looks
  * at our message queue and if there is a message, picks a connection
@@ -584,6 +640,21 @@ trigger_transmissions (void *cls);
 
 
 /**
+ * Create a new Axolotl ephemeral (ratchet) key.
+ *
+ * @param ax key material to update
+ */
+static void
+new_ephemeral (struct CadetTunnelAxolotl *ax)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating new ephemeral ratchet key (DHRs)\n");
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdhe_key_create2 (&ax->DHRs));
+}
+
+
+/**
  * Calculate HMAC.
  *
  * @param plaintext Content to HMAC.
@@ -609,7 +680,8 @@ t_hmac (const void *plaintext,
                                  key, sizeof (*key),
                                  ctx, sizeof (ctx),
                                  NULL);
-  /* Two step: CADET_Hash is only 256 bits, HashCode is 512. */
+  /* Two step: GNUNET_ShortHash is only 256 bits,
+     GNUNET_HashCode is 512, so we truncate. */
   GNUNET_CRYPTO_hmac (&auth_key,
                       plaintext,
                       size,
@@ -624,7 +696,7 @@ t_hmac (const void *plaintext,
  * Perform a HMAC.
  *
  * @param key Key to use.
- * @param hash[out] Resulting HMAC.
+ * @param[out] hash Resulting HMAC.
  * @param source Source key material (data to HMAC).
  * @param len Length of @a source.
  */
@@ -679,23 +751,21 @@ t_hmac_derive_key (const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
 /**
  * Encrypt data with the axolotl tunnel key.
  *
- * @param t Tunnel whose key to use.
+ * @param ax key material to use.
  * @param dst Destination with @a size bytes for the encrypted data.
  * @param src Source of the plaintext. Can overlap with @c dst, must contain @a size bytes
  * @param size Size of the buffers at @a src and @a dst
  */
 static void
-t_ax_encrypt (struct CadetTunnel *t,
+t_ax_encrypt (struct CadetTunnelAxolotl *ax,
               void *dst,
               const void *src,
               size_t size)
 {
   struct GNUNET_CRYPTO_SymmetricSessionKey MK;
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
   size_t out_size;
 
-  ax = &t->ax;
   ax->ratchet_counter++;
   if ( (GNUNET_YES == ax->ratchet_allowed) &&
        ( (ratchet_messages <= ax->ratchet_counter) ||
@@ -711,11 +781,11 @@ t_ax_encrypt (struct CadetTunnel *t,
     struct GNUNET_HashCode hmac;
     static const char ctx[] = "axolotl ratchet";
 
-    new_ephemeral (t);
+    new_ephemeral (ax);
     ax->HKs = ax->NHKs;
 
     /* RK, NHKs, CKs = KDF( HMAC-HASH(RK, DH(DHRs, DHRr)) ) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs,
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
                             &ax->DHRr,
                             &dh);
     t_ax_hmac_hash (&ax->RK,
@@ -765,23 +835,21 @@ t_ax_encrypt (struct CadetTunnel *t,
 /**
  * Decrypt data with the axolotl tunnel key.
  *
- * @param t Tunnel whose key to use.
+ * @param ax key material to use.
  * @param dst Destination for the decrypted data, must contain @a size bytes.
  * @param src Source of the ciphertext. Can overlap with @c dst, must contain @a size bytes.
  * @param size Size of the @a src and @a dst buffers
  */
 static void
-t_ax_decrypt (struct CadetTunnel *t,
+t_ax_decrypt (struct CadetTunnelAxolotl *ax,
               void *dst,
               const void *src,
               size_t size)
 {
   struct GNUNET_CRYPTO_SymmetricSessionKey MK;
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
   size_t out_size;
 
-  ax = &t->ax;
   t_hmac_derive_key (&ax->CKr,
                      &MK,
                      "0",
@@ -807,76 +875,72 @@ t_ax_decrypt (struct CadetTunnel *t,
 /**
  * Encrypt header with the axolotl header key.
  *
- * @param t Tunnel whose key to use.
- * @param msg Message whose header to encrypt.
+ * @param ax key material to use.
+ * @param[in|out] msg Message whose header to encrypt.
  */
 static void
-t_h_encrypt (struct CadetTunnel *t,
+t_h_encrypt (struct CadetTunnelAxolotl *ax,
              struct GNUNET_CADET_TunnelEncryptedMessage *msg)
 {
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
   size_t out_size;
 
-  ax = &t->ax;
   GNUNET_CRYPTO_symmetric_derive_iv (&iv,
                                      &ax->HKs,
                                      NULL, 0,
                                      NULL);
-  out_size = GNUNET_CRYPTO_symmetric_encrypt (&msg->Ns,
-                                              AX_HEADER_SIZE,
+  out_size = GNUNET_CRYPTO_symmetric_encrypt (&msg->ax_header,
+                                              sizeof (struct GNUNET_CADET_AxHeader),
                                               &ax->HKs,
                                               &iv,
-                                              &msg->Ns);
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
+                                              &msg->ax_header);
+  GNUNET_assert (sizeof (struct GNUNET_CADET_AxHeader) == out_size);
 }
 
 
 /**
  * Decrypt header with the current axolotl header key.
  *
- * @param t Tunnel whose current ax HK to use.
+ * @param ax key material to use.
  * @param src Message whose header to decrypt.
  * @param dst Where to decrypt header to.
  */
 static void
-t_h_decrypt (struct CadetTunnel *t,
+t_h_decrypt (struct CadetTunnelAxolotl *ax,
              const struct GNUNET_CADET_TunnelEncryptedMessage *src,
              struct GNUNET_CADET_TunnelEncryptedMessage *dst)
 {
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
   size_t out_size;
 
-  ax = &t->ax;
   GNUNET_CRYPTO_symmetric_derive_iv (&iv,
                                      &ax->HKr,
                                      NULL, 0,
                                      NULL);
-  out_size = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns,
-                                              AX_HEADER_SIZE,
+  out_size = GNUNET_CRYPTO_symmetric_decrypt (&src->ax_header.Ns,
+                                              sizeof (struct GNUNET_CADET_AxHeader),
                                               &ax->HKr,
                                               &iv,
-                                              &dst->Ns);
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
+                                              &dst->ax_header.Ns);
+  GNUNET_assert (sizeof (struct GNUNET_CADET_AxHeader) == out_size);
 }
 
 
 /**
  * Delete a key from the list of skipped keys.
  *
- * @param t Tunnel to delete from.
+ * @param ax key material to delete @a key from.
  * @param key Key to delete.
  */
 static void
-delete_skipped_key (struct CadetTunnel *t,
+delete_skipped_key (struct CadetTunnelAxolotl *ax,
                     struct CadetTunnelSkippedKey *key)
 {
-  GNUNET_CONTAINER_DLL_remove (t->ax.skipped_head,
-                               t->ax.skipped_tail,
+  GNUNET_CONTAINER_DLL_remove (ax->skipped_head,
+                               ax->skipped_tail,
                                key);
   GNUNET_free (key);
-  t->ax.skipped--;
+  ax->skipped--;
 }
 
 
@@ -884,14 +948,14 @@ delete_skipped_key (struct CadetTunnel *t,
  * Decrypt and verify data with the appropriate tunnel key and verify that the
  * data has not been altered since it was sent by the remote peer.
  *
- * @param t Tunnel whose key to use.
+ * @param ax key material to use.
  * @param dst Destination for the plaintext.
  * @param src Source of the message. Can overlap with @c dst.
  * @param size Size of the message.
  * @return Size of the decrypted data, -1 if an error was encountered.
  */
 static ssize_t
-try_old_ax_keys (struct CadetTunnel *t,
+try_old_ax_keys (struct CadetTunnelAxolotl *ax,
                  void *dst,
                  const struct GNUNET_CADET_TunnelEncryptedMessage *src,
                  size_t size)
@@ -913,10 +977,10 @@ try_old_ax_keys (struct CadetTunnel *t,
 
   /* Find a correct Header Key */
   valid_HK = NULL;
-  for (key = t->ax.skipped_head; NULL != key; key = key->next)
+  for (key = ax->skipped_head; NULL != key; key = key->next)
   {
-    t_hmac (&src->Ns,
-            AX_HEADER_SIZE + esize,
+    t_hmac (&src->ax_header,
+            sizeof (struct GNUNET_CADET_AxHeader) + esize,
             0,
             &key->HK,
             hmac);
@@ -941,15 +1005,15 @@ try_old_ax_keys (struct CadetTunnel *t,
                                      &key->HK,
                                      NULL, 0,
                                      NULL);
-  res = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns,
-                                         AX_HEADER_SIZE,
+  res = GNUNET_CRYPTO_symmetric_decrypt (&src->ax_header.Ns,
+                                         sizeof (struct GNUNET_CADET_AxHeader),
                                          &key->HK,
                                          &iv,
-                                         &plaintext_header.Ns);
-  GNUNET_assert (AX_HEADER_SIZE == res);
+                                         &plaintext_header.ax_header.Ns);
+  GNUNET_assert (sizeof (struct GNUNET_CADET_AxHeader) == res);
 
   /* Find the correct message key */
-  N = ntohl (plaintext_header.Ns);
+  N = ntohl (plaintext_header.ax_header.Ns);
   while ( (NULL != key) &&
           (N != key->Kn) )
     key = key->next;
@@ -970,7 +1034,7 @@ try_old_ax_keys (struct CadetTunnel *t,
                                          &key->MK,
                                          &iv,
                                          dst);
-  delete_skipped_key (t,
+  delete_skipped_key (ax,
                       key);
   return res;
 }
@@ -979,32 +1043,32 @@ try_old_ax_keys (struct CadetTunnel *t,
 /**
  * Delete a key from the list of skipped keys.
  *
- * @param t Tunnel to delete from.
+ * @param ax key material to delete from.
  * @param HKr Header Key to use.
  */
 static void
-store_skipped_key (struct CadetTunnel *t,
+store_skipped_key (struct CadetTunnelAxolotl *ax,
                    const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr)
 {
   struct CadetTunnelSkippedKey *key;
 
   key = GNUNET_new (struct CadetTunnelSkippedKey);
   key->timestamp = GNUNET_TIME_absolute_get ();
-  key->Kn = t->ax.Nr;
-  key->HK = t->ax.HKr;
-  t_hmac_derive_key (&t->ax.CKr,
+  key->Kn = ax->Nr;
+  key->HK = ax->HKr;
+  t_hmac_derive_key (&ax->CKr,
                      &key->MK,
                      "0",
                      1);
-  t_hmac_derive_key (&t->ax.CKr,
-                     &t->ax.CKr,
+  t_hmac_derive_key (&ax->CKr,
+                     &ax->CKr,
                      "1",
                      1);
-  GNUNET_CONTAINER_DLL_insert (t->ax.skipped_head,
-                               t->ax.skipped_tail,
+  GNUNET_CONTAINER_DLL_insert (ax->skipped_head,
+                               ax->skipped_tail,
                                key);
-  t->ax.skipped++;
-  t->ax.Nr++;
+  ax->skipped++;
+  ax->Nr++;
 }
 
 
@@ -1012,33 +1076,33 @@ store_skipped_key (struct CadetTunnel *t,
  * Stage skipped AX keys and calculate the message key.
  * Stores each HK and MK for skipped messages.
  *
- * @param t Tunnel where to stage the keys.
+ * @param ax key material to use
  * @param HKr Header key.
  * @param Np Received meesage number.
  * @return #GNUNET_OK if keys were stored.
- *         #GNUNET_SYSERR if an error ocurred (Np not expected).
+ *         #GNUNET_SYSERR if an error ocurred (@a Np not expected).
  */
 static int
-store_ax_keys (struct CadetTunnel *t,
+store_ax_keys (struct CadetTunnelAxolotl *ax,
                const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr,
                uint32_t Np)
 {
   int gap;
 
-  gap = Np - t->ax.Nr;
+  gap = Np - ax->Nr;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Storing skipped keys [%u, %u)\n",
-       t->ax.Nr,
+       ax->Nr,
        Np);
   if (MAX_KEY_GAP < gap)
   {
-    /* Avoid DoS (forcing peer to do 2^33 chain HMAC operations) */
+    /* Avoid DoS (forcing peer to do more than #MAX_KEY_GAP HMAC operations) */
     /* TODO: start new key exchange on return */
     GNUNET_break_op (0);
     LOG (GNUNET_ERROR_TYPE_WARNING,
          "Got message %u, expected %u+\n",
          Np,
-         t->ax.Nr);
+         ax->Nr);
     return GNUNET_SYSERR;
   }
   if (0 > gap)
@@ -1047,13 +1111,13 @@ store_ax_keys (struct CadetTunnel *t,
     return GNUNET_SYSERR;
   }
 
-  while (t->ax.Nr < Np)
-    store_skipped_key (t,
+  while (ax->Nr < Np)
+    store_skipped_key (ax,
                        HKr);
 
-  while (t->ax.skipped > MAX_SKIPPED_KEYS)
-    delete_skipped_key (t,
-                        t->ax.skipped_tail);
+  while (ax->skipped > MAX_SKIPPED_KEYS)
+    delete_skipped_key (ax,
+                        ax->skipped_tail);
   return GNUNET_OK;
 }
 
@@ -1062,19 +1126,18 @@ store_ax_keys (struct CadetTunnel *t,
  * Decrypt and verify data with the appropriate tunnel key and verify that the
  * data has not been altered since it was sent by the remote peer.
  *
- * @param t Tunnel whose key to use.
+ * @param ax key material to use
  * @param dst Destination for the plaintext.
  * @param src Source of the message. Can overlap with @c dst.
  * @param size Size of the message.
  * @return Size of the decrypted data, -1 if an error was encountered.
  */
 static ssize_t
-t_ax_decrypt_and_validate (struct CadetTunnel *t,
+t_ax_decrypt_and_validate (struct CadetTunnelAxolotl *ax,
                            void *dst,
                            const struct GNUNET_CADET_TunnelEncryptedMessage *src,
                            size_t size)
 {
-  struct CadetTunnelAxolotl *ax;
   struct GNUNET_ShortHashCode msg_hmac;
   struct GNUNET_HashCode hmac;
   struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
@@ -1083,11 +1146,10 @@ t_ax_decrypt_and_validate (struct CadetTunnel *t,
   size_t esize; /* Size of encryped payload */
 
   esize = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  ax = &t->ax;
 
   /* Try current HK */
-  t_hmac (&src->Ns,
-          AX_HEADER_SIZE + esize,
+  t_hmac (&src->ax_header,
+          sizeof (struct GNUNET_CADET_AxHeader) + esize,
           0, &ax->HKr,
           &msg_hmac);
   if (0 != memcmp (&msg_hmac,
@@ -1101,8 +1163,8 @@ t_ax_decrypt_and_validate (struct CadetTunnel *t,
     struct GNUNET_CRYPTO_EcdhePublicKey *DHRp;
 
     /* Try Next HK */
-    t_hmac (&src->Ns,
-            AX_HEADER_SIZE + esize,
+    t_hmac (&src->ax_header,
+            sizeof (struct GNUNET_CADET_AxHeader) + esize,
             0,
             &ax->NHKr,
             &msg_hmac);
@@ -1111,25 +1173,25 @@ t_ax_decrypt_and_validate (struct CadetTunnel *t,
                      sizeof (msg_hmac)))
     {
       /* Try the skipped keys, if that fails, we're out of luck. */
-      return try_old_ax_keys (t,
+      return try_old_ax_keys (ax,
                               dst,
                               src,
                               size);
     }
     HK = ax->HKr;
     ax->HKr = ax->NHKr;
-    t_h_decrypt (t,
+    t_h_decrypt (ax,
                  src,
                  &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
-    DHRp = &plaintext_header.DHRs;
-    store_ax_keys (t,
+    Np = ntohl (plaintext_header.ax_header.Ns);
+    PNp = ntohl (plaintext_header.ax_header.PNs);
+    DHRp = &plaintext_header.ax_header.DHRs;
+    store_ax_keys (ax,
                    &HK,
                    PNp);
 
     /* RKp, NHKp, CKp = KDF (HMAC-HASH (RK, DH (DHRp, DHRs))) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs,
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
                             DHRp,
                             &dh);
     t_ax_hmac_hash (&ax->RK,
@@ -1150,25 +1212,25 @@ t_ax_decrypt_and_validate (struct CadetTunnel *t,
   }
   else
   {
-    t_h_decrypt (t,
+    t_h_decrypt (ax,
                  src,
                  &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
+    Np = ntohl (plaintext_header.ax_header.Ns);
+    PNp = ntohl (plaintext_header.ax_header.PNs);
   }
   if ( (Np != ax->Nr) &&
-       (GNUNET_OK != store_ax_keys (t,
+       (GNUNET_OK != store_ax_keys (ax,
                                     &ax->HKr,
                                     Np)) )
   {
     /* Try the skipped keys, if that fails, we're out of luck. */
-    return try_old_ax_keys (t,
+    return try_old_ax_keys (ax,
                             dst,
                             src,
                             size);
   }
 
-  t_ax_decrypt (t,
+  t_ax_decrypt (ax,
                 dst,
                 &src[1],
                 esize);
@@ -1213,10 +1275,10 @@ GCT_change_estate (struct CadetTunnel *t,
 
   t->estate = state;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s estate changed from %d to %d\n",
+       "%s estate changed from %s to %s\n",
        GCT_2s (t),
-       old,
-       state);
+       estate2s (old),
+       estate2s (state));
 
   if ( (CADET_TUNNEL_KEY_OK != old) &&
        (CADET_TUNNEL_KEY_OK == t->estate) )
@@ -1226,15 +1288,14 @@ GCT_change_estate (struct CadetTunnel *t,
       GNUNET_SCHEDULER_cancel (t->kx_task);
       t->kx_task = NULL;
     }
-    if (CADET_TUNNEL_KEY_REKEY != old)
-    {
-      /* notify all channels that have been waiting */
-      GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
-                                               &notify_tunnel_up_cb,
-                                               t);
-    }
-
-    /* FIXME: schedule rekey task! */
+    /* notify all channels that have been waiting */
+    GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
+                                             &notify_tunnel_up_cb,
+                                             t);
+    if (NULL != t->send_task)
+      GNUNET_SCHEDULER_cancel (t->send_task);
+    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                             t);
   }
 }
 
@@ -1242,80 +1303,179 @@ GCT_change_estate (struct CadetTunnel *t,
 /**
  * Send a KX message.
  *
- * FIXME: does not take care of sender-authentication yet!
- *
- * @param t Tunnel on which to send it.
- * @param force_reply Force the other peer to reply with a KX message.
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
+ * @param ax axolotl key context to use
  */
 static void
 send_kx (struct CadetTunnel *t,
-         int force_reply)
+         struct CadetTConnection *ct,
+         struct CadetTunnelAxolotl *ax)
 {
-  struct CadetTunnelAxolotl *ax = &t->ax;
-  struct CadetTConnection *ct;
   struct CadetConnection *cc;
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_TunnelKeyExchangeMessage *msg;
   enum GNUNET_CADET_KX_Flags flags;
 
-  ct = get_ready_connection (t);
+  if ( (NULL == ct) ||
+       (GNUNET_NO == ct->is_ready) )
+    ct = get_ready_connection (t);
   if (NULL == ct)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Wanted to send KX on tunnel %s, but no connection is ready, deferring\n",
-         GCT_2s (t));
+         "Wanted to send %s in state %s, but no connection is ready, deferring\n",
+         GCT_2s (t),
+         estate2s (t->estate));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
     return;
   }
   cc = ct->cc;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending KX on tunnel %s using connection %s\n",
+       "Sending KX on %s via %s in state %s\n",
        GCT_2s (t),
-       GCC_2s (ct->cc));
-
-  // GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
+       GCC_2s (cc),
+       estate2s (t->estate));
   env = GNUNET_MQ_msg (msg,
                        GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX);
-  flags = GNUNET_CADET_KX_FLAG_NONE;
-  if (GNUNET_YES == force_reply)
-    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
+  flags = GNUNET_CADET_KX_FLAG_FORCE_REPLY; /* always for KX */
   msg->flags = htonl (flags);
   msg->cid = *GCC_get_id (cc);
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->kx_0,
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
                                       &msg->ephemeral_key);
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->DHRs,
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
                                       &msg->ratchet_key);
-  ct->is_ready = GNUNET_NO;
-  GCC_transmit (cc,
-                env);
+  mark_connection_unready (ct);
   t->kx_retry_delay = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
   t->next_kx_attempt = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
   if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
     GCT_change_estate (t,
-                       CADET_TUNNEL_KEY_SENT);
+                       CADET_TUNNEL_KEY_AX_SENT);
+  else if (CADET_TUNNEL_KEY_AX_RECV == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+  GCC_transmit (cc,
+                env);
+}
+
+
+/**
+ * Send a KX_AUTH message.
+ *
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
+ * @param ax axolotl key context to use
+ * @param force_reply Force the other peer to reply with a KX_AUTH message
+ *         (set if we would like to transmit right now, but cannot)
+ */
+static void
+send_kx_auth (struct CadetTunnel *t,
+              struct CadetTConnection *ct,
+              struct CadetTunnelAxolotl *ax,
+              int force_reply)
+{
+  struct CadetConnection *cc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg;
+  enum GNUNET_CADET_KX_Flags flags;
+
+  if ( (NULL == ct) ||
+       (GNUNET_NO == ct->is_ready) )
+    ct = get_ready_connection (t);
+  if (NULL == ct)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Wanted to send KX_AUTH on %s, but no connection is ready, deferring\n",
+         GCT_2s (t));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
+    t->kx_auth_requested = GNUNET_YES; /* queue KX_AUTH independent of estate */
+    return;
+  }
+  t->kx_auth_requested = GNUNET_NO; /* clear flag */
+  cc = ct->cc;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending KX_AUTH on %s using %s\n",
+       GCT_2s (t),
+       GCC_2s (ct->cc));
+
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH);
+  flags = GNUNET_CADET_KX_FLAG_NONE;
+  if (GNUNET_YES == force_reply)
+    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
+  msg->kx.flags = htonl (flags);
+  msg->kx.cid = *GCC_get_id (cc);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
+                                      &msg->kx.ephemeral_key);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
+                                      &msg->kx.ratchet_key);
+  /* Compute authenticator (this is the main difference to #send_kx()) */
+  GNUNET_CRYPTO_hash (&ax->RK,
+                      sizeof (ax->RK),
+                      &msg->auth);
+
+  /* Compute when to be triggered again; actual job will
+     be scheduled via #connection_ready_cb() */
+  t->kx_retry_delay
+    = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
+  t->next_kx_attempt
+    = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
+
+  /* Send via cc, mark it as unready */
+  mark_connection_unready (ct);
+
+  /* Update state machine, unless we are already OK */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_AUTH_SENT);
+
+  GCC_transmit (cc,
+                env);
 }
 
 
 /**
- * Handle KX message.
+ * Cleanup state used by @a ax.
  *
- * FIXME: sender-authentication in KX is missing!
+ * @param ax state to free, but not memory of @a ax itself
+ */
+static void
+cleanup_ax (struct CadetTunnelAxolotl *ax)
+{
+  while (NULL != ax->skipped_head)
+    delete_skipped_key (ax,
+                        ax->skipped_head);
+  GNUNET_assert (0 == ax->skipped);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->kx_0);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->DHRs);
+}
+
+
+/**
+ * Update our Axolotl key state based on the KX data we received.
+ * Computes the new chain keys, and root keys, etc, and also checks
+ * wether this is a replay of the current chain.
  *
- * @param ct connection/tunnel combo that received encrypted message
- * @param msg the key exchange message
+ * @param[in|out] axolotl chain key state to recompute
+ * @param pid peer identity of the other peer
+ * @param ephemeral_key ephemeral public key of the other peer
+ * @param ratchet_key senders next ephemeral public key
+ * @return #GNUNET_OK on success, #GNUNET_NO if the resulting
+ *       root key is already in @a ax and thus the KX is useless;
+ *       #GNUNET_SYSERR on hard errors (i.e. @a pid is #my_full_id)
  */
-void
-GCT_handle_kx (struct CadetTConnection *ct,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+static int
+update_ax_by_kx (struct CadetTunnelAxolotl *ax,
+                 const struct GNUNET_PeerIdentity *pid,
+                 const struct GNUNET_CRYPTO_EcdhePublicKey *ephemeral_key,
+                 const struct GNUNET_CRYPTO_EcdhePublicKey *ratchet_key)
 {
-  struct CadetTunnel *t = ct->t;
-  struct CadetTunnelAxolotl *ax = &t->ax;
   struct GNUNET_HashCode key_material[3];
   struct GNUNET_CRYPTO_SymmetricSessionKey keys[5];
   const char salt[] = "CADET Axolotl salt";
-  const struct GNUNET_PeerIdentity *pid;
   int am_I_alice;
 
-  pid = GCP_get_id (t->destination);
   if (0 > GNUNET_CRYPTO_cmp_peer_identity (&my_full_id,
                                            pid))
     am_I_alice = GNUNET_YES;
@@ -1325,41 +1485,30 @@ GCT_handle_kx (struct CadetTConnection *ct,
   else
   {
     GNUNET_break_op (0);
-    return;
-  }
-
-  if (0 != (GNUNET_CADET_KX_FLAG_FORCE_REPLY & ntohl (msg->flags)))
-  {
-    if (NULL != t->kx_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->kx_task);
-      t->kx_task = NULL;
-    }
-    send_kx (t,
-             GNUNET_NO);
+    return GNUNET_SYSERR;
   }
 
   if (0 == memcmp (&ax->DHRr,
-                   &msg->ratchet_key,
-                   sizeof (msg->ratchet_key)))
+                   ratchet_key,
+                   sizeof (*ratchet_key)))
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         " known ratchet key, exit\n");
-    return;
+         "Ratchet key already known. Ignoring KX.\n");
+    return GNUNET_NO;
   }
 
-  ax->DHRr = msg->ratchet_key;
+  ax->DHRr = *ratchet_key;
 
   /* ECDH A B0 */
   if (GNUNET_YES == am_I_alice)
   {
     GNUNET_CRYPTO_eddsa_ecdh (my_private_key,      /* A */
-                              &msg->ephemeral_key, /* B0 */
+                              ephemeral_key, /* B0 */
                               &key_material[0]);
   }
   else
   {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* B0 */
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* B0 */
                               &pid->public_key,    /* A */
                               &key_material[0]);
   }
@@ -1367,14 +1516,14 @@ GCT_handle_kx (struct CadetTConnection *ct,
   /* ECDH A0 B */
   if (GNUNET_YES == am_I_alice)
   {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* A0 */
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* A0 */
                               &pid->public_key,    /* B */
                               &key_material[1]);
   }
   else
   {
     GNUNET_CRYPTO_eddsa_ecdh (my_private_key,      /* A */
-                              &msg->ephemeral_key, /* B0 */
+                              ephemeral_key, /* B0 */
                               &key_material[1]);
 
 
@@ -1383,8 +1532,8 @@ GCT_handle_kx (struct CadetTConnection *ct,
   /* ECDH A0 B0 */
   /* (This is the triple-DH, we could probably safely skip this,
      as A0/B0 are already in the key material.) */
-  GNUNET_CRYPTO_ecc_ecdh (ax->kx_0,             /* A0 or B0 */
-                          &msg->ephemeral_key,  /* B0 or A0 */
+  GNUNET_CRYPTO_ecc_ecdh (&ax->kx_0,             /* A0 or B0 */
+                          ephemeral_key,  /* B0 or A0 */
                           &key_material[2]);
 
   /* KDF */
@@ -1397,13 +1546,10 @@ GCT_handle_kx (struct CadetTConnection *ct,
                    &keys[0],
                    sizeof (ax->RK)))
   {
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         " known handshake key, exit\n");
-    return;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Root key of handshake already known. Ignoring KX.\n");
+    return GNUNET_NO;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Handling KX message for tunnel %s\n",
-       GCT_2s (t));
 
   ax->RK = keys[0];
   if (GNUNET_YES == am_I_alice)
@@ -1421,45 +1567,305 @@ GCT_handle_kx (struct CadetTConnection *ct,
     ax->NHKs = keys[3];
     ax->CKs = keys[4];
     ax->ratchet_flag = GNUNET_NO;
-    ax->ratchet_allowed = GNUNET_NO;
-    ax->ratchet_counter = 0;
     ax->ratchet_expiration
       = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(),
                                   ratchet_time);
   }
-  ax->PNs = 0;
-  ax->Nr = 0;
-  ax->Ns = 0;
+  return GNUNET_OK;
+}
 
+
+/**
+ * Try to redo the KX or KX_AUTH handshake, if we can.
+ *
+ * @param cls the `struct CadetTunnel` to do KX for.
+ */
+static void
+retry_kx (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetTunnelAxolotl *ax;
+
+  t->kx_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Trying to make KX progress on %s in state %s\n",
+       GCT_2s (t),
+       estate2s (t->estate));
   switch (t->estate)
   {
-  case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_UNINITIALIZED: /* first attempt */
+  case CADET_TUNNEL_KEY_AX_SENT:       /* trying again */
+    send_kx (t,
+             NULL,
+             &t->ax);
+    break;
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+  case CADET_TUNNEL_KEY_OK:
+    /* Must have been the *other* peer asking us to
+       respond with a KX_AUTH. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Sending AX_AUTH in response to AX so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* Sending AX_AUTH in response to AX_AUTH */
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  GNUNET_NO);
+    break;
+  }
+}
+
+
+/**
+ * Handle KX message that lacks authentication (and which will thus
+ * only be considered authenticated after we respond with our own
+ * KX_AUTH and finally successfully decrypt payload).
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx (struct CadetTConnection *ct,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  struct CadetTunnelAxolotl *ax;
+  int ret;
+
+  if (0 ==
+      memcmp (&t->ax.DHRr,
+              &msg->ratchet_key,
+              sizeof (msg->ratchet_key)))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate KX. Firing back KX_AUTH.\n");
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_NO);
+    return;
+  }
+
+  /* We only keep ONE unverified KX around, so if there is an existing one,
+     clean it up. */
+  if (NULL != t->unverified_ax)
+  {
+    if (0 ==
+        memcmp (&t->unverified_ax->DHRr,
+                &msg->ratchet_key,
+                sizeof (msg->ratchet_key)))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got duplicate unverified KX on %s. Fire back KX_AUTH again.\n",
+           GCT_2s (t));
+      send_kx_auth (t,
+                    ct,
+                    t->unverified_ax,
+                    GNUNET_NO);
+      return;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping old unverified KX state. Got a fresh KX for %s.\n",
+         GCT_2s (t));
+    memset (t->unverified_ax,
+            0,
+            sizeof (struct CadetTunnelAxolotl));
+    t->unverified_ax->DHRs = t->ax.DHRs;
+    t->unverified_ax->kx_0 = t->ax.kx_0;
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Creating fresh unverified KX for %s.\n",
+         GCT_2s (t));
+    t->unverified_ax = GNUNET_new (struct CadetTunnelAxolotl);
+    t->unverified_ax->DHRs = t->ax.DHRs;
+    t->unverified_ax->kx_0 = t->ax.kx_0;
+  }
+  /* Set as the 'current' RK/DHRr the one we are currently using,
+     so that the duplicate-detection logic of
+     #update_ax_by_kx can work. */
+  t->unverified_ax->RK = t->ax.RK;
+  t->unverified_ax->DHRr = t->ax.DHRr;
+  t->unverified_attempts = 0;
+  ax = t->unverified_ax;
+
+  /* Update 'ax' by the new key material */
+  ret = update_ax_by_kx (ax,
+                         GCP_get_id (t->destination),
+                         &msg->ephemeral_key,
+                         &msg->ratchet_key);
+  GNUNET_break (GNUNET_SYSERR != ret);
+  if (GNUNET_OK != ret)
+    return; /* duplicate KX, nothing to do */
+
+  /* move ahead in our state machine */
+  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_RECV);
+  else if (CADET_TUNNEL_KEY_AX_SENT == t->estate)
     GCT_change_estate (t,
-                       CADET_TUNNEL_KEY_PING);
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+
+  /* KX is still not done, try again our end. */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+  {
+    if (NULL != t->kx_task)
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+    t->kx_task
+      = GNUNET_SCHEDULER_add_now (&retry_kx,
+                                  t);
+  }
+}
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx_auth (struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  struct CadetTunnelAxolotl ax_tmp;
+  struct GNUNET_HashCode kx_auth;
+  int ret;
+
+  if ( (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate) ||
+       (CADET_TUNNEL_KEY_AX_RECV == t->estate) )
+  {
+    /* Confusing, we got a KX_AUTH before we even send our own
+       KX. This should not happen. We'll send our own KX ASAP anyway,
+       so let's ignore this here. */
+    GNUNET_break_op (0);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Handling KX_AUTH message for %s\n",
+       GCT_2s (t));
+
+  /* We do everything in ax_tmp until we've checked the authentication
+     so we don't clobber anything we care about by accident. */
+  ax_tmp = t->ax;
+
+  /* Update 'ax' by the new key material */
+  ret = update_ax_by_kx (&ax_tmp,
+                         GCP_get_id (t->destination),
+                         &msg->kx.ephemeral_key,
+                         &msg->kx.ratchet_key);
+  if (GNUNET_OK != ret)
+  {
+    if (GNUNET_NO == ret)
+      GNUNET_STATISTICS_update (stats,
+                                "# redundant KX_AUTH received",
+                                1,
+                                GNUNET_NO);
+    else
+      GNUNET_break (0); /* connect to self!? */
+    return;
+  }
+  GNUNET_CRYPTO_hash (&ax_tmp.RK,
+                      sizeof (ax_tmp.RK),
+                      &kx_auth);
+  if (0 != memcmp (&kx_auth,
+                   &msg->auth,
+                   sizeof (kx_auth)))
+  {
+    /* This KX_AUTH is not using the latest KX/KX_AUTH data
+       we transmitted to the sender, refuse it, try KX again. */
+    GNUNET_STATISTICS_update (stats,
+                              "# KX_AUTH not using our last KX received (auth failure)",
+                              1,
+                              GNUNET_NO);
+    send_kx (t,
+             ct,
+             &t->ax);
+    return;
+  }
+  /* Yep, we're good. */
+  t->ax = ax_tmp;
+  if (NULL != t->unverified_ax)
+  {
+    /* We got some "stale" KX before, drop that. */
+    cleanup_ax (t->unverified_ax);
+    GNUNET_free (t->unverified_ax);
+    t->unverified_ax = NULL;
+  }
+
+  /* move ahead in our state machine */
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_AX_RECV:
+    /* Checked above, this is impossible. */
+    GNUNET_assert (0);
     break;
-  case CADET_TUNNEL_KEY_SENT:
-    /* Got a response to us sending our key; now
-       we can start transmitting! */
+  case CADET_TUNNEL_KEY_AX_SENT:      /* This is the normal case */
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV: /* both peers started KX */
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT: /* both peers now did KX_AUTH */
     GCT_change_estate (t,
                        CADET_TUNNEL_KEY_OK);
-    if (NULL != t->send_task)
-      GNUNET_SCHEDULER_cancel (t->send_task);
-    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
-                                             t);
-    break;
-  case CADET_TUNNEL_KEY_PING:
-    /* Got a key yet again; need encrypted payload to advance */
     break;
   case CADET_TUNNEL_KEY_OK:
-    /* Did not expect a key, but so what. */
-    break;
-  case CADET_TUNNEL_KEY_REKEY:
-    /* Got a key yet again; need encrypted payload to advance */
+    /* Did not expect another KX_AUTH, but so what, still acceptable.
+       Nothing to do here. */
     break;
   }
 }
 
 
+
 /* ************************************** end core crypto ***************************** */
 
 
@@ -1489,12 +1895,12 @@ get_next_free_ctn (struct CadetTunnel *t)
   ctn = ntohl (t->next_ctn.cn);
   while (NULL !=
          GNUNET_CONTAINER_multihashmap32_get (t->channels,
-                                              ctn))
+                                              ctn | highbit))
   {
-    ctn = ((ctn + 1) & (~ HIGH_BIT)) | highbit;
+    ctn = ((ctn + 1) & (~ HIGH_BIT));
   }
-  t->next_ctn.cn = htonl (((ctn + 1) & (~ HIGH_BIT)) | highbit);
-  ret.cn = ntohl (ctn);
+  t->next_ctn.cn = htonl ((ctn + 1) & (~ HIGH_BIT));
+  ret.cn = htonl (ctn | highbit);
   return ret;
 }
 
@@ -1515,23 +1921,97 @@ GCT_add_channel (struct CadetTunnel *t,
   struct GNUNET_CADET_ChannelTunnelNumber ctn;
 
   ctn = get_next_free_ctn (t);
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap32_put (t->channels,
                                                       ntohl (ctn.cn),
                                                       ch,
                                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding channel %s to tunnel %s\n",
+       "Adding %s to %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  if ( (CADET_TUNNEL_KEY_OK == t->estate) ||
-       (CADET_TUNNEL_KEY_REKEY == t->estate) )
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    /* waiting for connection to start KX */
+    break;
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* we're currently waiting for KX to complete */
+    break;
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* waiting for OTHER peer to send us data,
+       we might need to prompt more aggressively! */
+    if (NULL == t->kx_task)
+      t->kx_task
+        = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
+                                   &retry_kx,
+                                   t);
+    break;
+  case CADET_TUNNEL_KEY_OK:
+    /* We are ready. Tell the new channel that we are up. */
     GCCH_tunnel_up (ch);
+    break;
+  }
   return ctn;
 }
 
 
 /**
+ * We lost a connection, remove it from our list and clean up
+ * the connection object itself.
+ *
+ * @param ct binding of connection to tunnel of the connection that was lost.
+ */
+void
+GCT_connection_lost (struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = ct->t;
+
+  if (GNUNET_YES == ct->is_ready)
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                                 t->connection_ready_tail,
+                                 ct);
+    t->num_ready_connections--;
+  }
+  else
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                                 t->connection_busy_tail,
+                                 ct);
+    t->num_busy_connections--;
+  }
+  GNUNET_free (ct);
+}
+
+
+/**
+ * Clean up connection @a ct of a tunnel.
+ *
+ * @param cls the `struct CadetTunnel`
+ * @param ct connection to clean up
+ */
+static void
+destroy_t_connection (void *cls,
+                      struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetConnection *cc = ct->cc;
+
+  GNUNET_assert (ct->t == t);
+  GCT_connection_lost (ct);
+  GCC_destroy_without_tunnel (cc);
+}
+
+
+/**
  * This tunnel is no longer used, destroy it.
  *
  * @param cls the idle tunnel
@@ -1540,27 +2020,23 @@ static void
 destroy_tunnel (void *cls)
 {
   struct CadetTunnel *t = cls;
-  struct CadetTConnection *ct;
   struct CadetTunnelQueueEntry *tq;
 
   t->destroy_task = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying idle tunnel %s\n",
+       "Destroying idle %s\n",
        GCT_2s (t));
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (t->channels));
-  while (NULL != (ct = t->connection_head))
-  {
-    GNUNET_assert (ct->t == t);
-    GNUNET_CONTAINER_DLL_remove (t->connection_head,
-                                 t->connection_tail,
-                                 ct);
-    GCC_destroy (ct->cc);
-    GNUNET_free (ct);
-  }
+  GNUNET_assert (0 == GCT_count_channels (t));
+  GCT_iterate_connections (t,
+                           &destroy_t_connection,
+                           t);
+  GNUNET_assert (NULL == t->connection_ready_head);
+  GNUNET_assert (NULL == t->connection_busy_head);
   while (NULL != (tq = t->tq_head))
   {
     if (NULL != tq->cont)
-      tq->cont (tq->cont_cls);
+      tq->cont (tq->cont_cls,
+                NULL);
     GCT_send_cancel (tq);
   }
   GCP_drop_tunnel (t->destination,
@@ -1583,12 +2059,13 @@ destroy_tunnel (void *cls)
   }
   GNUNET_MST_destroy (t->mst);
   GNUNET_MQ_destroy (t->mq);
-  while (NULL != t->ax.skipped_head)
-    delete_skipped_key (t,
-                        t->ax.skipped_head);
-  GNUNET_assert (0 == t->ax.skipped);
-  GNUNET_free_non_null (t->ax.kx_0);
-  GNUNET_free_non_null (t->ax.DHRs);
+  if (NULL != t->unverified_ax)
+  {
+    cleanup_ax (t->unverified_ax);
+    GNUNET_free (t->unverified_ax);
+  }
+  cleanup_ax (&t->ax);
+  GNUNET_assert (NULL == t->destroy_task);
   GNUNET_free (t);
 }
 
@@ -1606,19 +2083,21 @@ GCT_remove_channel (struct CadetTunnel *t,
                     struct GNUNET_CADET_ChannelTunnelNumber ctn)
 {
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing channel %s from tunnel %s\n",
+       "Removing %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap32_remove (t->channels,
                                                          ntohl (ctn.cn),
                                                          ch));
-  if (0 ==
-      GNUNET_CONTAINER_multihashmap32_size (t->channels))
+  if ( (0 ==
+        GCT_count_channels (t)) &&
+       (NULL == t->destroy_task) )
   {
-    t->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
-                                                    &destroy_tunnel,
-                                                    t);
+    t->destroy_task
+      = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
+                                      &destroy_tunnel,
+                                      t);
   }
 }
 
@@ -1638,7 +2117,8 @@ destroy_remaining_channels (void *cls,
 {
   struct CadetChannel *ch = value;
 
-  GCCH_handle_remote_destroy (ch);
+  GCCH_handle_remote_destroy (ch,
+                              NULL);
   return GNUNET_OK;
 }
 
@@ -1656,7 +2136,7 @@ GCT_destroy_tunnel_now (struct CadetTunnel *t)
                                            &destroy_remaining_channels,
                                            t);
   GNUNET_assert (0 ==
-                 GNUNET_CONTAINER_multihashmap32_size (t->channels));
+                 GCT_count_channels (t));
   if (NULL != t->destroy_task)
   {
     GNUNET_SCHEDULER_cancel (t->destroy_task);
@@ -1667,25 +2147,6 @@ GCT_destroy_tunnel_now (struct CadetTunnel *t)
 
 
 /**
- * It's been a while, we should try to redo the KX, if we can.
- *
- * @param cls the `struct CadetTunnel` to do KX for.
- */
-static void
-retry_kx (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->kx_task = NULL;
-  send_kx (t,
-           ( (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate) ||
-             (CADET_TUNNEL_KEY_SENT == t->estate) )
-           ? GNUNET_YES
-           : GNUNET_NO);
-}
-
-
-/**
  * Send normal payload from queue in @a t via connection @a ct.
  * Does nothing if our payload queue is empty.
  *
@@ -1716,7 +2177,7 @@ try_send_normal_payload (struct CadetTunnel *t,
                                tq);
   if (NULL != tq->cid)
     *tq->cid = *GCC_get_id (ct->cc);
-  ct->is_ready = GNUNET_NO;
+  mark_connection_unready (ct);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Sending payload of %s on %s\n",
        GCT_2s (t),
@@ -1724,7 +2185,8 @@ try_send_normal_payload (struct CadetTunnel *t,
   GCC_transmit (ct->cc,
                 tq->env);
   if (NULL != tq->cont)
-    tq->cont (tq->cont_cls);
+    tq->cont (tq->cont_cls,
+              GCC_get_id (ct->cc));
   GNUNET_free (tq);
 }
 
@@ -1747,44 +2209,74 @@ connection_ready_cb (void *cls,
   if (GNUNET_NO == is_ready)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Connection %s no longer ready for tunnel %s\n",
+         "%s no longer ready for %s\n",
          GCC_2s (ct->cc),
          GCT_2s (t));
-    ct->is_ready = GNUNET_NO;
+    mark_connection_unready (ct);
     return;
   }
+  GNUNET_assert (GNUNET_NO == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_busy_connections);
+  t->num_busy_connections--;
   ct->is_ready = GNUNET_YES;
+  GNUNET_CONTAINER_DLL_insert_tail (t->connection_ready_head,
+                                    t->connection_ready_tail,
+                                    ct);
+  t->num_ready_connections++;
+
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Connection %s now ready for tunnel %s in state %s\n",
+       "%s now ready for %s in state %s\n",
        GCC_2s (ct->cc),
        GCT_2s (t),
        estate2s (t->estate));
   switch (t->estate)
   {
   case CADET_TUNNEL_KEY_UNINITIALIZED:
+    /* Do not begin KX if WE have no channels waiting! */
+    if (0 == GCT_count_channels (t))
+      return;
+    /* We are uninitialized, just transmit immediately,
+       without undue delay. */
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
     send_kx (t,
-             GNUNET_YES);
+             ct,
+             &t->ax);
     break;
-  case CADET_TUNNEL_KEY_SENT:
-  case CADET_TUNNEL_KEY_PING:
-    /* opportunity to #retry_kx() starts now, schedule job */
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* we're currently waiting for KX to complete, schedule job */
     if (NULL == t->kx_task)
-    {
       t->kx_task
         = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
                                    &retry_kx,
                                    t);
-    }
     break;
   case CADET_TUNNEL_KEY_OK:
+    if (GNUNET_YES == t->kx_auth_requested)
+    {
+      if (NULL != t->kx_task)
+      {
+        GNUNET_SCHEDULER_cancel (t->kx_task);
+        t->kx_task = NULL;
+      }
+      send_kx_auth (t,
+                    ct,
+                    &t->ax,
+                    GNUNET_NO);
+      return;
+    }
     try_send_normal_payload (t,
                              ct);
     break;
-  case CADET_TUNNEL_KEY_REKEY:
-    send_kx (t,
-             GNUNET_NO);
-    t->estate = CADET_TUNNEL_KEY_OK;
-    break;
   }
 }
 
@@ -1815,6 +2307,118 @@ trigger_transmissions (void *cls)
 
 
 /**
+ * Closure for #evaluate_connection. Used to assemble summary information
+ * about the existing connections so we can evaluate a new path.
+ */
+struct EvaluationSummary
+{
+
+  /**
+   * Minimum length of any of our connections, `UINT_MAX` if we have none.
+   */
+  unsigned int min_length;
+
+  /**
+   * Maximum length of any of our connections, 0 if we have none.
+   */
+  unsigned int max_length;
+
+  /**
+   * Minimum desirability of any of our connections, UINT64_MAX if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType min_desire;
+
+  /**
+   * Maximum desirability of any of our connections, 0 if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType max_desire;
+
+  /**
+   * Path we are comparing against for #evaluate_connection, can be NULL.
+   */
+  struct CadetPeerPath *path;
+
+  /**
+   * Connection deemed the "worst" so far encountered by #evaluate_connection,
+   * NULL if we did not yet encounter any connections.
+   */
+  struct CadetTConnection *worst;
+
+  /**
+   * Numeric score of @e worst, only set if @e worst is non-NULL.
+   */
+  double worst_score;
+
+  /**
+   * Set to #GNUNET_YES if we have a connection over @e path already.
+   */
+  int duplicate;
+
+};
+
+
+/**
+ * Evaluate a connection, updating our summary information in @a cls about
+ * what kinds of connections we have.
+ *
+ * @param cls the `struct EvaluationSummary *` to update
+ * @param ct a connection to include in the summary
+ */
+static void
+evaluate_connection (void *cls,
+                     struct CadetTConnection *ct)
+{
+  struct EvaluationSummary *es = cls;
+  struct CadetConnection *cc = ct->cc;
+  struct CadetPeerPath *ps = GCC_get_path (cc);
+  const struct CadetConnectionMetrics *metrics;
+  GNUNET_CONTAINER_HeapCostType ct_desirability;
+  struct GNUNET_TIME_Relative uptime;
+  struct GNUNET_TIME_Relative last_use;
+  uint32_t ct_length;
+  double score;
+  double success_rate;
+
+  if (ps == es->path)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate path %s.\n",
+         GCPP_2s (es->path));
+    es->duplicate = GNUNET_YES;
+    return;
+  }
+  ct_desirability = GCPP_get_desirability (ps);
+  ct_length = GCPP_get_length (ps);
+  metrics = GCC_get_metrics (cc);
+  uptime = GNUNET_TIME_absolute_get_duration (metrics->age);
+  last_use = GNUNET_TIME_absolute_get_duration (metrics->last_use);
+  /* We add 1.0 here to avoid division by zero. */
+  success_rate = (metrics->num_acked_transmissions + 1.0) / (metrics->num_successes + 1.0);
+  score
+    = ct_desirability
+    + 100.0 / (1.0 + ct_length) /* longer paths = better */
+    + sqrt (uptime.rel_value_us / 60000000LL) /* larger uptime = better */
+    - last_use.rel_value_us / 1000L;          /* longer idle = worse */
+  score *= success_rate;        /* weigh overall by success rate */
+
+  if ( (NULL == es->worst) ||
+       (score < es->worst_score) )
+  {
+    es->worst = ct;
+    es->worst_score = score;
+  }
+  es->min_length = GNUNET_MIN (es->min_length,
+                               ct_length);
+  es->max_length = GNUNET_MAX (es->max_length,
+                               ct_length);
+  es->min_desire = GNUNET_MIN (es->min_desire,
+                               ct_desirability);
+  es->max_desire = GNUNET_MAX (es->max_desire,
+                               ct_desirability);
+}
+
+
+/**
  * Consider using the path @a p for the tunnel @a t.
  * The tunnel destination is at offset @a off in path @a p.
  *
@@ -1829,31 +2433,24 @@ consider_path_cb (void *cls,
                   unsigned int off)
 {
   struct CadetTunnel *t = cls;
-  unsigned int min_length = UINT_MAX;
-  GNUNET_CONTAINER_HeapCostType max_desire = 0;
+  struct EvaluationSummary es;
   struct CadetTConnection *ct;
 
-  /* Check if we care about the new path. */
-  for (ct = t->connection_head;
-       NULL != ct;
-       ct = ct->next)
-  {
-    struct CadetPeerPath *ps;
-
-    ps = GCC_get_path (ct->cc);
-    if (ps == path)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Ignoring duplicate path %s for tunnel %s.\n",
-           GCPP_2s (path),
-           GCT_2s (t));
-      return GNUNET_YES; /* duplicate */
-    }
-    min_length = GNUNET_MIN (min_length,
-                             GCPP_get_length (ps));
-    max_desire = GNUNET_MAX (max_desire,
-                             GCPP_get_desirability (ps));
-  }
+  GNUNET_assert (off < GCPP_get_length (path));
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = path;
+  es.duplicate = GNUNET_NO;
+  es.worst = NULL;
+
+  /* Compute evaluation summary over existing connections. */
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if (GNUNET_YES == es.duplicate)
+    return GNUNET_YES;
 
   /* FIXME: not sure we should really just count
      'num_connections' here, as they may all have
@@ -1862,18 +2459,20 @@ consider_path_cb (void *cls,
   /* We iterate by increasing path length; if we have enough paths and
      this one is more than twice as long than what we are currently
      using, then ignore all of these super-long ones! */
-  if ( (t->num_connections > DESIRED_CONNECTIONS_PER_TUNNEL) &&
-       (min_length * 2 < off) )
+  if ( (GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (es.min_length * 2 < off) &&
+       (es.max_length < off) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Ignoring paths of length %u, they are way too long.\n",
-         min_length * 2);
+         es.min_length * 2);
     return GNUNET_NO;
   }
   /* If we have enough paths and this one looks no better, ignore it. */
-  if ( (t->num_connections >= DESIRED_CONNECTIONS_PER_TUNNEL) &&
-       (min_length < GCPP_get_length (path)) &&
-       (max_desire > GCPP_get_desirability (path)) )
+  if ( (GCT_count_any_connections (t) >= DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (es.min_length < GCPP_get_length (path)) &&
+       (es.min_desire > GCPP_get_desirability (path)) &&
+       (es.max_length < off) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Ignoring path (%u/%llu) to %s, got something better already.\n",
@@ -1890,19 +2489,22 @@ consider_path_cb (void *cls,
   ct->t = t;
   ct->cc = GCC_create (t->destination,
                        path,
+                       off,
+                       GNUNET_CADET_OPTION_DEFAULT, /* FIXME: set based on what channels want/need! */
                        ct,
                        &connection_ready_cb,
                        ct);
+
   /* FIXME: schedule job to kill connection (and path?)  if it takes
      too long to get ready! (And track performance data on how long
      other connections took with the tunnel!)
      => Note: to be done within 'connection'-logic! */
-  GNUNET_CONTAINER_DLL_insert (t->connection_head,
-                               t->connection_tail,
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
                                ct);
-  t->num_connections++;
+  t->num_busy_connections++;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Found interesting path %s for tunnel %s, created connection %s\n",
+       "Found interesting path %s for %s, created %s\n",
        GCPP_2s (path),
        GCT_2s (t),
        GCC_2s (ct->cc));
@@ -1927,17 +2529,50 @@ static void
 maintain_connections_cb (void *cls)
 {
   struct CadetTunnel *t = cls;
+  struct GNUNET_TIME_Relative delay;
+  struct EvaluationSummary es;
 
   t->maintain_connections_task = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Performing connection maintenance for tunnel %s.\n",
+       "Performing connection maintenance for %s.\n",
        GCT_2s (t));
 
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = NULL;
+  es.worst = NULL;
+  es.duplicate = GNUNET_NO;
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if ( (NULL != es.worst) &&
+       (GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL) )
+  {
+    /* Clear out worst-performing connection 'es.worst'. */
+    destroy_t_connection (t,
+                          es.worst);
+  }
+
+  /* Consider additional paths */
   (void) GCP_iterate_paths (t->destination,
                             &consider_path_cb,
                             t);
 
-  GNUNET_break (0); // FIXME: implement!
+  /* FIXME: calculate when to try again based on how well we are doing;
+     in particular, if we have to few connections, we might be able
+     to do without this (as PATHS should tell us whenever a new path
+     is available instantly; however, need to make sure this job is
+     restarted after that happens).
+     Furthermore, if the paths we do know are in a reasonably narrow
+     quality band and are plentyful, we might also consider us stabilized
+     and then reduce the frequency accordingly.  */
+  delay = GNUNET_TIME_UNIT_MINUTES;
+  t->maintain_connections_task
+    = GNUNET_SCHEDULER_add_delayed (delay,
+                                    &maintain_connections_cb,
+                                    t);
 }
 
 
@@ -1954,6 +2589,10 @@ GCT_consider_path (struct CadetTunnel *t,
                    struct CadetPeerPath *p,
                    unsigned int off)
 {
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Considering %s for %s\n",
+       GCPP_2s (p),
+       GCT_2s (t));
   (void) consider_path_cb (t,
                            p,
                            off);
@@ -1961,7 +2600,7 @@ GCT_consider_path (struct CadetTunnel *t,
 
 
 /**
- * NOT IMPLEMENTED.
+ * We got a keepalive. Track in statistics.
  *
  * @param cls the `struct CadetTunnel` for which we decrypted the message
  * @param msg  the message we received on the tunnel
@@ -1972,7 +2611,13 @@ handle_plaintext_keepalive (void *cls,
 {
   struct CadetTunnel *t = cls;
 
-  GNUNET_break (0); // FIXME
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KEEPALIVE on %s\n",
+       GCT_2s (t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives received",
+                            1,
+                            GNUNET_NO);
 }
 
 
@@ -2012,7 +2657,7 @@ handle_plaintext_data (void *cls,
     /* We don't know about such a channel, might have been destroyed on our
        end in the meantime, or never existed. Send back a DESTROY. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved %u bytes of application data for unknown channel %u, sending DESTROY\n",
+         "Received %u bytes of application data for unknown channel %u, sending DESTROY\n",
          (unsigned int) (ntohs (msg->header.size) - sizeof (*msg)),
          ntohl (msg->ctn.cn));
     GCT_send_channel_destroy (t,
@@ -2020,6 +2665,7 @@ handle_plaintext_data (void *cls,
     return;
   }
   GCCH_handle_channel_plaintext_data (ch,
+                                      GCC_get_id (t->current_ct->cc),
                                       msg);
 }
 
@@ -2046,13 +2692,14 @@ handle_plaintext_data_ack (void *cls,
     /* We don't know about such a channel, might have been destroyed on our
        end in the meantime, or never existed. Send back a DESTROY. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved DATA_ACK for unknown channel %u, sending DESTROY\n",
+         "Received DATA_ACK for unknown channel %u, sending DESTROY\n",
          ntohl (ack->ctn.cn));
     GCT_send_channel_destroy (t,
                               ack->ctn);
     return;
   }
   GCCH_handle_channel_plaintext_data_ack (ch,
+                                          GCC_get_id (t->current_ct->cc),
                                           ack);
 }
 
@@ -2076,21 +2723,27 @@ handle_plaintext_channel_open (void *cls,
   if (NULL != ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved duplicate channel OPEN on port %s from %s (%s), resending ACK\n",
-         GNUNET_h2s (&copen->port),
+         "Received duplicate channel CHANNEL_OPEN on h_port %s from %s (%s), resending ACK\n",
+         GNUNET_h2s (&copen->h_port),
          GCT_2s (t),
          GCCH_2s (ch));
-    GCCH_handle_duplicate_open (ch);
+    GCCH_handle_duplicate_open (ch,
+                                GCC_get_id (t->current_ct->cc));
     return;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Receicved channel OPEN on port %s from %s\n",
-       GNUNET_h2s (&copen->port),
+       "Received CHANNEL_OPEN on h_port %s from %s\n",
+       GNUNET_h2s (&copen->h_port),
        GCT_2s (t));
   ch = GCCH_channel_incoming_new (t,
                                   copen->ctn,
-                                  &copen->port,
+                                  &copen->h_port,
                                   ntohl (copen->opt));
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap32_put (t->channels,
                                                       ntohl (copen->ctn.cn),
@@ -2109,7 +2762,7 @@ void
 GCT_send_channel_destroy (struct CadetTunnel *t,
                           struct GNUNET_CADET_ChannelTunnelNumber ctn)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
+  struct GNUNET_CADET_ChannelDestroyMessage msg;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Sending DESTORY message for channel ID %u\n",
@@ -2135,7 +2788,7 @@ GCT_send_channel_destroy (struct CadetTunnel *t,
  */
 static void
 handle_plaintext_channel_open_ack (void *cls,
-                                   const struct GNUNET_CADET_ChannelManageMessage *cm)
+                                   const struct GNUNET_CADET_ChannelOpenAckMessage *cm)
 {
   struct CadetTunnel *t = cls;
   struct CadetChannel *ch;
@@ -2157,7 +2810,9 @@ handle_plaintext_channel_open_ack (void *cls,
        "Received channel OPEN_ACK on channel %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  GCCH_handle_channel_open_ack (ch);
+  GCCH_handle_channel_open_ack (ch,
+                                GCC_get_id (t->current_ct->cc),
+                                &cm->port);
 }
 
 
@@ -2170,7 +2825,7 @@ handle_plaintext_channel_open_ack (void *cls,
  */
 static void
 handle_plaintext_channel_destroy (void *cls,
-                                  const struct GNUNET_CADET_ChannelManageMessage *cm)
+                                  const struct GNUNET_CADET_ChannelDestroyMessage *cm)
 {
   struct CadetTunnel *t = cls;
   struct CadetChannel *ch;
@@ -2187,10 +2842,11 @@ handle_plaintext_channel_destroy (void *cls,
     return;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Receicved channel DESTROY on %s from %s\n",
+       "Received channel DESTROY on %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  GCCH_handle_remote_destroy (ch);
+  GCCH_handle_remote_destroy (ch,
+                              GCC_get_id (t->current_ct->cc));
 }
 
 
@@ -2208,6 +2864,7 @@ handle_decrypted (void *cls,
 {
   struct CadetTunnel *t = cls;
 
+  GNUNET_assert (NULL != t->current_ct);
   GNUNET_MQ_inject_message (t->mq,
                             msg);
   return GNUNET_OK;
@@ -2259,17 +2916,19 @@ GCT_create_tunnel (struct CadetPeer *destination)
                              t),
     GNUNET_MQ_hd_fixed_size (plaintext_channel_open_ack,
                              GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK,
-                             struct GNUNET_CADET_ChannelManageMessage,
+                             struct GNUNET_CADET_ChannelOpenAckMessage,
                              t),
     GNUNET_MQ_hd_fixed_size (plaintext_channel_destroy,
                              GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY,
-                             struct GNUNET_CADET_ChannelManageMessage,
+                             struct GNUNET_CADET_ChannelDestroyMessage,
                              t),
     GNUNET_MQ_handler_end ()
   };
 
-  new_ephemeral (t);
-  t->ax.kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
+  t->kx_retry_delay = INITIAL_KX_RETRY_DELAY;
+  new_ephemeral (&t->ax);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdhe_key_create2 (&t->ax.kx_0));
   t->destination = destination;
   t->channels = GNUNET_CONTAINER_multihashmap32_create (8);
   t->maintain_connections_task
@@ -2293,11 +2952,15 @@ GCT_create_tunnel (struct CadetPeer *destination)
  *
  * @param t a tunnel
  * @param cid connection identifer to use for the connection
+ * @param options options for the connection
  * @param path path to use for the connection
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_SYSERR on failure (duplicate connection)
  */
-void
+int
 GCT_add_inbound_connection (struct CadetTunnel *t,
                             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                            enum GNUNET_CADET_ChannelOption options,
                             struct CadetPeerPath *path)
 {
   struct CadetTConnection *ct;
@@ -2307,22 +2970,33 @@ GCT_add_inbound_connection (struct CadetTunnel *t,
   ct->t = t;
   ct->cc = GCC_create_inbound (t->destination,
                                path,
+                               options,
                                ct,
                                cid,
                                &connection_ready_cb,
                                ct);
+  if (NULL == ct->cc)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s refused inbound %s (duplicate)\n",
+         GCT_2s (t),
+         GCC_2s (ct->cc));
+    GNUNET_free (ct);
+    return GNUNET_SYSERR;
+  }
   /* FIXME: schedule job to kill connection (and path?)  if it takes
      too long to get ready! (And track performance data on how long
      other connections took with the tunnel!)
      => Note: to be done within 'connection'-logic! */
-  GNUNET_CONTAINER_DLL_insert (t->connection_head,
-                               t->connection_tail,
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
                                ct);
-  t->num_connections++;
+  t->num_busy_connections++;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s has new connection %s\n",
+       "%s has new %s\n",
        GCT_2s (t),
        GCC_2s (ct->cc));
+  return GNUNET_OK;
 }
 
 
@@ -2342,7 +3016,7 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
   ssize_t decrypted_size;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s received %u bytes of encrypted data in state %d\n",
+       "%s received %u bytes of encrypted data in state %d\n",
        GCT_2s (t),
        (unsigned int) size,
        t->estate);
@@ -2350,66 +3024,158 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
   switch (t->estate)
   {
   case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_AX_RECV:
     /* We did not even SEND our KX, how can the other peer
-       send us encrypted data? */
-    GNUNET_break_op (0);
+       send us encrypted data? Must have been that we went
+       down and the other peer still things we are up.
+       Let's send it KX back. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without any KX",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
-  case CADET_TUNNEL_KEY_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We send KX, and other peer send KX to us at the same time.
+       Neither KX is AUTH'ed, so let's try KX_AUTH this time. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without KX_AUTH",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_YES);
+    return;
+  case CADET_TUNNEL_KEY_AX_SENT:
     /* We did not get the KX of the other peer, but that
-       might have been lost.  Ask for KX again. */
+       might have been lost.  Send our KX again immediately. */
     GNUNET_STATISTICS_update (stats,
                               "# received encrypted without KX",
                               1,
                               GNUNET_NO);
     if (NULL != t->kx_task)
+    {
       GNUNET_SCHEDULER_cancel (t->kx_task);
-    t->kx_task = GNUNET_SCHEDULER_add_now (&retry_kx,
-                                           t);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
-  case CADET_TUNNEL_KEY_PING:
-    /* Great, first payload, we might graduate to OK */
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* Great, first payload, we might graduate to OK! */
   case CADET_TUNNEL_KEY_OK:
-  case CADET_TUNNEL_KEY_REKEY:
+    /* We are up and running, all good. */
     break;
   }
 
-  GNUNET_STATISTICS_update (stats,
-                            "# received encrypted",
-                            1,
-                            GNUNET_NO);
-  decrypted_size = t_ax_decrypt_and_validate (t,
-                                              cbuf,
-                                              msg,
-                                              size);
+  decrypted_size = -1;
+  if (CADET_TUNNEL_KEY_OK == t->estate)
+  {
+    /* We have well-established key material available,
+       try that. (This is the common case.) */
+    decrypted_size = t_ax_decrypt_and_validate (&t->ax,
+                                                cbuf,
+                                                msg,
+                                                size);
+  }
+
+  if ( (-1 == decrypted_size) &&
+       (NULL != t->unverified_ax) )
+  {
+    /* We have un-authenticated KX material available. We should try
+       this as a back-up option, in case the sender crashed and
+       switched keys. */
+    decrypted_size = t_ax_decrypt_and_validate (t->unverified_ax,
+                                                cbuf,
+                                                msg,
+                                                size);
+    if (-1 != decrypted_size)
+    {
+      /* It worked! Treat this as authentication of the AX data! */
+      cleanup_ax (&t->ax);
+      t->ax = *t->unverified_ax;
+      GNUNET_free (t->unverified_ax);
+      t->unverified_ax = NULL;
+    }
+    if (CADET_TUNNEL_KEY_AX_AUTH_SENT == t->estate)
+    {
+      /* First time it worked, move tunnel into production! */
+      GCT_change_estate (t,
+                         CADET_TUNNEL_KEY_OK);
+      if (NULL != t->send_task)
+        GNUNET_SCHEDULER_cancel (t->send_task);
+      t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                               t);
+    }
+  }
+  if (NULL != t->unverified_ax)
+  {
+    /* We had unverified KX material that was useless; so increment
+       counter and eventually move to ignore it.  Note that we even do
+       this increment if we successfully decrypted with the old KX
+       material and thus didn't even both with the new one.  This is
+       the ideal case, as a malicious injection of bogus KX data
+       basically only causes us to increment a counter a few times. */
+    t->unverified_attempts++;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Failed to decrypt message with unverified KX data %u times\n",
+         t->unverified_attempts);
+    if (t->unverified_attempts > MAX_UNVERIFIED_ATTEMPTS)
+    {
+      cleanup_ax (t->unverified_ax);
+      GNUNET_free (t->unverified_ax);
+      t->unverified_ax = NULL;
+    }
+  }
 
   if (-1 == decrypted_size)
   {
-    GNUNET_break_op (0);
+    /* Decryption failed for good, complain. */
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Tunnel %s failed to decrypt and validate encrypted data\n",
+         "%s failed to decrypt and validate encrypted data, retrying KX\n",
          GCT_2s (t));
     GNUNET_STATISTICS_update (stats,
                               "# unable to decrypt",
                               1,
                               GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
   }
-  if (CADET_TUNNEL_KEY_PING == t->estate)
-  {
-    GCT_change_estate (t,
-                       CADET_TUNNEL_KEY_OK);
-    if (NULL != t->send_task)
-      GNUNET_SCHEDULER_cancel (t->send_task);
-    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
-                                             t);
-  }
+  GNUNET_STATISTICS_update (stats,
+                            "# decrypted bytes",
+                            decrypted_size,
+                            GNUNET_NO);
+
   /* The MST will ultimately call #handle_decrypted() on each message. */
+  t->current_ct = ct;
   GNUNET_break_op (GNUNET_OK ==
                    GNUNET_MST_from_buffer (t->mst,
                                            cbuf,
                                            decrypted_size,
                                            GNUNET_YES,
                                            GNUNET_NO));
+  t->current_ct = NULL;
 }
 
 
@@ -2421,12 +3187,12 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
  * @param t Tunnel on which this message is transmitted.
  * @param cont Continuation to call once message is really sent.
  * @param cont_cls Closure for @c cont.
- * @return Handle to cancel message. NULL if @c cont is NULL.
+ * @return Handle to cancel message
  */
 struct CadetTunnelQueueEntry *
 GCT_send (struct CadetTunnel *t,
           const struct GNUNET_MessageHeader *message,
-          GNUNET_SCHEDULER_TaskCallback cont,
+          GCT_SendContinuation cont,
           void *cont_cls)
 {
   struct CadetTunnelQueueEntry *tq;
@@ -2434,26 +3200,38 @@ GCT_send (struct CadetTunnel *t,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_TunnelEncryptedMessage *ax_msg;
 
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
   payload_size = ntohs (message->size);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Encrypting %u bytes for tunnel %s\n",
+       "Encrypting %u bytes for %s\n",
        (unsigned int) payload_size,
        GCT_2s (t));
   env = GNUNET_MQ_msg_extra (ax_msg,
                              payload_size,
                              GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED);
-  t_ax_encrypt (t,
+  t_ax_encrypt (&t->ax,
                 &ax_msg[1],
                 message,
                 payload_size);
-  ax_msg->Ns = htonl (t->ax.Ns++);
-  ax_msg->PNs = htonl (t->ax.PNs);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax.DHRs,
-                                      &ax_msg->DHRs);
-  t_h_encrypt (t,
+  GNUNET_STATISTICS_update (stats,
+                            "# encrypted bytes",
+                            payload_size,
+                            GNUNET_NO);
+  ax_msg->ax_header.Ns = htonl (t->ax.Ns++);
+  ax_msg->ax_header.PNs = htonl (t->ax.PNs);
+  /* FIXME: we should do this once, not once per message;
+     this is a point multiplication, and DHRs does not
+     change all the time. */
+  GNUNET_CRYPTO_ecdhe_key_get_public (&t->ax.DHRs,
+                                      &ax_msg->ax_header.DHRs);
+  t_h_encrypt (&t->ax,
                ax_msg);
-  t_hmac (&ax_msg->Ns,
-          AX_HEADER_SIZE + payload_size,
+  t_hmac (&ax_msg->ax_header,
+          sizeof (struct GNUNET_CADET_AxHeader) + payload_size,
           0,
           &t->ax.HKs,
           &ax_msg->hmac);
@@ -2510,11 +3288,23 @@ GCT_iterate_connections (struct CadetTunnel *t,
                          GCT_ConnectionIterator iter,
                          void *iter_cls)
 {
-  for (struct CadetTConnection *ct = t->connection_head;
+  struct CadetTConnection *n;
+  for (struct CadetTConnection *ct = t->connection_ready_head;
        NULL != ct;
-       ct = ct->next)
+       ct = n)
+  {
+    n = ct->next;
+    iter (iter_cls,
+          ct);
+  }
+  for (struct CadetTConnection *ct = t->connection_busy_head;
+       NULL != ct;
+       ct = n)
+  {
+    n = ct->next;
     iter (iter_cls,
-          ct->cc);
+          ct);
+  }
 }
 
 
@@ -2628,10 +3418,7 @@ GCT_debug (const struct CadetTunnel *t,
         GCT_2s (t),
         estate2s (t->estate),
         t->tq_len,
-        t->num_connections);
-#if DUMP_KEYS_TO_STDERR
-  ax_debug (t->ax, level);
-#endif
+        GCT_count_any_connections (t));
   LOG2 (level,
         "TTT channels:\n");
   GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
@@ -2639,7 +3426,10 @@ GCT_debug (const struct CadetTunnel *t,
                                            &level);
   LOG2 (level,
         "TTT connections:\n");
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = iter_c->next)
+  for (iter_c = t->connection_ready_head; NULL != iter_c; iter_c = iter_c->next)
+    GCC_debug (iter_c->cc,
+               level);
+  for (iter_c = t->connection_busy_head; NULL != iter_c; iter_c = iter_c->next)
     GCC_debug (iter_c->cc,
                level);
 
diff --git a/src/cadet/gnunet-service-cadet-new_tunnels.h b/src/cadet/gnunet-service-cadet_tunnels.h
index c867a9e82..4a3619ab6 100644
--- a/src/cadet/gnunet-service-cadet-new_tunnels.h
+++ b/src/cadet/gnunet-service-cadet_tunnels.h
@@ -20,7 +20,7 @@
 */
 
 /**
- * @file cadet/gnunet-service-cadet-new_tunnels.h
+ * @file cadet/gnunet-service-cadet_tunnels.h
  * @brief Information we track per tunnel.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
@@ -28,7 +28,7 @@
 #ifndef GNUNET_SERVICE_CADET_TUNNELS_H
 #define GNUNET_SERVICE_CADET_TUNNELS_H
 
-#include "gnunet-service-cadet-new.h"
+#include "gnunet-service-cadet.h"
 #include "cadet_protocol.h"
 
 
@@ -50,29 +50,36 @@ enum CadetTunnelEState
   CADET_TUNNEL_KEY_UNINITIALIZED,
 
   /**
-   * Ephemeral key sent, waiting for peer's key.
+   * KX message sent, waiting for other peer's KX_AUTH.
    */
-  CADET_TUNNEL_KEY_SENT,
+  CADET_TUNNEL_KEY_AX_SENT,
 
   /**
-   * Key received and we sent ours back, but we got no traffic yet.
+   * KX message received, trying to send back KX_AUTH.
+   */
+  CADET_TUNNEL_KEY_AX_RECV,
+
+  /**
+   * KX message sent and received, trying to send back KX_AUTH.
+   */
+  CADET_TUNNEL_KEY_AX_SENT_AND_RECV,
+
+  /**
+   * KX received and we sent KX_AUTH back, but we got no traffic yet,
+   * so we're waiting for either KX_AUTH or ENCRYPED traffic from
+   * the other peer.
+   *
    * We will not yet send traffic, as this might have been a replay.
    * The other (initiating) peer should send a CHANNEL_OPEN next
-   * anyway.
+   * anyway, and then we are in business!
    */
-  CADET_TUNNEL_KEY_PING,
+  CADET_TUNNEL_KEY_AX_AUTH_SENT,
 
   /**
    * Handshake completed: session key available.
    */
-  CADET_TUNNEL_KEY_OK,
+  CADET_TUNNEL_KEY_OK
 
-  /**
-   * New ephemeral key and ping sent, waiting for pong. Unlike KEY_PING,
-   * we still have a valid session key and therefore we *can* still send
-   * traffic on the tunnel.
-   */
-  CADET_TUNNEL_KEY_REKEY
 };
 
 
@@ -112,15 +119,29 @@ GCT_destroy_tunnel_now (struct CadetTunnel *t);
  *
  * @param t a tunnel
  * @param cid connection identifer to use for the connection
+ * @param options options for the connection
  * @param path path to use for the connection
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_SYSERR on failure (duplicate connection)
  */
-void
+int
 GCT_add_inbound_connection (struct CadetTunnel *t,
                             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                            enum GNUNET_CADET_ChannelOption options,
                             struct CadetPeerPath *path);
 
 
 /**
+ * We lost a connection, remove it from our list and clean up
+ * the connection object itself.
+ *
+ * @param ct binding of connection to tunnel of the connection that was lost.
+ */
+void
+GCT_connection_lost (struct CadetTConnection *ct);
+
+
+/**
  * Return the peer to which this tunnel goes.
  *
  * @param t a tunnel
@@ -181,6 +202,19 @@ GCT_send_channel_destroy (struct CadetTunnel *t,
 
 
 /**
+ * Function called when a transmission requested using #GCT_send is done.
+ *
+ * @param cls closure
+ * @param ctn identifier of the connection used for transmission, NULL if
+ *            the transmission failed (to be used to match ACKs to the
+ *            respective connection for connection performance evaluation)
+ */
+typedef void
+(*GCT_SendContinuation)(void *cls,
+                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
  * Sends an already built message on a tunnel, encrypting it and
  * choosing the best connection if not provided.
  *
@@ -188,12 +222,12 @@ GCT_send_channel_destroy (struct CadetTunnel *t,
  * @param t Tunnel on which this message is transmitted.
  * @param cont Continuation to call once message is really sent.
  * @param cont_cls Closure for @c cont.
- * @return Handle to cancel message. NULL if @c cont is NULL.
+ * @return Handle to cancel message.
  */
 struct CadetTunnelQueueEntry *
 GCT_send (struct CadetTunnel *t,
           const struct GNUNET_MessageHeader *message,
-          GNUNET_SCHEDULER_TaskCallback cont,
+          GCT_SendContinuation cont,
           void *cont_cls);
 
 
@@ -227,18 +261,18 @@ GCT_count_channels (struct CadetTunnel *t);
  * @return number of connections available for the tunnel
  */
 unsigned int
-GCT_count_any_connections (struct CadetTunnel *t);
+GCT_count_any_connections (const struct CadetTunnel *t);
 
 
 /**
  * Iterator over connections.
  *
  * @param cls closure
- * @param c one of the connections
+ * @param ct one of the connections
  */
 typedef void
 (*GCT_ConnectionIterator) (void *cls,
-                           struct CadetConnection *c);
+                           struct CadetTConnection *ct);
 
 
 /**
@@ -301,6 +335,17 @@ GCT_handle_kx (struct CadetTConnection *ct,
 
 
 /**
+ * Handle KX_AUTH message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx_auth (struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
+
+
+/**
  * Handle encrypted message.
  *
  * @param ct connection/tunnel combo that received encrypted message
diff --git a/src/cadet/test_cadet.c b/src/cadet/test_cadet.c
index 3a1042eba..72df2203c 100644
--- a/src/cadet/test_cadet.c
+++ b/src/cadet/test_cadet.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
+     Copyright (C) 2011, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,8 +19,9 @@
 */
 /**
  * @file cadet/test_cadet.c
- *
- * @brief Test for the cadet service: retransmission of traffic.
+ * @author Bart Polot
+ * @author Christian Grothoff
+ * @brief Test for the cadet service using mq API.
  */
 #include <stdio.h>
 #include "platform.h"
@@ -31,9 +32,20 @@
 
 
 /**
- * How many messages to send
+ * Ugly workaround to unify data handlers on incoming and outgoing channels.
+ */
+struct CadetTestChannelWrapper
+{
+  /**
+   * Channel pointer.
+   */
+  struct GNUNET_CADET_Channel *ch;
+};
+
+/**
+ * How many messages to send by default.
  */
-#define TOTAL_PACKETS 500 /* Cannot exceed 64k! */
+#define TOTAL_PACKETS 500       /* Cannot exceed 64k! */
 
 /**
  * How long until we give up on connecting the peers?
@@ -41,7 +53,7 @@
 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120)
 
 /**
- * Time to wait for stuff that should be rather fast
+ * Time to wait by default  for stuff that should be rather fast.
  */
 #define SHORT_TIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 20)
 
@@ -64,7 +76,7 @@ static int test;
 /**
  * String with test name
  */
-char *test_name;
+static char *test_name;
 
 /**
  * Flag to send traffic leaf->root in speed tests to test BCK_ACK logic.
@@ -72,6 +84,16 @@ char *test_name;
 static int test_backwards = GNUNET_NO;
 
 /**
+ * How many packets to send.
+ */
+static unsigned int total_packets;
+
+/**
+ * Time to wait for fast operations.
+ */
+static struct GNUNET_TIME_Relative short_time;
+
+/**
  * How many events have happened
  */
 static int ok;
@@ -79,32 +101,32 @@ static int ok;
 /**
  * Number of events expected to conclude the test successfully.
  */
-int ok_goal;
+static int ok_goal;
 
 /**
- * Size of each test packet
+ * Size of each test packet's payload
  */
-size_t size_payload = sizeof (struct GNUNET_MessageHeader) + sizeof (uint32_t);
+static size_t size_payload = sizeof (uint32_t);
 
 /**
  * Operation to get peer ids.
  */
-struct GNUNET_TESTBED_Operation *t_op[2];
+static struct GNUNET_TESTBED_Operation *t_op[2];
 
 /**
  * Peer ids.
  */
-struct GNUNET_PeerIdentity *p_id[2];
+static struct GNUNET_PeerIdentity *p_id[2];
 
 /**
  * Port ID
  */
-struct GNUNET_HashCode port;
+static struct GNUNET_HashCode port;
 
 /**
  * Peer ids counter.
  */
-unsigned int p_ids;
+static unsigned int p_ids;
 
 /**
  * Is the setup initialized?
@@ -157,9 +179,9 @@ static struct GNUNET_SCHEDULER_Task *disconnect_task;
 static struct GNUNET_SCHEDULER_Task *test_task;
 
 /**
- * Task runnining #data_task().
+ * Task runnining #send_next_msg().
  */
-static struct GNUNET_SCHEDULER_Task *data_job;
+static struct GNUNET_SCHEDULER_Task *send_next_msg_task;
 
 /**
  * Cadet handle for the root peer
@@ -174,7 +196,7 @@ static struct GNUNET_CADET_Handle *h2;
 /**
  * Channel handle for the root peer
  */
-static struct GNUNET_CADET_Channel *ch;
+static struct GNUNET_CADET_Channel *outgoing_ch;
 
 /**
  * Channel handle for the dest peer
@@ -182,17 +204,6 @@ static struct GNUNET_CADET_Channel *ch;
 static struct GNUNET_CADET_Channel *incoming_ch;
 
 /**
- * Transmit handle for root data calls
- */
-static struct GNUNET_CADET_TransmitHandle *th;
-
-/**
- * Transmit handle for root data calls
- */
-static struct GNUNET_CADET_TransmitHandle *incoming_th;
-
-
-/**
  * Time we started the data transmission (after channel has been established
  * and initilized).
  */
@@ -218,21 +229,32 @@ static unsigned int ka_sent;
  */
 static unsigned int ka_received;
 
+/**
+ * How many messages were dropped by CADET because of full buffers?
+ */
+static unsigned int msg_dropped;
+
+
+/******************************************************************************/
+
+
+/******************************************************************************/
+
 
 /**
- * Get the client number considered as the "target" or "receiver", depending on
+ * Get the channel considered as the "target" or "receiver", depending on
  * the test type and size.
  *
- * @return Peer # of the target client, either 0 (for backward tests) or
- *         the last peer in the line (for other tests).
+ * @return Channel handle of the target client, either 0 (for backward tests)
+ *         or the last peer in the line (for other tests).
  */
-static unsigned int
-get_expected_target ()
+static struct GNUNET_CADET_Channel *
+get_target_channel ()
 {
   if (SPEED == test && GNUNET_YES == test_backwards)
-    return 0;
+    return outgoing_ch;
   else
-    return peers_requested - 1;
+    return incoming_ch;
 }
 
 
@@ -245,18 +267,15 @@ show_end_data (void)
   static struct GNUNET_TIME_Absolute end_time;
   static struct GNUNET_TIME_Relative total_time;
 
-  end_time = GNUNET_TIME_absolute_get();
-  total_time = GNUNET_TIME_absolute_get_difference(start_time, end_time);
+  end_time = GNUNET_TIME_absolute_get ();
+  total_time = GNUNET_TIME_absolute_get_difference (start_time, end_time);
   FPRINTF (stderr, "\nResults of test \"%s\"\n", test_name);
   FPRINTF (stderr, "Test time %s\n",
-           GNUNET_STRINGS_relative_time_to_string (total_time,
-                                                   GNUNET_YES));
-  FPRINTF (stderr, "Test bandwidth: %f kb/s\n",
-           4 * TOTAL_PACKETS * 1.0 / (total_time.rel_value_us / 1000)); // 4bytes * ms
-  FPRINTF (stderr, "Test throughput: %f packets/s\n\n",
-           TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000)); // packets * ms
+           GNUNET_STRINGS_relative_time_to_string (total_time, GNUNET_YES));
+  FPRINTF (stderr, "Test bandwidth: %f kb/s\n", 4 * total_packets * 1.0 / (total_time.rel_value_us / 1000));    // 4bytes * ms
+  FPRINTF (stderr, "Test throughput: %f packets/s\n\n", total_packets * 1000.0 / (total_time.rel_value_us / 1000));     // packets * ms
   GAUGER ("CADET", test_name,
-          TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000),
+          total_packets * 1000.0 / (total_time.rel_value_us / 1000),
           "packets/s");
 }
 
@@ -275,29 +294,19 @@ disconnect_cadet_peers (void *cls)
 
   disconnect_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "disconnecting cadet service of peers, called from line %ld\n",
-              line);
+              "disconnecting cadet service of peers, called from line %ld\n",
+              line);
   for (i = 0; i < 2; i++)
   {
     GNUNET_TESTBED_operation_done (t_op[i]);
   }
-  if (NULL != ch)
+  if (NULL != outgoing_ch)
   {
-    if (NULL != th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-      th = NULL;
-    }
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
+    GNUNET_CADET_channel_destroy (outgoing_ch);
+    outgoing_ch = NULL;
   }
   if (NULL != incoming_ch)
   {
-    if (NULL != incoming_th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (incoming_th);
-      incoming_th = NULL;
-    }
     GNUNET_CADET_channel_destroy (incoming_ch);
     incoming_ch = NULL;
   }
@@ -316,10 +325,10 @@ static void
 shutdown_task (void *cls)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Ending test.\n");
-  if (NULL != data_job)
+  if (NULL != send_next_msg_task)
   {
-    GNUNET_SCHEDULER_cancel (data_job);
-    data_job = NULL;
+    GNUNET_SCHEDULER_cancel (send_next_msg_task);
+    send_next_msg_task = NULL;
   }
   if (NULL != test_task)
   {
@@ -329,8 +338,8 @@ shutdown_task (void *cls)
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                                (void *) __LINE__);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) __LINE__);
   }
 }
 
@@ -349,14 +358,16 @@ stats_cont (void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " KA sent: %u, KA received: %u\n",
               ka_sent, ka_received);
-  if (KEEPALIVE == test && (ka_sent < 2 || ka_sent > ka_received + 1))
+  if ((KEEPALIVE == test) && ((ka_sent < 2) || (ka_sent > ka_received + 1)))
+  {
+    GNUNET_break (0);
     ok--;
+  }
   GNUNET_TESTBED_operation_done (stats_op);
 
   if (NULL != disconnect_task)
     GNUNET_SCHEDULER_cancel (disconnect_task);
-  disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                              cls);
+  disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, cls);
 }
 
 
@@ -372,29 +383,27 @@ stats_cont (void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
  * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
  */
 static int
-stats_iterator (void *cls,
-                const struct GNUNET_TESTBED_Peer *peer,
-                const char *subsystem,
-                const char *name,
-                uint64_t value,
+stats_iterator (void *cls, const struct GNUNET_TESTBED_Peer *peer,
+                const char *subsystem, const char *name, uint64_t value,
                 int is_persistent)
 {
   static const char *s_sent = "# keepalives sent";
   static const char *s_recv = "# keepalives received";
+  static const char *rdrops = "# messages dropped due to full buffer";
+  static const char *cdrops = "# messages dropped due to slow client";
   uint32_t i;
 
   i = GNUNET_TESTBED_get_index (peer);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "STATS PEER %u - %s [%s]: %llu\n",
-              i,
-              subsystem,
-              name,
-              (unsigned long long) value);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "STATS PEER %u - %s [%s]: %llu\n", i,
+              subsystem, name, (unsigned long long) value);
   if (0 == strncmp (s_sent, name, strlen (s_sent)) && 0 == i)
     ka_sent = value;
-
-  if (0 == strncmp(s_recv, name, strlen (s_recv)) && peers_requested - 1 == i)
+  if (0 == strncmp (s_recv, name, strlen (s_recv)) && peers_requested - 1 == i)
     ka_received = value;
+  if (0 == strncmp (rdrops, name, strlen (rdrops)))
+    msg_dropped += value;
+  if (0 == strncmp (cdrops, name, strlen (cdrops)))
+    msg_dropped += value;
 
   return GNUNET_OK;
 }
@@ -403,7 +412,7 @@ stats_iterator (void *cls,
 /**
  * Task to gather all statistics.
  *
- * @param cls Closure (NULL).
+ * @param cls Closure (line from which the task was scheduled).
  */
 static void
 gather_stats_and_exit (void *cls)
@@ -412,21 +421,20 @@ gather_stats_and_exit (void *cls)
 
   disconnect_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "gathering statistics from line %d\n",
-              (int) l);
-  if (NULL != ch)
+              "gathering statistics from line %ld\n",
+              l);
+  if (NULL != outgoing_ch)
   {
-    if (NULL != th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-      th = NULL;
-    }
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
+    GNUNET_CADET_channel_destroy (outgoing_ch);
+    outgoing_ch = NULL;
   }
-  stats_op = GNUNET_TESTBED_get_statistics (peers_running, testbed_peers,
-                                            "cadet", NULL,
-                                            stats_iterator, stats_cont, cls);
+  stats_op = GNUNET_TESTBED_get_statistics (peers_running,
+                                            testbed_peers,
+                                            "cadet",
+                                            NULL,
+                                            &stats_iterator,
+                                            stats_cont,
+                                            cls);
 }
 
 
@@ -439,165 +447,154 @@ gather_stats_and_exit (void *cls)
 static void
 abort_test (long line)
 {
-  if (disconnect_task != NULL)
+  if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Aborting test from %ld\n", line);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                                (void *) line);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) line);
   }
 }
 
-/**
- * Transmit ready callback.
- *
- * @param cls Closure (message type).
- * @param size Size of the tranmist buffer.
- * @param buf Pointer to the beginning of the buffer.
- *
- * @return Number of bytes written to buf.
- */
-static size_t
-tmt_rdy (void *cls, size_t size, void *buf);
-
 
 /**
- * Task to request a new data transmission.
+ * Send a message on the channel with the appropriate size and payload.
+ *
+ * Update the appropriate *_sent counter.
  *
- * @param cls Closure (peer #).
+ * @param channel Channel to send the message on.
  */
 static void
-data_task (void *cls)
+send_test_message (struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_CADET_Channel *channel;
-  static struct GNUNET_CADET_TransmitHandle **pth;
-  long src;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+  uint32_t *data;
+  int payload;
+  int size;
 
-  data_job = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Data task\n");
-  if (GNUNET_YES == test_backwards)
-  {
-    channel = incoming_ch;
-    pth = &incoming_th;
-    src = peers_requested - 1;
-  }
-  else
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending test message on channel %p\n",
+              channel);
+  size = size_payload;
+  if (GNUNET_NO == initialized)
   {
-    channel = ch;
-    pth = &th;
-    src = 0;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending INITIALIZER\n");
+    size += 1000;
+    payload = data_sent;
+    if (SPEED_ACK == test) // FIXME unify SPEED_ACK with an initializer
+        data_sent++;
   }
-
-  GNUNET_assert (NULL != channel);
-  GNUNET_assert (NULL == *pth);
-
-  *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             size_payload + data_sent,
-                                             &tmt_rdy, (void *) src);
-  if (NULL == *pth)
+  else if (SPEED == test || SPEED_ACK == test)
   {
-    unsigned long i = (unsigned long) cls;
-
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Retransmission\n");
-    if (0 == i)
+    if (get_target_channel() == channel)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "  in 1 ms\n");
-      data_job = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
-                                               &data_task, (void *) 1L);
+      payload = ack_sent;
+      size += ack_sent;
+      ack_sent++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending ACK %u [%d bytes]\n",
+                  payload, size);
     }
     else
     {
-      i++;
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "in %llu ms\n",
-                  (unsigned long long) i);
-      data_job = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                                                              i),
-                                               &data_task, (void *) i);
+      payload = data_sent;
+      size += data_sent;
+      data_sent++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending DATA %u [%d bytes]\n",
+                  data_sent, size);
     }
   }
-}
+  else if (FORWARD == test)
+  {
+    payload = ack_sent;
+  }
+  else if (P2P_SIGNAL == test)
+  {
+    payload = data_sent;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
+  env = GNUNET_MQ_msg_extra (msg, size, GNUNET_MESSAGE_TYPE_DUMMY);
 
+  data = (uint32_t *) &msg[1];
+  *data = htonl (payload);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (channel), env);
+}
 
 /**
- * Transmit ready callback
+ * Task to request a new data transmission in a SPEED test, without waiting
+ * for previous messages to be sent/arrrive.
  *
- * @param cls Closure (peer # which is sending the data).
- * @param size Size of the buffer we have.
- * @param buf Buffer to copy data to.
+ * @param cls Closure (unused).
  */
-static size_t
-tmt_rdy (void *cls, size_t size, void *buf)
+static void
+send_next_msg (void *cls)
 {
-  struct GNUNET_MessageHeader *msg = buf;
-  size_t msg_size;
-  uint32_t *data;
-  long id = (long) cls;
-  unsigned int counter;
+  struct GNUNET_CADET_Channel *channel;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "tmt_rdy on %ld, filling buffer\n",
-              id);
-  if (0 == id)
-    th = NULL;
-  else if ((peers_requested - 1) == id)
-    incoming_th = NULL;
-  else
-    GNUNET_assert (0);
-  counter = get_expected_target () == id ? ack_sent : data_sent;
-  msg_size = size_payload + counter;
-  GNUNET_assert (msg_size > sizeof (struct GNUNET_MessageHeader));
-  if ( (size < msg_size) ||
-       (NULL == buf) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "size %u, buf %p, data_sent %u, ack_received %u\n",
-                (unsigned int) size,
-                buf,
-                data_sent,
-                ack_received);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ok %u, ok goal %u\n", ok, ok_goal);
-    GNUNET_break (ok >= ok_goal - 2);
-
-    return 0;
-  }
-  msg->size = htons (msg_size);
-  msg->type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
-  data = (uint32_t *) &msg[1];
-  *data = htonl (counter);
-  if (GNUNET_NO == initialized)
+  send_next_msg_task = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending next message: %d\n", data_sent);
+
+  channel = GNUNET_YES == test_backwards ? incoming_ch : outgoing_ch;
+  GNUNET_assert (NULL != channel);
+  GNUNET_assert (SPEED == test);
+  send_test_message (channel);
+  if (data_sent < total_packets)
   {
+    /* SPEED test: Send all messages as soon as possible */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "sending initializer\n");
-    msg_size = size_payload + 1000;
-    msg->size = htons (msg_size);
-  if (SPEED_ACK == test)
-      data_sent++;
+                "Scheduling message %d\n",
+                data_sent + 1);
+    send_next_msg_task =
+        GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_UNIT_SECONDS,
+                                     &send_next_msg,
+                                     NULL);
   }
-  else if ( (SPEED == test) ||
-            (SPEED_ACK == test) )
+}
+
+
+/**
+ * Every few messages cancel the timeout task and re-schedule it again, to
+ * avoid timing out when traffic keeps coming.
+ *
+ * @param line Code line number to log if a timeout occurs.
+ */
+static void
+reschedule_timeout_task (long line)
+{
+  if ((ok % 10) == 0)
   {
-    if (get_expected_target() == id)
-      ack_sent++;
-    else
-      data_sent++;
-    counter++;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                " Sent message %u size %u\n",
-                counter,
-                (unsigned int) msg_size);
-    if ( (data_sent < TOTAL_PACKETS) &&
-         (SPEED == test) )
+    if (NULL != disconnect_task)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  " Scheduling message %d\n",
-                  counter + 1);
-      data_job = GNUNET_SCHEDULER_add_now (&data_task, NULL);
+                  " reschedule timeout every 10 messages\n");
+      GNUNET_SCHEDULER_cancel (disconnect_task);
+      disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
+                                                      &gather_stats_and_exit,
+                                                      (void *) line);
     }
   }
+}
 
-  return msg_size;
+
+/**
+ * Check if payload is sane (size contains payload).
+ *
+ * @param cls should match #ch
+ * @param message The actual message.
+ * @return #GNUNET_OK to keep the channel open,
+ *         #GNUNET_SYSERR to close it (signal serious error).
+ */
+static int
+check_data (void *cls, const struct GNUNET_MessageHeader *message)
+{
+  if (sizeof (struct GNUNET_MessageHeader) >= ntohs (message->size))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;             /* all is well-formed */
 }
 
 
@@ -605,75 +602,50 @@ tmt_rdy (void *cls, size_t size, void *buf)
  * Function is called whenever a message is received.
  *
  * @param cls closure (set from GNUNET_CADET_connect(), peer number)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
  * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
  */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
+static void
+handle_data (void *cls, const struct GNUNET_MessageHeader *message)
 {
-  struct GNUNET_CADET_TransmitHandle **pth;
-  long client = (long) cls;
-  long expected_target_client;
+  struct CadetTestChannelWrapper *ch = cls;
+  struct GNUNET_CADET_Channel *channel = ch->ch;
   uint32_t *data;
   uint32_t payload;
-  unsigned int counter;
+  int *counter;
 
   ok++;
-  counter = get_expected_target () == client ? data_received : ack_received;
-
   GNUNET_CADET_receive_done (channel);
+  counter = get_target_channel () == channel ? &data_received : &ack_received;
 
-  if ((ok % 10) == 0)
+  reschedule_timeout_task ((long) __LINE__);
+
+  if (channel == outgoing_ch)
   {
-    if (NULL != disconnect_task)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  " reschedule timeout\n");
-      GNUNET_SCHEDULER_cancel (disconnect_task);
-      disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                                      &gather_stats_and_exit,
-                                                      (void *) __LINE__);
-    }
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Root client got a message.\n");
   }
-
-  switch (client)
+  else if (channel == incoming_ch)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Leaf client got a message.\n");
+  }
+  else
   {
-  case 0L:
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Root client got a message!\n");
-    GNUNET_assert (channel == ch);
-    pth = &th;
-    break;
-  case 1L:
-  case 4L:
-    GNUNET_assert (client == peers_requested - 1);
-    GNUNET_assert (channel == incoming_ch);
-    pth = &incoming_th;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Leaf client %ld got a message.\n",
-                client);
-    break;
-  default:
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Client %ld not valid.\n", client);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Unknown channel %p.\n", channel);
     GNUNET_assert (0);
   }
+
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: (%d/%d)\n", ok, ok_goal);
   data = (uint32_t *) &message[1];
   payload = ntohl (*data);
-  if (payload == counter)
+  if (payload == *counter)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO, " payload as expected: %u\n", payload);
   }
   else
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, " payload %u, expected: %u\n",
-                payload, counter);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                " payload %u, expected: %u\n",
+                payload, *counter);
   }
-  expected_target_client = get_expected_target ();
 
   if (GNUNET_NO == initialized)
   {
@@ -681,188 +653,152 @@ data_callback (void *cls,
     start_time = GNUNET_TIME_absolute_get ();
     if (SPEED == test)
     {
-      GNUNET_assert (peers_requested - 1 == client);
-      data_job = GNUNET_SCHEDULER_add_now (&data_task, NULL);
-      return GNUNET_OK;
+      GNUNET_assert (incoming_ch == channel);
+      send_next_msg_task = GNUNET_SCHEDULER_add_now (&send_next_msg, NULL);
+      return;
     }
   }
 
-  counter++;
-  if (client == expected_target_client) /* Normally 4 */
+  (*counter)++;
+  if (get_target_channel () == channel) /* Got "data" */
   {
-    data_received++;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received data %u\n", data_received);
     if (SPEED != test || (ok_goal - 2) == ok)
     {
       /* Send ACK */
-      GNUNET_assert (NULL == *pth);
-      *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                                 GNUNET_TIME_UNIT_FOREVER_REL,
-                                                 size_payload + ack_sent,
-                                                 &tmt_rdy, (void *) client);
-      return GNUNET_OK;
+      send_test_message (channel);
+      return;
     }
     else
     {
-      if (data_received < TOTAL_PACKETS)
-        return GNUNET_OK;
+      if (data_received < total_packets)
+        return;
     }
   }
-  else /* Normally 0 */
+  else /* Got "ack" */
   {
     if (SPEED_ACK == test || SPEED == test)
     {
-      ack_received++;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received ack %u\n", ack_received);
-      /* send more data */
-      GNUNET_assert (NULL == *pth);
-      *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                                 GNUNET_TIME_UNIT_FOREVER_REL,
-                                                 size_payload + data_sent,
-                                                 &tmt_rdy, (void *) client);
-      if (ack_received < TOTAL_PACKETS && SPEED != test)
-        return GNUNET_OK;
+      /* Send more data */
+      send_test_message (channel);
+      if (ack_received < total_packets && SPEED != test)
+        return;
       if (ok == 2 && SPEED == test)
-        return GNUNET_OK;
-      show_end_data();
+        return;
+      show_end_data ();
     }
     if (test == P2P_SIGNAL)
     {
-      if (NULL != incoming_th)
-      {
-        GNUNET_CADET_notify_transmit_ready_cancel (incoming_th);
-        incoming_th = NULL;
-      }
       GNUNET_CADET_channel_destroy (incoming_ch);
       incoming_ch = NULL;
     }
     else
     {
-      if (NULL != th)
-      {
-        GNUNET_CADET_notify_transmit_ready_cancel (th);
-        th = NULL;
-      }
-      GNUNET_CADET_channel_destroy (ch);
-      ch = NULL;
+      GNUNET_CADET_channel_destroy (outgoing_ch);
+      outgoing_ch = NULL;
     }
   }
-
-  return GNUNET_OK;
 }
 
 
 /**
- * Data handlers for every message type of CADET's payload.
- * {callback_function, message_type, size_expected}
- */
-static struct GNUNET_CADET_MessageHandler handlers[] = {
-  {&data_callback,
-   GNUNET_MESSAGE_TYPE_DUMMY,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
+ * Method called whenever a peer connects to a port in MQ-based CADET.
  *
- * @param cls Closure.
+ * @param cls Closure from #GNUNET_CADET_open_port (peer # as long).
  * @param channel New handle to the channel.
- * @param initiator Peer that started the channel.
- * @param port Port this channel is connected to.
- * @param options channel option flags
- * @return Initial channel context for the channel
- *         (can be NULL -- that's not an error).
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
  */
 static void *
-incoming_channel (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  const struct GNUNET_PeerIdentity *initiator,
-                  const struct GNUNET_HashCode *port,
-                  enum GNUNET_CADET_ChannelOption options)
+connect_handler (void *cls, struct GNUNET_CADET_Channel *channel,
+                 const struct GNUNET_PeerIdentity *source)
 {
+  struct CadetTestChannelWrapper *ch;
+  long peer = (long) cls;
+
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Incoming channel from %s to peer %d:%s\n",
-              GNUNET_i2s (initiator),
-              (int) (long) cls, GNUNET_h2s (port));
+              "Incoming channel from %s to %ld: %p\n",
+              GNUNET_i2s (source), peer, channel);
   ok++;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
-  if ((long) cls == peers_requested - 1)
+  if (peer == peers_requested - 1)
   {
     if (NULL != incoming_ch)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Duplicate incoming channel for client %lu\n",
-                  (long) cls);
-      GNUNET_break(0);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Duplicate incoming channel for client %lu\n", (long) cls);
+      GNUNET_assert (0);
     }
     incoming_ch = channel;
   }
   else
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Incoming channel for unknown client %lu\n", (long) cls);
-    GNUNET_break(0);
+                "Incoming channel for unexpected peer #%lu\n", (long) cls);
+    GNUNET_assert (0);
   }
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
+    disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
                                                     &gather_stats_and_exit,
                                                     (void *) __LINE__);
   }
 
-  return NULL;
+  /* TODO: cannot return channel as-is, in order to unify the data handlers */
+  ch = GNUNET_new (struct CadetTestChannelWrapper);
+  ch->ch = channel;
+
+  return ch;
 }
 
 
 /**
- * Function called whenever an inbound channel is destroyed.  Should clean up
- * any associated state.
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
  *
- * @param cls closure (set from GNUNET_CADET_connect, peer number)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
+ *
+ * @param cls Channel closure (channel wrapper).
+ * @param channel Connection to the other end (henceforth invalid).
  */
 static void
-channel_cleaner (void *cls,
-                 const struct GNUNET_CADET_Channel *channel,
-                 void *channel_ctx)
+disconnect_handler (void *cls, const struct GNUNET_CADET_Channel *channel)
 {
-  long i = (long) cls;
+  struct CadetTestChannelWrapper *ch_w = cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Incoming channel disconnected at peer %ld\n",
-              i);
-  if (peers_running - 1 == i)
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Channel disconnected\n");
+  GNUNET_assert (ch_w->ch == channel);
+  if (channel == incoming_ch)
   {
     ok++;
-    GNUNET_break (channel == incoming_ch);
     incoming_ch = NULL;
   }
-  else if (0L == i)
+  else if (outgoing_ch == channel
+  )
   {
     if (P2P_SIGNAL == test)
     {
       ok++;
     }
-    GNUNET_break (channel == ch);
-    ch = NULL;
+    outgoing_ch = NULL;
   }
   else
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Unknown peer! %d\n",
-                (int) i);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unknown channel! %p\n", channel);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
 
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&gather_stats_and_exit,
-                                                (void *) __LINE__);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&gather_stats_and_exit, (void *) __LINE__);
   }
 }
 
@@ -876,13 +812,20 @@ channel_cleaner (void *cls,
  * @param cls Closure (unused).
  */
 static void
-do_test (void *cls)
+start_test (void *cls)
 {
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (data,
+                           GNUNET_MESSAGE_TYPE_DUMMY,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
+  struct CadetTestChannelWrapper *ch;
   enum GNUNET_CADET_ChannelOption flags;
 
   test_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "do_test\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "start_test\n");
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
@@ -896,30 +839,33 @@ do_test (void *cls)
     flags |= GNUNET_CADET_OPTION_RELIABLE;
   }
 
-  ch = GNUNET_CADET_channel_create (h1,
-                                    NULL,
-                                    p_id[1],
-                                    &port,
-                                    flags);
+  ch = GNUNET_new (struct CadetTestChannelWrapper);
+  outgoing_ch = GNUNET_CADET_channel_create (h1,
+                                             ch,
+                                             p_id[1],
+                                             &port,
+                                             flags,
+                                             NULL,
+                                             &disconnect_handler,
+                                             handlers);
 
-  disconnect_task
-    = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                    &gather_stats_and_exit,
-                                    (void *) __LINE__);
+  ch->ch = outgoing_ch;
+
+  disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
+                                                  &gather_stats_and_exit,
+                                                  (void *) __LINE__);
   if (KEEPALIVE == test)
-    return; /* Don't send any data. */
+    return;                     /* Don't send any data. */
+
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending data initializer...\n");
   data_received = 0;
   data_sent = 0;
   ack_received = 0;
   ack_sent = 0;
-  th = GNUNET_CADET_notify_transmit_ready (ch,
-                                           GNUNET_NO,
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           size_payload + 1000,
-                                           &tmt_rdy, (void *) 0L);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending data initializer on channel %p...\n",
+              outgoing_ch);
+  send_test_message (outgoing_ch);
 }
 
 
@@ -933,35 +879,26 @@ do_test (void *cls)
  *             NULL if the operation is successfull
  */
 static void
-pi_cb (void *cls,
-       struct GNUNET_TESTBED_Operation *op,
-       const struct GNUNET_TESTBED_PeerInformation *pinfo,
-       const char *emsg)
+pi_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
+       const struct GNUNET_TESTBED_PeerInformation *pinfo, const char *emsg)
 {
   long i = (long) cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "id callback for %ld\n", i);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ID callback for %ld\n", i);
 
-  if ( (NULL == pinfo) ||
-       (NULL != emsg) )
+  if ((NULL == pinfo) || (NULL != emsg))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "pi_cb: %s\n", emsg);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "pi_cb: %s\n", emsg);
     abort_test (__LINE__);
     return;
   }
   p_id[i] = pinfo->result.id;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "  id: %s\n", GNUNET_i2s (p_id[i]));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  id: %s\n", GNUNET_i2s (p_id[i]));
   p_ids++;
   if (p_ids < 2)
     return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got all IDs, starting test\n");
-  test_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                            &do_test,
-                                            NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got all IDs, starting test\n");
+  test_task = GNUNET_SCHEDULER_add_now (&start_test, NULL);
 }
 
 
@@ -972,7 +909,7 @@ pi_cb (void *cls,
  * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
  * @param num_peers Number of peers that are running.
  * @param peers Array of peers.
- * @param cadetes Handle to each of the CADETs of the peers.
+ * @param cadets Handle to each of the CADETs of the peers.
  */
 static void
 tmain (void *cls,
@@ -989,16 +926,18 @@ tmain (void *cls,
   testbed_peers = peers;
   h1 = cadets[0];
   h2 = cadets[num_peers - 1];
-  disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
+  disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
                                                   &disconnect_cadet_peers,
                                                   (void *) __LINE__);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
   t_op[0] = GNUNET_TESTBED_peer_get_information (peers[0],
                                                  GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb, (void *) 0L);
+                                                 &pi_cb,
+                                                 (void *) 0L);
   t_op[1] = GNUNET_TESTBED_peer_get_information (peers[num_peers - 1],
                                                  GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb, (void *) 1L);
+                                                 &pi_cb,
+                                                 (void *) 1L);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "requested peer ids\n");
 }
 
@@ -1009,16 +948,46 @@ tmain (void *cls,
 int
 main (int argc, char *argv[])
 {
-  initialized = GNUNET_NO;
   static const struct GNUNET_HashCode *ports[2];
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (data,
+                           GNUNET_MESSAGE_TYPE_DUMMY,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
   const char *config_file;
   char port_id[] = "test port";
-  GNUNET_CRYPTO_hash (port_id, sizeof (port_id), &port);
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "time",
+                                            "short_time",
+                                            gettext_noop ("set short timeout"),
+                                            &short_time),
+
+    GNUNET_GETOPT_option_uint ('m',
+                                   "messages",
+                                   "NUM_MESSAGES",
+                                   gettext_noop ("set number of messages to send"),
+                                   &total_packets),
 
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
+  initialized = GNUNET_NO;
   GNUNET_log_setup ("test", "DEBUG", NULL);
-  config_file = "test_cadet.conf";
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start\n");
+  total_packets = TOTAL_PACKETS;
+  short_time = SHORT_TIME;
+  if (-1 == GNUNET_GETOPT_run (argv[0], options, argc, argv))
+  {
+    FPRINTF (stderr, "test failed: problem with CLI parameters\n");
+    exit (1);
+  }
+
+  config_file = "test_cadet.conf";
+  GNUNET_CRYPTO_hash (port_id, sizeof (port_id), &port);
 
   /* Find out requested size */
   if (strstr (argv[0], "_2_") != NULL)
@@ -1056,11 +1025,11 @@ main (int argc, char *argv[])
   {
     /* Test is supposed to generate the following callbacks:
      * 1 incoming channel (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
-     * TOTAL_PACKETS received data packet (@orig)
+     * total_packets received data packet (@dest)
+     * total_packets received data packet (@orig)
      * 1 received channel destroy (@dest)
      */
-    ok_goal = TOTAL_PACKETS * 2 + 2;
+    ok_goal = total_packets * 2 + 2;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED_ACK\n");
     test = SPEED_ACK;
     test_name = "speed ack";
@@ -1070,11 +1039,11 @@ main (int argc, char *argv[])
     /* Test is supposed to generate the following callbacks:
      * 1 incoming channel (@dest)
      * 1 initial packet (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
+     * total_packets received data packet (@dest)
      * 1 received data packet (@orig)
      * 1 received channel destroy (@dest)
      */
-    ok_goal = TOTAL_PACKETS + 4;
+    ok_goal = total_packets + 4;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED\n");
     if (strstr (argv[0], "_reliable") != NULL)
     {
@@ -1115,20 +1084,22 @@ main (int argc, char *argv[])
   p_ids = 0;
   ports[0] = &port;
   ports[1] = NULL;
-  GNUNET_CADET_TEST_run ("test_cadet_small",
-                        config_file,
-                        peers_requested,
-                        &tmain,
-                        NULL, /* tmain cls */
-                        &incoming_channel,
-                        &channel_cleaner,
-                        handlers,
-                        ports);
-
-  if (ok_goal > ok)
+  GNUNET_CADET_TEST_ruN ("test_cadet_small",
+                         config_file,
+                         peers_requested,
+                         &tmain,
+                         NULL,        /* tmain cls */
+                         &connect_handler,
+                         NULL,
+                         &disconnect_handler,
+                         handlers,
+                         ports);
+  if (NULL != strstr (argv[0], "_reliable"))
+    msg_dropped = 0;            /* dropped should be retransmitted */
+
+  if (ok_goal > ok - msg_dropped)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "FAILED! (%d/%d)\n", ok, ok_goal);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "FAILED! (%d/%d)\n", ok, ok_goal);
     return 1;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "success\n");
diff --git a/src/cadet/test_cadet_local.c b/src/cadet/test_cadet_local.c
deleted file mode 100644
index 2b915ab81..000000000
--- a/src/cadet/test_cadet_local.c
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/test_cadet_local.c
- * @brief test cadet local: test of cadet channels with just one peer
- * @author Bartlomiej Polot
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_testing_lib.h"
-#include "gnunet_cadet_service.h"
-
-struct GNUNET_TESTING_Peer *me;
-
-static struct GNUNET_CADET_Handle *cadet_peer_1;
-
-static struct GNUNET_CADET_Handle *cadet_peer_2;
-
-static struct GNUNET_CADET_Channel *ch;
-
-static int result = GNUNET_OK;
-
-static int got_data = GNUNET_NO;
-
-static struct GNUNET_SCHEDULER_Task *abort_task;
-
-static struct GNUNET_SCHEDULER_Task *connect_task;
-
-static struct GNUNET_CADET_TransmitHandle *mth;
-
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls);
-
-
-/**
- * Shutdown nicely
- */
-static void
-do_shutdown (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "shutdown\n");
-  if (NULL != abort_task)
-  {
-    GNUNET_SCHEDULER_cancel (abort_task);
-    abort_task = NULL;
-  }
-  if (NULL != ch)
-  {
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect client 1\n");
-  if (NULL != cadet_peer_1)
-  {
-    GNUNET_CADET_disconnect (cadet_peer_1);
-    cadet_peer_1 = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect client 2\n");
-  if (NULL != cadet_peer_2)
-  {
-    GNUNET_CADET_disconnect (cadet_peer_2);
-    cadet_peer_2 = NULL;
-  }
-  if (NULL != connect_task)
-  {
-    GNUNET_SCHEDULER_cancel (connect_task);
-    connect_task = NULL;
-  }
-}
-
-
-/**
- * Something went wrong and timed out. Kill everything and set error flag
- */
-static void
-do_abort (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ABORT\n");
-  result = GNUNET_SYSERR;
-  abort_task = NULL;
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Function is called whenever a message is received.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Data callback! Shutting down.\n");
-  got_data = GNUNET_YES;
-  GNUNET_SCHEDULER_shutdown ();
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port port number
- * @param options channel options
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
- */
-static void *
-inbound_channel (void *cls,
-                 struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *initiator,
-                 const struct GNUNET_HashCode *port,
-                 enum GNUNET_CADET_ChannelOption options)
-{
-  long id = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "received incoming channel on peer %d, port %s\n",
-              (int) id,
-              GNUNET_h2s (port));
-  if (id != 2L)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "wrong peer\n");
-    result = GNUNET_SYSERR;
-  }
-  return NULL;
-}
-
-
-/**
- * Function called whenever an channel is destroyed.  Should clean up
- * any associated state.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                    with the channel is stored
- */
-static void
-channel_end (void *cls,
-             const struct GNUNET_CADET_Channel *channel,
-             void *channel_ctx)
-{
-  long id = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "incoming channel closed at peer %ld\n",
-              id);
-  if (NULL != mth)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (mth);
-    mth = NULL;
-  }
-  if (channel == ch)
-    ch = NULL;
-  if (GNUNET_NO == got_data)
-  {
-    if (NULL == connect_task)
-      connect_task
-        = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
-                                                                       2),
-                                        &do_connect,
-                                        NULL);
-  }
-}
-
-
-/**
- * Handler array for traffic received on peer1
- */
-static struct GNUNET_CADET_MessageHandler handlers1[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Handler array for traffic received on peer2 (none expected)
- */
-static struct GNUNET_CADET_MessageHandler handlers2[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Data send callback: fillbuffer with test packet.
- *
- * @param cls Closure (unused).
- * @param size Buffer size.
- * @param buf Buffer to fill.
- *
- * @return size of test packet.
- */
-static size_t
-do_send (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_MessageHeader *m = buf;
-
-  mth = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_break (0);
-    result = GNUNET_SYSERR;
-    return 0;
-  }
-  m->size = htons (sizeof (struct GNUNET_MessageHeader));
-  m->type = htons (1);
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  return sizeof (struct GNUNET_MessageHeader);
-}
-
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls)
-{
-  struct GNUNET_PeerIdentity id;
-
-  connect_task = NULL;
-  GNUNET_TESTING_peer_get_identity (me, &id);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "CONNECT BY PORT\n");
-  ch = GNUNET_CADET_channel_create (cadet_peer_1,
-                                    NULL,
-                                    &id, GC_u2h (1),
-                                    GNUNET_CADET_OPTION_DEFAULT);
-  mth = GNUNET_CADET_notify_transmit_ready (ch, GNUNET_NO,
-                                            GNUNET_TIME_UNIT_FOREVER_REL,
-                                            sizeof (struct GNUNET_MessageHeader),
-                                            &do_send, NULL);
-}
-
-
-/**
- * Initialize framework and start test
- *
- * @param cls Closure (unused).
- * @param cfg Configuration handle.
- * @param peer Testing peer handle.
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *cfg,
-     struct GNUNET_TESTING_Peer *peer)
-{
-  me = peer;
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
-                                 NULL);
-  abort_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_SECONDS, 15),
-                                    &do_abort,
-                                    NULL);
-  cadet_peer_1 = GNUNET_CADET_connect (cfg,       /* configuration */
-                                     (void *) 1L,       /* cls */
-                                     &channel_end,      /* channel end hndlr */
-                                     handlers1); /* traffic handlers */
-  cadet_peer_2 = GNUNET_CADET_connect (cfg,       /* configuration */
-                                     (void *) 2L,     /* cls */
-                                     &channel_end,      /* channel end hndlr */
-                                     handlers2); /* traffic handlers */
-
-  if ( (NULL == cadet_peer_1) ||
-       (NULL == cadet_peer_2) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Couldn't connect to cadet :(\n");
-    result = GNUNET_SYSERR;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  GNUNET_CADET_open_port (cadet_peer_2,
-                          GC_u2h (1),
-                          &inbound_channel,
-                          (void *) 2L);
-  if (NULL == connect_task)
-    connect_task
-      = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
-                                                                     2),
-                                      &do_connect,
-                                      NULL);
-}
-
-
-/**
- * Main
- */
-int
-main (int argc, char *argv[])
-{
-  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
-                                    "test_cadet.conf",
-                                &run, NULL))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "run failed\n");
-    return 2;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Final result: %d\n", result);
-  return (result == GNUNET_OK) ? 0 : 1;
-}
-
-/* end of test_cadet_local_1.c */
diff --git a/src/cadet/test_cadet_local_mq.c b/src/cadet/test_cadet_local_mq.c
new file mode 100644
index 000000000..3089c7fbb
--- /dev/null
+++ b/src/cadet/test_cadet_local_mq.c
@@ -0,0 +1,332 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file cadet/test_cadet_local.c
+ * @brief test cadet local: test of cadet channels with just one peer
+ * @author Bartlomiej Polot
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_dht_service.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_cadet_service.h"
+
+#define TEST_MESSAGE_TYPE 1
+#define TEST_PORT_ID 1
+
+/**
+ * Test message structure.
+ */
+struct GNUNET_CADET_TestMsg
+{
+  /**
+   * Type: #TEST_MESSAGE_TYPE
+   *
+   * Size: sizeof(struct GNUNET_CADET_TestMsg)
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Test payload.
+   */
+  uint64_t payload;
+};
+
+struct GNUNET_TESTING_Peer *me;
+
+static struct GNUNET_CADET_Handle *cadet_peer_1;
+
+static struct GNUNET_CADET_Handle *cadet_peer_2;
+
+static struct GNUNET_CADET_Channel *ch;
+
+static int result = GNUNET_OK;
+
+static int got_data = GNUNET_NO;
+
+static struct GNUNET_SCHEDULER_Task *abort_task;
+
+static struct GNUNET_SCHEDULER_Task *connect_task;
+
+
+/**
+ * Connect to other client and send data
+ *
+ * @param cls Closue (unused).
+ */
+static void
+do_connect (void *cls);
+
+
+/**
+ * Shutdown nicely
+ */
+static void
+do_shutdown (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "shutdown\n");
+  if (NULL != abort_task)
+  {
+    GNUNET_SCHEDULER_cancel (abort_task);
+    abort_task = NULL;
+  }
+  if (NULL != ch)
+  {
+    GNUNET_CADET_channel_destroy (ch);
+    ch = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Disconnect client 1\n");
+  if (NULL != cadet_peer_1)
+  {
+    GNUNET_CADET_disconnect (cadet_peer_1);
+    cadet_peer_1 = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Disconnect client 2\n");
+  if (NULL != cadet_peer_2)
+  {
+    GNUNET_CADET_disconnect (cadet_peer_2);
+    cadet_peer_2 = NULL;
+  }
+  if (NULL != connect_task)
+  {
+    GNUNET_SCHEDULER_cancel (connect_task);
+    connect_task = NULL;
+  }
+}
+
+
+/**
+ * Something went wrong and timed out. Kill everything and set error flag
+ */
+static void
+do_abort (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "ABORT from line %ld\n", (long) cls);
+  result = GNUNET_SYSERR;
+  abort_task = NULL;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+/**
+ * Method called whenever a peer connects to a port in MQ-based CADET.
+ *
+ * @param cls Closure from #GNUNET_CADET_open_port.
+ * @param channel New handle to the channel.
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
+ */
+static void *
+connected (void *cls,
+           struct GNUNET_CADET_Channel *channel,
+           const struct GNUNET_PeerIdentity *source)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "connected %s, cls: %p\n",
+              GNUNET_i2s(source), cls);
+  return channel;
+}
+
+/**
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
+ *
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ */
+static void
+disconnected (void *cls,
+              const struct GNUNET_CADET_Channel *channel)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "disconnected channel %p, cls: %p\n",
+              channel, cls);
+  if (channel == ch)
+    ch = NULL;
+}
+
+
+/**
+ * Handle test data
+ *
+ * @param h     The cadet handle
+ * @param msg   A message with the details of the new incoming channel
+ */
+static void
+handle_data_received (void *cls,
+                      const struct GNUNET_CADET_TestMsg *msg)
+{
+  uint64_t payload;
+
+  payload = GNUNET_ntohll (msg->payload);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Data callback payload %llu with cls: %p! Shutting down.\n",
+              (unsigned long long) payload,
+              cls);
+  GNUNET_assert (42 == payload);
+  got_data = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Signature of the main function of a task.
+ *
+ * @param cls Closure (unused).
+ */
+static void
+message_sent (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "message sent\n");
+}
+
+
+/**
+ * Connect to other client and send data
+ *
+ * @param cls Closure (unused).
+ */
+static void
+do_connect (void *cls)
+{
+  struct GNUNET_PeerIdentity id;
+  struct GNUNET_MQ_Handle *mq;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TestMsg *msg;
+
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (data_received,
+                             TEST_MESSAGE_TYPE,
+                             struct GNUNET_CADET_TestMsg,
+                             cadet_peer_1),
+    GNUNET_MQ_handler_end ()
+  };
+
+  connect_task = NULL;
+  GNUNET_TESTING_peer_get_identity (me, &id);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "creating channel\n");
+  ch = GNUNET_CADET_channel_create (cadet_peer_1, /* cadet handle */
+                                    NULL,         /* channel cls */
+                                    &id,          /* destination */
+                                    GC_u2h (TEST_MESSAGE_TYPE), /* port */
+                                    GNUNET_CADET_OPTION_DEFAULT, /* opt */
+                                    NULL,          /* window change */
+                                    &disconnected, /* disconnect handler */
+                                    handlers       /* traffic handlers */
+                                   );
+  env = GNUNET_MQ_msg (msg, TEST_MESSAGE_TYPE);
+  msg->payload = GNUNET_htonll (42);
+  mq = GNUNET_CADET_get_mq (ch);
+  GNUNET_MQ_notify_sent (env, &message_sent, NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "sending message\n");
+  GNUNET_MQ_send (mq, env);
+}
+
+
+/**
+ * Initialize framework and start test
+ *
+ * @param cls Closure (unused).
+ * @param cfg Configuration handle.
+ * @param peer Testing peer handle.
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (data_received,
+                             TEST_MESSAGE_TYPE,
+                             struct GNUNET_CADET_TestMsg,
+                             cadet_peer_2),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_TIME_Relative delay;
+
+  me = peer;
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
+  delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15);
+  abort_task = GNUNET_SCHEDULER_add_delayed (delay,
+                                             &do_abort,
+                                             (void *) (long) __LINE__);
+  cadet_peer_1 = GNUNET_CADET_connect (cfg);
+  cadet_peer_2 = GNUNET_CADET_connect (cfg);
+
+  if ( (NULL == cadet_peer_1) ||
+       (NULL == cadet_peer_2) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Couldn't connect to cadet\n");
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 1: %p\n", cadet_peer_1);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 2: %p\n", cadet_peer_2);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handlers 2: %p\n", handlers);
+  GNUNET_CADET_open_port (cadet_peer_2,          /* cadet handle */
+                          GC_u2h (TEST_PORT_ID), /* port id */
+                          &connected,            /* connect handler */
+                          (void *) 2L,           /* handle for #connected */
+                          NULL,                  /* window size handler */
+                          &disconnected,         /* disconnect handler */
+                          handlers);             /* traffic handlers */
+  delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2);
+  if (NULL == connect_task)
+    connect_task = GNUNET_SCHEDULER_add_delayed (delay,
+                                                 &do_connect,
+                                                 NULL);
+}
+
+
+/**
+ * Main
+ */
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
+                                    "test_cadet.conf",
+                                &run, NULL))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "run failed\n");
+    return 2;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Final result: %d\n", result);
+  return (result == GNUNET_OK) ? 0 : 1;
+}
+
+/* end of test_cadet_local_1.c */
diff --git a/src/cadet/test_cadet_single.c b/src/cadet/test_cadet_single.c
deleted file mode 100644
index b45b0af5d..000000000
--- a/src/cadet/test_cadet_single.c
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/test_cadet_single.c
- * @brief test cadet single: test of cadet channels with just one client
- * @author Bartlomiej Polot
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_testing_lib.h"
-#include "gnunet_cadet_service.h"
-
-#define REPETITIONS 5
-#define DATA_SIZE 35000
-
-struct GNUNET_TESTING_Peer *me;
-
-static struct GNUNET_CADET_Handle *cadet;
-
-static struct GNUNET_CADET_Channel *ch1;
-
-static struct GNUNET_CADET_Channel *ch2;
-
-static int result;
-
-static struct GNUNET_SCHEDULER_Task *abort_task;
-
-static struct GNUNET_SCHEDULER_Task *connect_task;
-
-static unsigned int repetition;
-
-static struct GNUNET_CADET_TransmitHandle *nth;
-
-static struct GNUNET_CADET_Port *port;
-
-
-/* forward declaration */
-static size_t
-do_send (void *cls, size_t size, void *buf);
-
-
-/**
- * Shutdown nicely
- */
-static void
-do_shutdown (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "shutdown\n");
-  if (NULL != port)
-  {
-    GNUNET_CADET_close_port (port);
-    port = NULL;
-  }
-  if (NULL != nth)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (nth);
-    nth = NULL;
-  }
-  if (NULL != abort_task)
-  {
-    GNUNET_SCHEDULER_cancel (abort_task);
-    abort_task = NULL;
-  }
-  if (NULL != connect_task)
-  {
-    GNUNET_SCHEDULER_cancel (connect_task);
-    connect_task = NULL;
-  }
-  if (NULL != ch1)
-  {
-    GNUNET_CADET_channel_destroy (ch1);
-    ch1 = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect clients\n");
-  if (NULL != cadet)
-  {
-    GNUNET_CADET_disconnect (cadet);
-    cadet = NULL;
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
-}
-
-
-/**
- * Something went wrong and timed out. Kill everything and set error flag
- */
-static void
-do_abort (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ABORT\n");
-  result = GNUNET_SYSERR;
-  abort_task = NULL;
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Function is called whenever a message is received.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Data callback! Repetition %u/%u\n",
-              repetition, REPETITIONS);
-  repetition++;
-  if (repetition < REPETITIONS)
-  {
-    struct GNUNET_CADET_Channel *my_channel;
-    if (0 == repetition % 2)
-      my_channel = ch1;
-    else
-      my_channel = ch2;
-    nth = GNUNET_CADET_notify_transmit_ready (my_channel,
-                                              GNUNET_NO,
-                                              GNUNET_TIME_UNIT_FOREVER_REL,
-                                              sizeof (struct GNUNET_MessageHeader)
-                                              + DATA_SIZE,
-                                              &do_send, NULL);
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "All data OK. Destroying channel.\n");
-  GNUNET_assert (NULL == nth);
-  GNUNET_CADET_channel_destroy (ch1);
-  ch1 = NULL;
-  return GNUNET_OK;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port port number
- * @param options channel option flags
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
- */
-static void *
-inbound_channel (void *cls,
-                 struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *initiator,
-                 const struct GNUNET_HashCode *port,
-                 enum GNUNET_CADET_ChannelOption options)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "received incoming channel on port %s\n",
-              GNUNET_h2s (port));
-  ch2 = channel;
-  return NULL;
-}
-
-
-/**
- * Function called whenever an inbound channel is destroyed.  Should clean up
- * any associated state.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
- */
-static void
-channel_end (void *cls,
-             const struct GNUNET_CADET_Channel *channel,
-             void *channel_ctx)
-{
-  long id = (long) cls;
-
-  nth = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "incoming channel closed at peer %ld\n",
-              id);
-  if ( (REPETITIONS == repetition) &&
-       (channel == ch2) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "everything fine! finishing!\n");
-    result = GNUNET_OK;
-    GNUNET_SCHEDULER_shutdown ();
-  }
-  if (channel == ch2)
-    ch2 = NULL;
-  if (channel == ch1)
-    ch1 = NULL;
-}
-
-
-/**
- * Handler array for traffic received on peer1
- */
-static struct GNUNET_CADET_MessageHandler handlers1[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Data send callback: fillbuffer with test packet.
- *
- * @param cls Closure (unused).
- * @param size Buffer size.
- * @param buf Buffer to fill.
- * @return size of test packet.
- */
-static size_t
-do_send (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_MessageHeader *m = buf;
-
-  nth = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_break (0);
-    result = GNUNET_SYSERR;
-    return 0;
-  }
-  m->size = htons (sizeof (struct GNUNET_MessageHeader) + DATA_SIZE);
-  m->type = htons (1);
-  memset (&m[1], 0, DATA_SIZE);
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader) + DATA_SIZE);
-  return sizeof (struct GNUNET_MessageHeader) + DATA_SIZE;
-}
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls)
-{
-  struct GNUNET_PeerIdentity id;
-  size_t size = sizeof (struct GNUNET_MessageHeader) + DATA_SIZE;
-
-  connect_task = NULL;
-  GNUNET_TESTING_peer_get_identity (me, &id);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CONNECT BY PORT\n");
-  ch1 = GNUNET_CADET_channel_create (cadet, NULL, &id, GC_u2h (1),
-                                     GNUNET_CADET_OPTION_DEFAULT);
-  nth = GNUNET_CADET_notify_transmit_ready (ch1,
-                                            GNUNET_NO,
-                                            GNUNET_TIME_UNIT_FOREVER_REL,
-                                            size,
-                                            &do_send,
-                                            NULL);
-}
-
-
-/**
- * Initialize framework and start test
- *
- * @param cls Closure (unused).
- * @param cfg Configuration handle.
- * @param peer Testing peer handle.
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *cfg,
-     struct GNUNET_TESTING_Peer *peer)
-{
-  me = peer;
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
-  abort_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_SECONDS, 15), &do_abort,
-                                    NULL);
-  cadet = GNUNET_CADET_connect (cfg,       /* configuration */
-                              (void *) 1L,     /* cls */
-                              &channel_end,      /* inbound end hndlr */
-                              handlers1); /* traffic handlers */
-  port = GNUNET_CADET_open_port (cadet,
-                                 GC_u2h (1),
-                                 &inbound_channel,
-                                 (void *) 1L);
-
-
-  if (NULL == cadet)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Couldn't connect to cadet :(\n");
-    result = GNUNET_SYSERR;
-    return;
-  }
-  connect_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                    &do_connect,
-                                    NULL);
-}
-
-
-/**
- * Main
- */
-int
-main (int argc,
-      char *argv[])
-{
-  result = GNUNET_NO;
-  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
-                                    "test_cadet.conf",
-                                    &run, NULL))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "run failed\n");
-    return 2;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Final result: %d\n",
-              result);
-  return (result == GNUNET_OK) ? 0 : 1;
-}
-
-/* end of test_cadet_single.c */
diff --git a/src/consensus/.gitignore b/src/consensus/.gitignore
index 8bb7caafe..d49147d17 100644
--- a/src/consensus/.gitignore
+++ b/src/consensus/.gitignore
@@ -1,3 +1,5 @@
 gnunet-service-evil-consensus
 gnunet-consensus-profiler
 gnunet-service-consensus
+test_consensus_api
+resource.log.master
diff --git a/src/consensus/Makefile.am b/src/consensus/Makefile.am
index c63434f94..c0205ee5d 100644
--- a/src/consensus/Makefile.am
+++ b/src/consensus/Makefile.am
@@ -5,6 +5,8 @@ pkgcfgdir= $(pkgdatadir)/config.d/
 
 libexecdir= $(pkglibdir)/libexec/
 
+plugindir = $(libdir)/gnunet
+
 pkgcfg_DATA = \
   consensus.conf
 
@@ -16,8 +18,6 @@ if USE_COVERAGE
   AM_CFLAGS = -fprofile-arcs -ftest-coverage
 endif
 
-bin_PROGRAMS = \
- gnunet-consensus-profiler
 
 libexec_PROGRAMS = \
  gnunet-service-consensus
@@ -67,13 +67,34 @@ libgnunetconsensus_la_LIBADD = \
 libgnunetconsensus_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS)
 
+
+plugin_LTLIBRARIES = \
+  libgnunet_plugin_block_consensus.la
+
+libgnunet_plugin_block_consensus_la_SOURCES = \
+  plugin_block_consensus.c
+libgnunet_plugin_block_consensus_la_LIBADD = \
+  $(top_builddir)/src/block/libgnunetblock.la \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(LTLIBINTL)
+libgnunet_plugin_block_consensus_la_LDFLAGS = \
+ $(GN_PLUGIN_LDFLAGS)
+
+
+
+if HAVE_TESTING
+bin_PROGRAMS = \
+ gnunet-consensus-profiler
+
 check_PROGRAMS = \
  test_consensus_api
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
+endif
 
 test_consensus_api_SOURCES = \
  test_consensus_api.c
diff --git a/src/consensus/consensus_protocol.h b/src/consensus/consensus_protocol.h
index fb3bde628..f2933ed6f 100644
--- a/src/consensus/consensus_protocol.h
+++ b/src/consensus/consensus_protocol.h
@@ -29,6 +29,7 @@
 #define GNUNET_CONSENSUS_PROTOCOL_H
 
 #include "platform.h"
+#include "gnunet_util_lib.h"
 #include "gnunet_common.h"
 #include "gnunet_protocols.h"
 
@@ -44,37 +45,37 @@ GNUNET_NETWORK_STRUCT_BEGIN
 struct GNUNET_CONSENSUS_RoundContextMessage
 {
   /**
-   * Type: GNUNET_MESSAGE_TYPE_CONSENSUS_P2P_ROUND_CONTEXT
+   * Type: #GNUNET_MESSAGE_TYPE_CONSENSUS_P2P_ROUND_CONTEXT
    */
   struct GNUNET_MessageHeader header;
 
   /**
    * A value from 'enum PhaseKind'.
    */
-  uint16_t kind;
+  uint16_t kind GNUNET_PACKED;
 
   /**
    * Number of the first peer
    * in canonical order.
    */
-  int16_t peer1;
+  int16_t peer1 GNUNET_PACKED;
 
   /**
    * Number of the second peer in canonical order.
    */
-  int16_t peer2;
+  int16_t peer2 GNUNET_PACKED;
 
   /**
    * Repetition of the gradecast phase.
    */
-  int16_t repetition;
+  int16_t repetition GNUNET_PACKED;
 
   /**
    * Leader in the gradecast phase.
    *
    * Can be different from both peer1 and peer2.
    */
-  int16_t leader;
+  int16_t leader GNUNET_PACKED;
 
   /**
    * Non-zero if this set reconciliation
@@ -84,9 +85,51 @@ struct GNUNET_CONSENSUS_RoundContextMessage
    *
    * Ignored for set operations that are not within gradecasts.
    */
-  uint16_t is_contested;
+  uint16_t is_contested GNUNET_PACKED;
 };
 
+
+enum {
+  CONSENSUS_MARKER_CONTESTED = 1,
+  CONSENSUS_MARKER_SIZE = 2,
+};
+
+
+/**
+ * Consensus element, either marker or payload.
+ */
+struct ConsensusElement
+{
+  /**
+   * Payload element_type, only valid
+   * if this is not a marker element.
+   */
+  uint16_t payload_type GNUNET_PACKED;
+
+  /**
+   * Is this a marker element?
+   */
+  uint8_t marker;
+
+  /* rest: element data */
+};
+
+
+struct ConsensusSizeElement
+{
+  struct ConsensusElement ce;
+
+  uint64_t size GNUNET_PACKED;
+  uint8_t sender_index;
+};
+
+struct ConsensusStuffedElement
+{
+  struct ConsensusElement ce;
+  struct GNUNET_HashCode rand GNUNET_PACKED;
+};
+
+
 GNUNET_NETWORK_STRUCT_END
 
 #endif
diff --git a/src/consensus/gnunet-consensus-profiler.c b/src/consensus/gnunet-consensus-profiler.c
index 290263d95..68c2ad594 100644
--- a/src/consensus/gnunet-consensus-profiler.c
+++ b/src/consensus/gnunet-consensus-profiler.c
@@ -55,6 +55,8 @@ static struct GNUNET_HashCode session_id;
 
 static unsigned int peers_done = 0;
 
+static int dist_static;
+
 static unsigned *results_for_peer;
 
 /**
@@ -217,26 +219,45 @@ do_consensus ()
 {
   int unique_indices[replication];
   unsigned int i;
+  unsigned int j;
+  struct GNUNET_HashCode val;
+  struct GNUNET_SET_Element element;
 
-  for (i = 0; i < num_values; i++)
+  if (dist_static)
   {
-    unsigned int j;
-    struct GNUNET_HashCode val;
-    struct GNUNET_SET_Element element;
+    for (i = 0; i < num_values; i++)
+    {
 
-    generate_indices (unique_indices);
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &val);
+      GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &val);
 
-    element.data = &val;
-    element.size = sizeof (val);
-    for (j = 0; j < replication; j++)
+      element.data = &val;
+      element.size = sizeof (val);
+      for (j = 0; j < replication; j++)
+      {
+        GNUNET_CONSENSUS_insert (consensus_handles[j],
+                                 &element,
+                                 NULL, NULL);
+      }
+    }
+  }
+  else
+  {
+    for (i = 0; i < num_values; i++)
     {
-      int cid;
+      generate_indices (unique_indices);
+      GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &val);
+
+      element.data = &val;
+      element.size = sizeof (val);
+      for (j = 0; j < replication; j++)
+      {
+        int cid;
 
-      cid = unique_indices[j];
-      GNUNET_CONSENSUS_insert (consensus_handles[cid],
-                               &element,
-                               NULL, NULL);
+        cid = unique_indices[j];
+        GNUNET_CONSENSUS_insert (consensus_handles[cid],
+                                 &element,
+                                 NULL, NULL);
+      }
     }
   }
 
@@ -494,28 +515,55 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'n', "num-peers", NULL,
-        gettext_noop ("number of peers in consensus"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers },
-      { 'k', "value-replication", NULL,
-        gettext_noop ("how many peers (random selection without replacement) receive one value?"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &replication },
-      { 'x', "num-values", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_values },
-      { 't', "timeout", NULL,
-        gettext_noop ("consensus timeout"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &conclude_timeout },
-      { 'd', "delay", NULL,
-        gettext_noop ("delay until consensus starts"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &consensus_delay },
-      { 's', "statistics", NULL,
-        gettext_noop ("write statistics to file"),
-        GNUNET_YES, &GNUNET_GETOPT_set_filename, &statistics_filename },
-      { 'V', "verbose", NULL,
-        gettext_noop ("be more verbose (print received values)"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &verbose },
+   struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+      GNUNET_GETOPT_option_uint ('n',
+                                     "num-peers",
+                                     NULL,
+                                     gettext_noop ("number of peers in consensus"),
+                                     &num_peers),
+
+      GNUNET_GETOPT_option_uint ('k',
+                                     "value-replication",
+                                     NULL,
+                                     gettext_noop ("how many peers (random selection without replacement) receive one value?"),
+                                     &replication),
+
+      GNUNET_GETOPT_option_uint ('x',
+                                     "num-values",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_values),
+
+      GNUNET_GETOPT_option_relative_time ('t',
+                                              "timeout",
+                                              NULL,
+                                              gettext_noop ("consensus timeout"),
+                                              &conclude_timeout),
+
+
+      GNUNET_GETOPT_option_relative_time ('d',
+                                              "delay",
+                                              NULL,
+                                              gettext_noop ("delay until consensus starts"),
+                                              &consensus_delay),
+
+      GNUNET_GETOPT_option_filename ('s',
+                                     "statistics",
+                                     "FILENAME",
+                                     gettext_noop ("write statistics to file"),
+                                     &statistics_filename),
+
+      GNUNET_GETOPT_option_flag ('S',
+                                    "dist-static",
+                                    gettext_noop ("distribute elements to a static subset of good peers"),
+                                    &dist_static),
+
+      GNUNET_GETOPT_option_flag ('V',
+                                    "verbose",
+                                    gettext_noop ("be more verbose (print received values)"),
+                                    &verbose),
+
       GNUNET_GETOPT_OPTION_END
   };
   conclude_timeout = GNUNET_TIME_UNIT_SECONDS;
diff --git a/src/consensus/gnunet-service-consensus.c b/src/consensus/gnunet-service-consensus.c
index 64decc29e..4af7199aa 100644
--- a/src/consensus/gnunet-service-consensus.c
+++ b/src/consensus/gnunet-service-consensus.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2012, 2013 GNUnet e.V.
+      Copyright (C) 2012, 2013, 2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -26,6 +26,7 @@
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_block_lib.h"
 #include "gnunet_protocols.h"
 #include "gnunet_applications.h"
 #include "gnunet_set_service.h"
@@ -34,8 +35,6 @@
 #include "consensus_protocol.h"
 #include "consensus.h"
 
-#define ELEMENT_TYPE_CONTESTED_MARKER (GNUNET_CONSENSUS_ELEMENT_TYPE_USER_MAX + 1)
-
 
 enum ReferendumVote
 {
@@ -65,11 +64,6 @@ enum EarlyStoppingPhase
 
 GNUNET_NETWORK_STRUCT_BEGIN
 
-
-struct ContestedPayload
-{
-};
-
 /**
  * Tuple of integers that together
  * identify a task uniquely.
@@ -147,6 +141,7 @@ GNUNET_NETWORK_STRUCT_END
 enum PhaseKind
 {
   PHASE_KIND_ALL_TO_ALL,
+  PHASE_KIND_ALL_TO_ALL_2,
   PHASE_KIND_GRADECAST_LEADER,
   PHASE_KIND_GRADECAST_ECHO,
   PHASE_KIND_GRADECAST_ECHO_GRADE,
@@ -398,6 +393,14 @@ struct DiffEntry
   struct GNUNET_CONTAINER_MultiHashMap *changes;
 };
 
+struct SetHandle
+{
+  struct SetHandle *prev;
+  struct SetHandle *next;
+
+  struct GNUNET_SET_Handle *h;
+};
+
 
 
 /**
@@ -451,7 +454,7 @@ struct ConsensusSession
   /**
    * Client that inhabits the session
    */
-  struct GNUNET_SERVER_Client *client;
+  struct GNUNET_SERVICE_Client *client;
 
   /**
    * Queued messages to the client.
@@ -492,6 +495,21 @@ struct ConsensusSession
    * State of our early stopping scheme.
    */
   int early_stopping;
+
+  /**
+   * Our set size from the first round.
+   */
+  uint64_t first_size;
+
+  uint64_t *first_sizes_received;
+
+  /**
+   * Bounded Eppstein lower bound.
+   */
+  uint64_t lower_bound;
+
+  struct SetHandle *set_handles_head;
+  struct SetHandle *set_handles_tail;
 };
 
 /**
@@ -510,11 +528,6 @@ static struct ConsensusSession *sessions_tail;
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
 /**
- * Handle to the server for this service.
- */
-static struct GNUNET_SERVER_Handle *srv;
-
-/**
  * Peer that runs this service.
  */
 static struct GNUNET_PeerIdentity my_peer;
@@ -528,15 +541,18 @@ struct GNUNET_STATISTICS_Handle *statistics;
 static void
 finish_task (struct TaskEntry *task);
 
+
 static void
 run_ready_steps (struct ConsensusSession *session);
 
+
 static const char *
 phasename (uint16_t phase)
 {
   switch (phase)
   {
     case PHASE_KIND_ALL_TO_ALL: return "ALL_TO_ALL";
+    case PHASE_KIND_ALL_TO_ALL_2: return "ALL_TO_ALL_2";
     case PHASE_KIND_FINISH: return "FINISH";
     case PHASE_KIND_GRADECAST_LEADER: return "GRADECAST_LEADER";
     case PHASE_KIND_GRADECAST_ECHO: return "GRADECAST_ECHO";
@@ -653,36 +669,6 @@ debug_str_rfn_key (const struct RfnKey *rk)
 
 
 /**
- * Destroy a session, free all resources associated with it.
- *
- * @param session the session to destroy
- */
-static void
-destroy_session (struct ConsensusSession *session)
-{
-  GNUNET_CONTAINER_DLL_remove (sessions_head, sessions_tail, session);
-  if (NULL != session->set_listener)
-  {
-    GNUNET_SET_listen_cancel (session->set_listener);
-    session->set_listener = NULL;
-  }
-  if (NULL != session->client_mq)
-  {
-    GNUNET_MQ_destroy (session->client_mq);
-    session->client_mq = NULL;
-    /* The MQ cleanup will also disconnect the underlying client. */
-    session->client = NULL;
-  }
-  if (NULL != session->client)
-  {
-    GNUNET_SERVER_client_disconnect (session->client);
-    session->client = NULL;
-  }
-  GNUNET_free (session);
-}
-
-
-/**
  * Send the final result set of the consensus to the client, element by
  * element.
  *
@@ -702,16 +688,25 @@ send_to_client_iter (void *cls,
   if (NULL != element)
   {
     struct GNUNET_CONSENSUS_ElementMessage *m;
+    const struct ConsensusElement *ce;
+
+    GNUNET_assert (GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT == element->element_type);
+    ce = element->data;
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "marker is %u\n", (unsigned) ce->marker);
+
+    if (0 != ce->marker)
+      return GNUNET_YES;
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "P%d: sending element %s to client\n",
                 session->local_peer_idx,
                 debug_str_element (element));
 
-    ev = GNUNET_MQ_msg_extra (m, element->size,
+    ev = GNUNET_MQ_msg_extra (m, element->size - sizeof (struct ConsensusElement),
                               GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_RECEIVED_ELEMENT);
-    m->element_type = htons (element->element_type);
-    GNUNET_memcpy (&m[1], element->data, element->size);
+    m->element_type = ce->payload_type;
+    GNUNET_memcpy (&m[1], &ce[1], element->size - sizeof (struct ConsensusElement));
     GNUNET_MQ_send (session->client_mq, ev);
   }
   else
@@ -881,7 +876,7 @@ rfn_vote (struct ReferendumEntry *rfn,
 }
 
 
-uint16_t
+static uint16_t
 task_other_peer (struct TaskEntry *task)
 {
   uint16_t me = task->step->session->local_peer_idx;
@@ -891,17 +886,33 @@ task_other_peer (struct TaskEntry *task)
 }
 
 
+static int
+cmp_uint64_t (const void *pa, const void *pb)
+{
+  uint64_t a = *(uint64_t *) pa;
+  uint64_t b = *(uint64_t *) pb;
+
+  if (a == b)
+    return 0;
+  if (a < b)
+    return -1;
+  return 1;
+}
+
+
 /**
  * Callback for set operation results. Called for each element
  * in the result set.
  *
  * @param cls closure
- * @param element a result element, only valid if status is GNUNET_SET_STATUS_OK
+ * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status see enum GNUNET_SET_Status
  */
 static void
 set_result_cb (void *cls,
                const struct GNUNET_SET_Element *element,
+               uint64_t current_size,
                enum GNUNET_SET_Status status)
 {
   struct TaskEntry *task = cls;
@@ -911,6 +922,18 @@ set_result_cb (void *cls,
   struct ReferendumEntry *output_rfn = NULL;
   unsigned int other_idx;
   struct SetOpCls *setop;
+  const struct ConsensusElement *consensus_element = NULL;
+
+  if (NULL != element)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "P%u: got element of type %u, status %u\n",
+                session->local_peer_idx,
+                (unsigned) element->element_type,
+                (unsigned) status);
+    GNUNET_assert (GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT == element->element_type);
+    consensus_element = element->data;
+  }
 
   setop = &task->cls.setop;
 
@@ -963,19 +986,54 @@ set_result_cb (void *cls,
       return;
   }
 
-  if ( (GNUNET_SET_STATUS_ADD_LOCAL == status) || (GNUNET_SET_STATUS_ADD_REMOTE == status) )
+  if ( (NULL != consensus_element) && (0 != consensus_element->marker) )
   {
-    if ( (GNUNET_YES == setop->transceive_contested) && (ELEMENT_TYPE_CONTESTED_MARKER == element->element_type) )
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "P%u: got some marker\n",
+                  session->local_peer_idx);
+    if ( (GNUNET_YES == setop->transceive_contested) &&
+         (CONSENSUS_MARKER_CONTESTED == consensus_element->marker) )
     {
       GNUNET_assert (NULL != output_rfn);
       rfn_contest (output_rfn, task_other_peer (task));
       return;
     }
+
+    if (CONSENSUS_MARKER_SIZE == consensus_element->marker)
+    {
+
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "P%u: got size marker\n",
+                  session->local_peer_idx);
+
+
+      struct ConsensusSizeElement *cse = (void *) consensus_element;
+
+      if (cse->sender_index == other_idx)
+      {
+        if (NULL == session->first_sizes_received)
+          session->first_sizes_received = GNUNET_new_array (session->num_peers, uint64_t);
+        session->first_sizes_received[other_idx] = GNUNET_ntohll (cse->size);
+
+        uint64_t *copy = GNUNET_memdup (session->first_sizes_received, sizeof (uint64_t) * session->num_peers);
+        qsort (copy, session->num_peers, sizeof (uint64_t), cmp_uint64_t);
+        session->lower_bound = copy[session->num_peers / 3 + 1];
+        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                    "P%u: lower bound %llu\n",
+                    session->local_peer_idx,
+                    (long long) session->lower_bound);
+        GNUNET_free (copy);
+      }
+      return;
+    }
+
+    return;
   }
 
   switch (status)
   {
     case GNUNET_SET_STATUS_ADD_LOCAL:
+      GNUNET_assert (NULL != consensus_element);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Adding element in Task {%s}\n",
                   debug_str_task_key (&task->key));
@@ -1022,9 +1080,10 @@ set_result_cb (void *cls,
       // XXX: add result to structures in task
       break;
     case GNUNET_SET_STATUS_ADD_REMOTE:
+      GNUNET_assert (NULL != consensus_element);
       if (GNUNET_YES == setop->do_not_remove)
         break;
-      if (ELEMENT_TYPE_CONTESTED_MARKER == element->element_type)
+      if (CONSENSUS_MARKER_CONTESTED == consensus_element->marker)
         break;
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Removing element in Task {%s}\n",
@@ -1080,6 +1139,10 @@ set_result_cb (void *cls,
       {
         rfn_commit (output_rfn, task_other_peer (task));
       }
+      if (PHASE_KIND_ALL_TO_ALL == task->key.kind)
+      {
+        session->first_size = current_size;
+      }
       finish_task (task);
       break;
     case GNUNET_SET_STATUS_FAILURE:
@@ -1102,6 +1165,7 @@ enum EvilnessType
   EVILNESS_CRAM_LEAD,
   EVILNESS_CRAM_ECHO,
   EVILNESS_SLACK,
+  EVILNESS_SLACK_A2A,
 };
 
 enum EvilnessSubType
@@ -1194,6 +1258,10 @@ get_evilness (struct ConsensusSession *session, struct Evilness *evil)
       {
         evil->type = EVILNESS_SLACK;
       }
+      if (0 == strcmp ("slack-a2a", evil_type_str))
+      {
+        evil->type = EVILNESS_SLACK_A2A;
+      }
       else if (0 == strcmp ("cram-all", evil_type_str))
       {
         evil->type = EVILNESS_CRAM_ALL;
@@ -1259,6 +1327,34 @@ commit_set (struct ConsensusSession *session,
   set = lookup_set (session, &setop->input_set);
   GNUNET_assert (NULL != set);
 
+  if ( (GNUNET_YES == setop->transceive_contested) && (GNUNET_YES == set->is_contested) )
+  {
+    struct GNUNET_SET_Element element;
+    struct ConsensusElement ce = { 0 };
+    ce.marker = CONSENSUS_MARKER_CONTESTED;
+    element.data = &ce;
+    element.size = sizeof (struct ConsensusElement);
+    element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
+    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
+  }
+
+  if (PHASE_KIND_ALL_TO_ALL_2 == task->key.kind)
+  {
+    struct GNUNET_SET_Element element;
+    struct ConsensusSizeElement cse = {
+      .size = 0,
+      .sender_index = 0
+    };
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "inserting size marker\n");
+    cse.ce.marker = CONSENSUS_MARKER_SIZE;
+    cse.size = GNUNET_htonll (session->first_size);
+    cse.sender_index = session->local_peer_idx;
+    element.data = &cse;
+    element.size = sizeof (struct ConsensusSizeElement);
+    element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
+    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
+  }
+
 #ifdef EVIL
   {
     unsigned int i;
@@ -1300,21 +1396,24 @@ commit_set (struct ConsensusSession *session,
         }
         for (i = 0; i < evil.num; i++)
         {
-          struct GNUNET_HashCode hash;
           struct GNUNET_SET_Element element;
-          element.data = &hash;
-          element.size = sizeof (struct GNUNET_HashCode);
-          element.element_type = 0;
+          struct ConsensusStuffedElement se = {
+            .ce.payload_type = 0,
+            .ce.marker = 0,
+          };
+          element.data = &se;
+          element.size = sizeof (struct ConsensusStuffedElement);
+          element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
 
           if (EVILNESS_SUB_REPLACEMENT == evil.subtype)
           {
             /* Always generate a new element. */
-            GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &hash);
+            GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &se.rand);
           }
           else if (EVILNESS_SUB_NO_REPLACEMENT == evil.subtype)
           {
             /* Always cram the same elements, derived from counter. */
-            GNUNET_CRYPTO_hash (&i, sizeof (i), &hash);
+            GNUNET_CRYPTO_hash (&i, sizeof (i), &se.rand);
           }
           else
           {
@@ -1341,6 +1440,19 @@ commit_set (struct ConsensusSession *session,
                     "P%u: evil peer: slacking\n",
                     (unsigned int) session->local_peer_idx);
         /* Do nothing. */
+      case EVILNESS_SLACK_A2A:
+        if ( (PHASE_KIND_ALL_TO_ALL_2 == task->key.kind ) ||
+             (PHASE_KIND_ALL_TO_ALL == task->key.kind) )
+        {
+          struct GNUNET_SET_Handle *empty_set;
+          empty_set = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+          GNUNET_SET_commit (setop->op, empty_set);
+          GNUNET_SET_destroy (empty_set);
+        }
+        else
+        {
+          GNUNET_SET_commit (setop->op, set->h);
+        }
         break;
       case EVILNESS_NONE:
         GNUNET_SET_commit (setop->op, set->h);
@@ -1348,15 +1460,6 @@ commit_set (struct ConsensusSession *session,
     }
   }
 #else
-  if ( (GNUNET_YES == setop->transceive_contested) && (GNUNET_YES == set->is_contested) )
-  {
-    struct GNUNET_SET_Element element;
-    struct ContestedPayload payload;
-    element.data = &payload;
-    element.size = sizeof (struct ContestedPayload);
-    element.element_type = ELEMENT_TYPE_CONTESTED_MARKER;
-    GNUNET_SET_add_element (set->h, &element, NULL, NULL);
-  }
   if (GNUNET_NO == session->peers_blacklisted[task_other_peer (task)])
   {
     GNUNET_SET_commit (setop->op, set->h);
@@ -1511,12 +1614,14 @@ rfn_create (uint16_t size)
 }
 
 
-void
+#if UNUSED
+static void
 diff_destroy (struct DiffEntry *diff)
 {
   GNUNET_CONTAINER_multihashmap_destroy (diff->changes);
   GNUNET_free (diff);
 }
+#endif
 
 
 /**
@@ -1576,6 +1681,12 @@ set_copy_cb (void *cls, struct GNUNET_SET_Handle *copy)
   struct TaskEntry *task = scc->task;
   struct SetKey dst_set_key = scc->dst_set_key;
   struct SetEntry *set;
+  struct SetHandle *sh = GNUNET_new (struct SetHandle);
+
+  sh->h = copy;
+  GNUNET_CONTAINER_DLL_insert (task->step->session->set_handles_head,
+                               task->step->session->set_handles_tail,
+                               sh);
 
   GNUNET_free (scc);
   set = GNUNET_new (struct SetEntry);
@@ -2018,7 +2129,7 @@ task_start_reconcile (struct TaskEntry *task)
 
   if (task->key.peer1 == session->local_peer_idx)
   {
-    struct GNUNET_CONSENSUS_RoundContextMessage rcm = { 0 };
+    struct GNUNET_CONSENSUS_RoundContextMessage rcm;
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "P%u: Looking up set {%s} to run remote union\n",
@@ -2033,17 +2144,24 @@ task_start_reconcile (struct TaskEntry *task)
     rcm.peer2 = htons (task->key.peer2);
     rcm.leader = htons (task->key.leader);
     rcm.repetition = htons (task->key.repetition);
+    rcm.is_contested = htons (0);
 
     GNUNET_assert (NULL == setop->op);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: initiating set op with P%u, our set is %s\n",
                 session->local_peer_idx, task->key.peer2, debug_str_set_key (&setop->input_set));
 
+    struct GNUNET_SET_Option opts[] = {
+      { GNUNET_SET_OPTION_BYZANTINE, { .num = session->lower_bound } },
+      { GNUNET_SET_OPTION_END },
+    };
+
     // XXX: maybe this should be done while
     // setting up tasks alreays?
     setop->op = GNUNET_SET_prepare (&session->peers[task->key.peer2],
                                     &session->global_id,
                                     &rcm.header,
                                     GNUNET_SET_RESULT_SYMMETRIC,
+                                    opts,
                                     set_result_cb,
                                     task);
 
@@ -2328,45 +2446,48 @@ peer_id_cmp (const void *h1, const void *h2)
 /**
  * Create the sorted list of peers for the session,
  * add the local peer if not in the join message.
+ *
+ * @param session session to initialize
+ * @param join_msg join message with the list of peers participating at the end
  */
 static void
 initialize_session_peer_list (struct ConsensusSession *session,
-                              struct GNUNET_CONSENSUS_JoinMessage *join_msg)
+                              const struct GNUNET_CONSENSUS_JoinMessage *join_msg)
 {
-  unsigned int local_peer_in_list;
-  uint32_t listed_peers;
-  const struct GNUNET_PeerIdentity *msg_peers;
-  unsigned int i;
-
-  GNUNET_assert (NULL != join_msg);
-
-  /* peers in the join message, may or may not include the local peer */
-  listed_peers = ntohl (join_msg->num_peers);
+  const struct GNUNET_PeerIdentity *msg_peers
+    = (const struct GNUNET_PeerIdentity *) &join_msg[1];
+  int local_peer_in_list;
 
-  session->num_peers = listed_peers;
-
-  msg_peers = (struct GNUNET_PeerIdentity *) &join_msg[1];
+  session->num_peers = ntohl (join_msg->num_peers);
 
+  /* Peers in the join message, may or may not include the local peer,
+     Add it if it is missing. */
   local_peer_in_list = GNUNET_NO;
-  for (i = 0; i < listed_peers; i++)
+  for (unsigned int i = 0; i < session->num_peers; i++)
   {
-    if (0 == memcmp (&msg_peers[i], &my_peer, sizeof (struct GNUNET_PeerIdentity)))
+    if (0 == memcmp (&msg_peers[i],
+                     &my_peer,
+                     sizeof (struct GNUNET_PeerIdentity)))
     {
       local_peer_in_list = GNUNET_YES;
       break;
     }
   }
-
   if (GNUNET_NO == local_peer_in_list)
     session->num_peers++;
 
-  session->peers = GNUNET_malloc (session->num_peers * sizeof (struct GNUNET_PeerIdentity));
-
+  session->peers = GNUNET_new_array (session->num_peers,
+                                     struct GNUNET_PeerIdentity);
   if (GNUNET_NO == local_peer_in_list)
     session->peers[session->num_peers - 1] = my_peer;
 
-  GNUNET_memcpy (session->peers, msg_peers, listed_peers * sizeof (struct GNUNET_PeerIdentity));
-  qsort (session->peers, session->num_peers, sizeof (struct GNUNET_PeerIdentity), &peer_id_cmp);
+  GNUNET_memcpy (session->peers,
+                 msg_peers,
+                 ntohl (join_msg->num_peers) * sizeof (struct GNUNET_PeerIdentity));
+  qsort (session->peers,
+         session->num_peers,
+         sizeof (struct GNUNET_PeerIdentity),
+         &peer_id_cmp);
 }
 
 
@@ -2465,8 +2586,14 @@ set_listen_cb (void *cls,
   GNUNET_assert (! ((task->key.peer1 == session->local_peer_idx) &&
                     (task->key.peer2 == session->local_peer_idx)));
 
+  struct GNUNET_SET_Option opts[] = {
+    { GNUNET_SET_OPTION_BYZANTINE, { .num = session->lower_bound } },
+    { GNUNET_SET_OPTION_END },
+  };
+
   task->cls.setop.op = GNUNET_SET_accept (request,
                                           GNUNET_SET_RESULT_SYMMETRIC,
+                                          opts,
                                           set_result_cb,
                                           task);
 
@@ -2862,11 +2989,42 @@ construct_task_graph (struct ConsensusSession *session)
     put_task (session->taskmap, &task);
   }
 
+  round += 1;
   prev_step = step;
-  step = NULL;
+  step = create_step (session, round, GNUNET_NO);;
+#ifdef GNUNET_EXTRA_LOGGING
+  step->debug_name = GNUNET_strdup ("all to all 2");
+#endif
+  step_depend_on (step, prev_step);
+
+
+  for (i = 0; i < n; i++)
+  {
+    uint16_t p1;
+    uint16_t p2;
+
+    p1 = me;
+    p2 = i;
+    arrange_peers (&p1, &p2, n);
+    task = ((struct TaskEntry) {
+      .key = (struct TaskKey) { PHASE_KIND_ALL_TO_ALL_2, p1, p2, -1, -1 },
+      .step = step,
+      .start = task_start_reconcile,
+      .cancel = task_cancel_reconcile,
+    });
+    task.cls.setop.input_set = (struct SetKey) { SET_KIND_CURRENT, 0 };
+    task.cls.setop.output_set = task.cls.setop.input_set;
+    task.cls.setop.do_not_remove = GNUNET_YES;
+    put_task (session->taskmap, &task);
+  }
 
   round += 1;
 
+  prev_step = step;
+  step = NULL;
+
+
+
   /* Byzantine union */
 
   /* sequential repetitions of the gradecasts */
@@ -2924,250 +3082,230 @@ construct_task_graph (struct ConsensusSession *session)
 }
 
 
+
+/**
+ * Check join message.
+ *
+ * @param cls session of client that sent the message
+ * @param m message sent by the client
+ * @return #GNUNET_OK if @a m is well-formed
+ */
+static int
+check_client_join (void *cls,
+                   const struct GNUNET_CONSENSUS_JoinMessage *m)
+{
+  uint32_t listed_peers = ntohl (m->num_peers);
+
+  if ( (ntohs (m->header.size) - sizeof (*m)) !=
+       listed_peers * sizeof (struct GNUNET_PeerIdentity))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
 /**
- * Initialize the session, continue receiving messages from the owning client
+ * Called when a client wants to join a consensus session.
  *
- * @param session the session to initialize
- * @param join_msg the join message from the client
+ * @param cls session of client that sent the message
+ * @param m message sent by the client
  */
 static void
-initialize_session (struct ConsensusSession *session,
-                    struct GNUNET_CONSENSUS_JoinMessage *join_msg)
+handle_client_join (void *cls,
+                    const struct GNUNET_CONSENSUS_JoinMessage *m)
 {
+  struct ConsensusSession *session = cls;
   struct ConsensusSession *other_session;
 
-  initialize_session_peer_list (session, join_msg);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "session with %u peers\n", session->num_peers);
-  compute_global_id (session, &join_msg->session_id);
+  initialize_session_peer_list (session,
+                                m);
+  compute_global_id (session,
+                     &m->session_id);
 
   /* Check if some local client already owns the session.
      It is only legal to have a session with an existing global id
      if all other sessions with this global id are finished.*/
-  other_session = sessions_head;
-  while (NULL != other_session)
+  for (other_session = sessions_head;
+       NULL != other_session;
+       other_session = other_session->next)
   {
-    if ((other_session != session) &&
-        (0 == GNUNET_CRYPTO_hash_cmp (&session->global_id, &other_session->global_id)))
-    {
-      //if (CONSENSUS_ROUND_FINISH != other_session->current_round)
-      //{
-      //  GNUNET_break (0);
-      //  destroy_session (session);
-      //  return;
-      //}
+    if ( (other_session != session) &&
+         (0 == GNUNET_CRYPTO_hash_cmp (&session->global_id,
+                                       &other_session->global_id)) )
       break;
-    }
-    other_session = other_session->next;
   }
 
-  session->conclude_deadline = GNUNET_TIME_absolute_ntoh (join_msg->deadline);
-  session->conclude_start = GNUNET_TIME_absolute_ntoh (join_msg->start);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "consensus with timeout %llums created\n",
-              (long long) (GNUNET_TIME_absolute_get_difference (session->conclude_start, session->conclude_deadline)).rel_value_us / 1000);
-
-  session->local_peer_idx = get_peer_idx (&my_peer, session);
+  session->conclude_deadline
+    = GNUNET_TIME_absolute_ntoh (m->deadline);
+  session->conclude_start
+    = GNUNET_TIME_absolute_ntoh (m->start);
+  session->local_peer_idx = get_peer_idx (&my_peer,
+                                          session);
   GNUNET_assert (-1 != session->local_peer_idx);
-  session->set_listener = GNUNET_SET_listen (cfg, GNUNET_SET_OPERATION_UNION,
-                                             &session->global_id,
-                                             set_listen_cb, session);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d is the local peer\n", session->local_peer_idx);
 
-  session->setmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->taskmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->diffmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-  session->rfnmap = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Joining consensus session %s containing %u peers as %u with timeout %s\n",
+              GNUNET_h2s (&m->session_id),
+              session->num_peers,
+              session->local_peer_idx,
+              GNUNET_STRINGS_relative_time_to_string
+              (GNUNET_TIME_absolute_get_difference (session->conclude_start,
+                                                    session->conclude_deadline),
+               GNUNET_YES));
+
+  session->set_listener
+    = GNUNET_SET_listen (cfg,
+                         GNUNET_SET_OPERATION_UNION,
+                         &session->global_id,
+                         &set_listen_cb,
+                         session);
+
+  session->setmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                          GNUNET_NO);
+  session->taskmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                           GNUNET_NO);
+  session->diffmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                           GNUNET_NO);
+  session->rfnmap = GNUNET_CONTAINER_multihashmap_create (1,
+                                                          GNUNET_NO);
 
   {
     struct SetEntry *client_set;
+
     client_set = GNUNET_new (struct SetEntry);
-    client_set->h = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+    client_set->h = GNUNET_SET_create (cfg,
+                                       GNUNET_SET_OPERATION_UNION);
+    struct SetHandle *sh = GNUNET_new (struct SetHandle);
+    sh->h = client_set->h;
+    GNUNET_CONTAINER_DLL_insert (session->set_handles_head,
+                                 session->set_handles_tail,
+                                 sh);
     client_set->key = ((struct SetKey) { SET_KIND_CURRENT, 0, 0 });
-    put_set (session, client_set);
+    put_set (session,
+             client_set);
   }
 
-  session->peers_blacklisted = GNUNET_new_array (session->num_peers, int);
+  session->peers_blacklisted = GNUNET_new_array (session->num_peers,
+                                                 int);
 
   /* Just construct the task graph,
      but don't run anything until the client calls conclude. */
   construct_task_graph (session);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "session %s initialized\n", GNUNET_h2s (&session->global_id));
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
-static struct ConsensusSession *
-get_session_by_client (struct GNUNET_SERVER_Client *client)
+static void
+client_insert_done (void *cls)
 {
-  struct ConsensusSession *session;
-
-  session = sessions_head;
-  while (NULL != session)
-  {
-    if (session->client == client)
-      return session;
-    session = session->next;
-  }
-  return NULL;
+  // FIXME: implement
 }
 
 
 /**
- * Called when a client wants to join a consensus session.
+ * Called when a client performs an insert operation.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client handle
+ * @param msg message sent by the client
+ * @return #GNUNET_OK (always well-formed)
  */
-static void
-client_join (void *cls,
-             struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *m)
-{
-  struct ConsensusSession *session;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "join message sent by client\n");
-
-  session = get_session_by_client (client);
-  if (NULL != session)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  session = GNUNET_new (struct ConsensusSession);
-  session->client = client;
-  session->client_mq = GNUNET_MQ_queue_for_server_client (client);
-  GNUNET_CONTAINER_DLL_insert (sessions_head, sessions_tail, session);
-  initialize_session (session, (struct GNUNET_CONSENSUS_JoinMessage *) m);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "join done\n");
-}
-
-
-static void
-client_insert_done (void *cls)
+static int
+check_client_insert (void *cls,
+                      const struct GNUNET_CONSENSUS_ElementMessage *msg)
 {
-  // FIXME: implement
+  return GNUNET_OK;
 }
 
 
 /**
  * Called when a client performs an insert operation.
  *
- * @param cls (unused)
- * @param client client handle
- * @param m message sent by the client
+ * @param cls client handle
+ * @param msg message sent by the client
  */
-void
-client_insert (void *cls,
-               struct GNUNET_SERVER_Client *client,
-               const struct GNUNET_MessageHeader *m)
+static void
+handle_client_insert (void *cls,
+                      const struct GNUNET_CONSENSUS_ElementMessage *msg)
 {
-  struct ConsensusSession *session;
-  struct GNUNET_CONSENSUS_ElementMessage *msg;
-  struct GNUNET_SET_Element *element;
+  struct ConsensusSession *session = cls;
   ssize_t element_size;
   struct GNUNET_SET_Handle *initial_set;
-
-  session = get_session_by_client (client);
-
-  if (NULL == session)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
+  struct ConsensusElement *ce;
 
   if (GNUNET_YES == session->conclude_started)
   {
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (session->client);
     return;
   }
 
-  msg = (struct GNUNET_CONSENSUS_ElementMessage *) m;
   element_size = ntohs (msg->header.size) - sizeof (struct GNUNET_CONSENSUS_ElementMessage);
-  if (element_size < 0)
-  {
-    GNUNET_break (0);
-    return;
-  }
+  ce = GNUNET_malloc (sizeof (struct ConsensusElement) + element_size);
+  GNUNET_memcpy (&ce[1], &msg[1], element_size);
+  ce->payload_type = msg->element_type;
+
+  struct GNUNET_SET_Element element = {
+    .element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT,
+    .size = sizeof (struct ConsensusElement) + element_size,
+    .data = ce,
+  };
 
-  element = GNUNET_malloc (sizeof (struct GNUNET_SET_Element) + element_size);
-  element->element_type = msg->element_type;
-  element->size = element_size;
-  GNUNET_memcpy (&element[1], &msg[1], element_size);
-  element->data = &element[1];
   {
     struct SetKey key = { SET_KIND_CURRENT, 0, 0 };
     struct SetEntry *entry;
-    entry = lookup_set (session, &key);
+
+    entry = lookup_set (session,
+                        &key);
     GNUNET_assert (NULL != entry);
     initial_set = entry->h;
   }
+
   session->num_client_insert_pending++;
-  GNUNET_SET_add_element (initial_set, element, client_insert_done, session);
+  GNUNET_SET_add_element (initial_set,
+                          &element,
+                          &client_insert_done,
+                          session);
 
 #ifdef GNUNET_EXTRA_LOGGING
   {
-    struct GNUNET_HashCode hash;
-
-    GNUNET_SET_element_hash (element, &hash);
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: element %s added\n",
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "P%u: element %s added\n",
                 session->local_peer_idx,
-                GNUNET_h2s (&hash));
+                debug_str_element (&element));
   }
 #endif
-
-  GNUNET_free (element);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_free (ce);
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
 /**
  * Called when a client performs the conclude operation.
  *
- * @param cls (unused)
- * @param client client handle
+ * @param cls client handle
  * @param message message sent by the client
  */
 static void
-client_conclude (void *cls,
-                 struct GNUNET_SERVER_Client *client,
-                 const struct GNUNET_MessageHeader *message)
+handle_client_conclude (void *cls,
+                        const struct GNUNET_MessageHeader *message)
 {
-  struct ConsensusSession *session;
-
-  session = get_session_by_client (client);
-  if (NULL == session)
-  {
-    /* client not found */
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
+  struct ConsensusSession *session = cls;
 
   if (GNUNET_YES == session->conclude_started)
   {
     /* conclude started twice */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
-    destroy_session (session);
+    GNUNET_SERVICE_client_drop (session->client);
     return;
   }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "conclude requested\n");
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "conclude requested\n");
   session->conclude_started = GNUNET_YES;
-
   install_step_timeouts (session);
   run_ready_steps (session);
-
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (session->client);
 }
 
 
@@ -3179,82 +3317,123 @@ client_conclude (void *cls,
 static void
 shutdown_task (void *cls)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "shutting down\n");
-  while (NULL != sessions_head)
-    destroy_session (sessions_head);
-
-  GNUNET_STATISTICS_destroy (statistics, GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "shutting down\n");
+  GNUNET_STATISTICS_destroy (statistics,
+                             GNUNET_NO);
+  statistics = NULL;
 }
 
 
 /**
- * Clean up after a client after it is
- * disconnected (either by us or by itself)
+ * Start processing consensus requests.
  *
- * @param cls closure, unused
- * @param client the client to clean up after
+ * @param cls closure
+ * @param c configuration to use
+ * @param service the initialized service
  */
-void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  struct ConsensusSession *session;
-
-  session = get_session_by_client (client);
-  if (NULL == session)
+  cfg = c;
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_get_peer_identity (cfg,
+                                       &my_peer))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Could not retrieve host identity\n");
+    GNUNET_SCHEDULER_shutdown ();
     return;
-  // FIXME: destroy if we can
+  }
+  statistics = GNUNET_STATISTICS_create ("consensus",
+                                         cfg);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
 }
 
 
+/**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct ConsensusSession *session = GNUNET_new (struct ConsensusSession);
+
+  session->client = c;
+  session->client_mq = mq;
+  GNUNET_CONTAINER_DLL_insert (sessions_head,
+                               sessions_tail,
+                               session);
+  return session;
+}
+
 
 /**
- * Start processing consensus requests.
+ * Callback called when a client disconnected from the service
  *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  static const struct GNUNET_SERVER_MessageHandler server_handlers[] = {
-    {&client_conclude, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_CONCLUDE,
-        sizeof (struct GNUNET_MessageHeader)},
-    {&client_insert, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_INSERT, 0},
-    {&client_join, NULL, GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_JOIN, 0},
-    {NULL, NULL, 0, 0}
-  };
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
+{
+  struct ConsensusSession *session = internal_cls;
 
-  cfg = c;
-  srv = server;
-  if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &my_peer))
+  if (NULL != session->set_listener)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+    GNUNET_SET_listen_cancel (session->set_listener);
+    session->set_listener = NULL;
+  }
+  GNUNET_CONTAINER_DLL_remove (sessions_head,
+                               sessions_tail,
+                               session);
+
+  while (session->set_handles_head)
+  {
+    struct SetHandle *sh = session->set_handles_head;
+    session->set_handles_head = sh->next;
+    GNUNET_SET_destroy (sh->h);
+    GNUNET_free (sh);
   }
-  statistics = GNUNET_STATISTICS_create ("consensus", cfg);
-  GNUNET_SERVER_add_handlers (server, server_handlers);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-  GNUNET_SERVER_disconnect_notify (server, handle_client_disconnect, NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "consensus running\n");
+  GNUNET_free (session);
 }
 
 
 /**
- * The main function for the consensus service.
- *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * Define "main" method using service macro.
  */
-int
-main (int argc, char *const *argv)
-{
-  int ret;
-  ret = GNUNET_SERVICE_run (argc, argv, "consensus", GNUNET_SERVICE_OPTION_NONE, &run, NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "exit (%d)\n", GNUNET_OK != ret);
-  return (GNUNET_OK == ret) ? 0 : 1;
-}
+GNUNET_SERVICE_MAIN
+("consensus",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (client_conclude,
+                          GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_CONCLUDE,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_var_size (client_insert,
+                        GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_INSERT,
+                        struct GNUNET_CONSENSUS_ElementMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (client_join,
+                        GNUNET_MESSAGE_TYPE_CONSENSUS_CLIENT_JOIN,
+                        struct GNUNET_CONSENSUS_JoinMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
+/* end of gnunet-service-consensus.c */
diff --git a/src/consensus/plugin_block_consensus.c b/src/consensus/plugin_block_consensus.c
new file mode 100644
index 000000000..de0f0886f
--- /dev/null
+++ b/src/consensus/plugin_block_consensus.c
@@ -0,0 +1,137 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file consensus/plugin_block_consensus.c
+ * @brief consensus block, either nested block or marker
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "consensus_protocol.h"
+#include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
+
+
+/**
+ * Function called to validate a reply or a request.  For
+ * request evaluation, simply pass "NULL" for the reply_block.
+ *
+ * @param cls closure
+ * @param ctx context
+ * @param type block type
+ * @param group block group to use
+ * @param eo control flags
+ * @param query original query (hash)
+ * @param xquery extrended query data (can be NULL, depending on type)
+ * @param xquery_size number of bytes in xquery
+ * @param reply_block response to validate
+ * @param reply_block_size number of bytes in reply block
+ * @return characterization of result
+ */
+static enum GNUNET_BLOCK_EvaluationResult
+block_plugin_consensus_evaluate (void *cls,
+                                 struct GNUNET_BLOCK_Context *ctx,
+                                 enum GNUNET_BLOCK_Type type,
+                                 struct GNUNET_BLOCK_Group *group,
+                                 enum GNUNET_BLOCK_EvaluationOptions eo,
+                                 const struct GNUNET_HashCode *query,
+                                 const void *xquery,
+                                 size_t xquery_size,
+                                 const void *reply_block,
+                                 size_t reply_block_size)
+{
+  if (reply_block_size < sizeof (struct ConsensusElement))
+    return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
+
+  const struct ConsensusElement *ce = reply_block;
+
+  if ( (0 != ce->marker) ||
+       (0 == ce->payload_type ) )
+    return GNUNET_BLOCK_EVALUATION_OK_MORE;
+
+  return GNUNET_BLOCK_evaluate (ctx,
+                                type,
+                                group,
+                                eo,
+                                query,
+                                xquery,
+                                xquery_size,
+                                &ce[1],
+                                reply_block_size - sizeof (struct ConsensusElement));
+}
+
+
+/**
+ * Function called to obtain the key for a block.
+ *
+ * @param cls closure
+ * @param type block type
+ * @param block block to get the key for
+ * @param block_size number of bytes in block
+ * @param key set to the key (query) for the given block
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
+ *         (or if extracting a key from a block of this type does not work)
+ */
+static int
+block_plugin_consensus_get_key (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               const void *block,
+                               size_t block_size,
+                               struct GNUNET_HashCode *key)
+{
+  return GNUNET_SYSERR;
+}
+
+
+/**
+ * Entry point for the plugin.
+ */
+void *
+libgnunet_plugin_block_consensus_init (void *cls)
+{
+  static enum GNUNET_BLOCK_Type types[] =
+  {
+    GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT,
+    GNUNET_BLOCK_TYPE_ANY       /* end of list */
+  };
+  struct GNUNET_BLOCK_PluginFunctions *api;
+
+  api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
+  api->evaluate = &block_plugin_consensus_evaluate;
+  api->get_key = &block_plugin_consensus_get_key;
+  api->types = types;
+  return api;
+}
+
+
+/**
+ * Exit point from the plugin.
+ */
+void *
+libgnunet_plugin_block_consensus_done (void *cls)
+{
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
+
+  GNUNET_free (api);
+  return NULL;
+}
+
+/* end of plugin_block_consensus.c */
diff --git a/src/consensus/test_consensus.conf b/src/consensus/test_consensus.conf
index 8f144fa5e..881251a66 100644
--- a/src/consensus/test_consensus.conf
+++ b/src/consensus/test_consensus.conf
@@ -2,9 +2,11 @@
 GNUNET_TEST_HOME = /tmp/test-consensus/
 
 [consensus]
-OPTIONS = -L INFO
+#OPTIONS = -L INFO
 BINARY = gnunet-service-evil-consensus
 
+PREFIX = valgrind
+
 #EVIL_SPEC = 0;cram-all;noreplace;5
 #EVIL_SPEC = 0;cram;5/1;cram;5
 #EVIL_SPEC = 0;cram;5/1;cram;5/2;cram;5
@@ -15,19 +17,22 @@ BINARY = gnunet-service-evil-consensus
 RESOURCE_DIAGNOSTICS = resource.log.${PEERID:-master}
 
 [core]
-FORECESTART = YES
+FORCESTART = YES
 
 [revocation]
-FORECESTART = NO
+FORCESTART = NO
 
 [fs]
-FORECESTART = NO
+FORCESTART = NO
 
 [gns]
-FORECESTART = NO
+FORCESTART = NO
+
+[zonemaster]
+FORCESTART = NO
 
 [hostlist]
-FORECESTART = NO
+FORCESTART = NO
 
 [cadet]
 #PREFIX = valgrind
@@ -37,7 +42,7 @@ PLUGINS = unix
 OPTIONS = -LERROR
 
 [set]
-OPTIONS = -L INFO
+#OPTIONS = -L INFO
 #PREFIX = valgrind --leak-check=full
 #PREFIX = valgrind
 
diff --git a/src/conversation/.gitignore b/src/conversation/.gitignore
index 0e4345a8b..e74e259bf 100644
--- a/src/conversation/.gitignore
+++ b/src/conversation/.gitignore
@@ -3,3 +3,6 @@ gnunet-conversation
 gnunet-conversation-test
 gnunet-helper-audio-playback
 gnunet-helper-audio-record
+test_conversation_api
+test_conversation_api_reject
+test_conversation_api_twocalls
diff --git a/src/conversation/Makefile.am b/src/conversation/Makefile.am
index f61173a66..cbffc3283 100644
--- a/src/conversation/Makefile.am
+++ b/src/conversation/Makefile.am
@@ -101,7 +101,7 @@ libexec_PROGRAMS += \
   $(AUDIO_HELPER_RECD) \
   $(AUDIO_HELPER_PLAY)
 
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 if ENABLE_TEST_RUN
 TESTS = $(AUDIO_TESTS)
 endif
@@ -138,7 +138,7 @@ gnunet_helper_audio_record_CFLAGS = \
  $(GST_CFLAGS)
 
 gnunet_helper_audio_playback_SOURCES = \
-    gnunet_gst_test.c gnunet_gst.c
+    gnunet-helper-audio-playback-gst.c
 gnunet_helper_audio_playback_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(GST_LIBS) \
diff --git a/src/conversation/conversation_api_call.c b/src/conversation/conversation_api_call.c
index a6bc506bc..7e4a147a0 100644
--- a/src/conversation/conversation_api_call.c
+++ b/src/conversation/conversation_api_call.c
@@ -603,7 +603,6 @@ GNUNET_CONVERSATION_call_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                                         &my_zone,
                                         GNUNET_GNSRECORD_TYPE_PHONE,
                                         GNUNET_NO,
-                                        NULL /* FIXME: add shortening support */,
                                         &handle_gns_response, call);
   GNUNET_assert (NULL != call->gns_lookup);
   return call;
diff --git a/src/conversation/gnunet-conversation.c b/src/conversation/gnunet-conversation.c
index 925db4665..8f9ddec25 100644
--- a/src/conversation/gnunet-conversation.c
+++ b/src/conversation/gnunet-conversation.c
@@ -1265,13 +1265,20 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "ego", "NAME",
-     gettext_noop ("sets the NAME of the ego to use for the phone (and name resolution)"),
-     1, &GNUNET_GETOPT_set_string, &ego_name},
-    {'p', "phone", "LINE",
-      gettext_noop ("sets the LINE to use for the phone"),
-     1, &GNUNET_GETOPT_set_string, &line},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "ego",
+                                 "NAME",
+                                 gettext_noop ("sets the NAME of the ego to use for the phone (and name resolution)"),
+                                 &ego_name),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "phone",
+                                 "LINE",
+                                 gettext_noop ("sets the LINE to use for the phone"),
+                                 &line),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/conversation/gnunet-helper-audio-playback-gst.c b/src/conversation/gnunet-helper-audio-playback-gst.c
index 9307cd12e..264b14e76 100644
--- a/src/conversation/gnunet-helper-audio-playback-gst.c
+++ b/src/conversation/gnunet-helper-audio-playback-gst.c
@@ -56,7 +56,7 @@
 /**
  * Tokenizer for the data we get from stdin
  */
-struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst;
+struct GNUNET_MessageStreamTokenizer *stdin_mst;
 
 /**
  * Main pipeline.
@@ -82,20 +82,20 @@ static int abort_read;
 
 static void
 sink_child_added (GstChildProxy *child_proxy,
-                  GObject *object, 
+                  GObject *object,
                   gchar *name,
                   gpointer user_data)
 {
   if (GST_IS_AUDIO_BASE_SRC (object))
     g_object_set (object,
-                  "buffer-time", (gint64) BUFFER_TIME, 
+                  "buffer-time", (gint64) BUFFER_TIME,
                   "latency-time", (gint64) LATENCY_TIME,
                   NULL);
 }
 
 
 static void
-ogg_pad_added (GstElement *element, 
+ogg_pad_added (GstElement *element,
                GstPad *pad,
                gpointer data)
 {
@@ -127,7 +127,7 @@ quit ()
 static gboolean
 bus_call (GstBus *bus, GstMessage *msg, gpointer data)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Bus message\n");
   switch (GST_MESSAGE_TYPE (msg))
   {
@@ -141,15 +141,15 @@ bus_call (GstBus *bus, GstMessage *msg, gpointer data)
     {
       gchar  *debug;
       GError *error;
-      
+
       gst_message_parse_error (msg, &error, &debug);
       g_free (debug);
-      
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 
-                  "Error: %s\n", 
+
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Error: %s\n",
                   error->message);
       g_error_free (error);
-      
+
       quit ();
       break;
     }
@@ -196,7 +196,7 @@ feed_buffer_to_gst (const char *audio, size_t b_len)
   switch (flow)
   {
   case GST_FLOW_OK:
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Fed %u bytes to the pipeline\n",
                 (unsigned int) b_len);
     break;
@@ -207,7 +207,7 @@ feed_buffer_to_gst (const char *audio, size_t b_len)
     break;
   case GST_FLOW_EOS:
     /* end of stream */
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "EOS\n");
     break;
   default:
@@ -224,7 +224,6 @@ feed_buffer_to_gst (const char *audio, size_t b_len)
  */
 static int
 stdin_receiver (void *cls,
-                void *client,
                 const struct GNUNET_MessageHeader *msg)
 {
   struct AudioMessage *audio;
@@ -253,21 +252,21 @@ main (int argc, char **argv)
   uint64_t toff;
 
   typedef void (*SignalHandlerPointer) (int);
- 
+
   SignalHandlerPointer inthandler, termhandler;
 #ifdef DEBUG_READ_PURE_OGG
   int read_pure_ogg = getenv ("GNUNET_READ_PURE_OGG") ? 1 : 0;
 #endif
 
-  inthandler = signal (SIGINT, 
+  inthandler = signal (SIGINT,
                        &signalhandler);
-  termhandler = signal (SIGTERM, 
+  termhandler = signal (SIGTERM,
                         &signalhandler);
-  
+
 #ifdef WINDOWS
   setmode (0, _O_BINARY);
 #endif
-  
+
   /* Initialisation */
   gst_init (&argc, &argv);
 
@@ -275,13 +274,13 @@ main (int argc, char **argv)
                  GNUNET_log_setup ("gnunet-helper-audio-playback-gst",
                                    "WARNING",
                                    NULL));
-  
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Audio sink starts\n");
-  
-  stdin_mst = GNUNET_SERVER_mst_create (&stdin_receiver, 
-                                        NULL);
-  
+
+  stdin_mst = GNUNET_MST_create (&stdin_receiver,
+                                 NULL);
+
   /* Create gstreamer elements */
   pipeline = gst_pipeline_new ("audio-player");
   source   = gst_element_factory_make ("appsrc",        "audio-input");
@@ -298,13 +297,13 @@ main (int argc, char **argv)
     return -1;
   }
 
-  g_signal_connect (sink, 
+  g_signal_connect (sink,
                     "child-added",
-                    G_CALLBACK (sink_child_added), 
+                    G_CALLBACK (sink_child_added),
                     NULL);
-  g_signal_connect (demuxer, 
+  g_signal_connect (demuxer,
                     "pad-added",
-                    G_CALLBACK (ogg_pad_added), 
+                    G_CALLBACK (ogg_pad_added),
                     decoder);
 
   /* Keep a reference to it, we operate on it */
@@ -362,7 +361,7 @@ main (int argc, char **argv)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Received %d bytes of audio data (total: %llu)\n",
                 (int) ret,
-                toff);
+                (unsigned long long) toff);
     if (0 == ret)
       break;
 #ifdef DEBUG_READ_PURE_OGG
@@ -372,20 +371,22 @@ main (int argc, char **argv)
     }
     else
 #endif
-    GNUNET_SERVER_mst_receive (stdin_mst, NULL,
-                               readbuf, ret,
-                               GNUNET_NO, GNUNET_NO);
+    GNUNET_MST_from_buffer (stdin_mst,
+                            readbuf,
+                            ret,
+                            GNUNET_NO,
+                            GNUNET_NO);
   }
-  GNUNET_SERVER_mst_destroy (stdin_mst);
+  GNUNET_MST_destroy (stdin_mst);
 
   signal (SIGINT, inthandler);
   signal (SIGINT, termhandler);
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Returned, stopping playback\n");
   quit ();
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Deleting pipeline\n");
   gst_object_unref (GST_OBJECT (source));
   source = NULL;
diff --git a/src/conversation/gnunet-helper-audio-playback.c b/src/conversation/gnunet-helper-audio-playback.c
index e965cb2aa..4344e1d41 100644
--- a/src/conversation/gnunet-helper-audio-playback.c
+++ b/src/conversation/gnunet-helper-audio-playback.c
@@ -549,7 +549,6 @@ ogg_demux_and_decode ()
  */
 static int
 stdin_receiver (void *cls,
-                void *client,
                 const struct GNUNET_MessageHeader *msg)
 {
   struct AudioMessage *audio;
@@ -727,12 +726,14 @@ ogg_init ()
   ogg_sync_init (&oy);
 }
 
+
 static void
 drain_callback (pa_stream*s, int success, void *userdata)
 {
   pa_threaded_mainloop_signal (m, 0);
 }
 
+
 /**
  * The main function for the playback helper.
  *
@@ -746,7 +747,7 @@ main (int argc, char *argv[])
   static unsigned long long toff;
 
   char readbuf[MAXLINE];
-  struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst;
+  struct GNUNET_MessageStreamTokenizer *stdin_mst;
   char c;
   ssize_t ret;
 #ifdef DEBUG_READ_PURE_OGG
@@ -762,7 +763,7 @@ main (int argc, char *argv[])
     GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "pipe");
     return 1;
   }
-  stdin_mst = GNUNET_SERVER_mst_create (&stdin_receiver, NULL);
+  stdin_mst = GNUNET_MST_create (&stdin_receiver, NULL);
   ogg_init ();
   pa_init ();
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -802,11 +803,11 @@ main (int argc, char *argv[])
     }
     else
 #endif
-    GNUNET_SERVER_mst_receive (stdin_mst, NULL,
-                               readbuf, ret,
-                               GNUNET_NO, GNUNET_NO);
+    GNUNET_MST_from_buffer (stdin_mst,
+                            readbuf, ret,
+                            GNUNET_NO, GNUNET_NO);
   }
-  GNUNET_SERVER_mst_destroy (stdin_mst);
+  GNUNET_MST_destroy (stdin_mst);
   if (stream_out)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/conversation/gnunet-helper-audio-record-gst.c b/src/conversation/gnunet-helper-audio-record-gst.c
index 7f1f3368b..b9aed4ea7 100644
--- a/src/conversation/gnunet-helper-audio-record-gst.c
+++ b/src/conversation/gnunet-helper-audio-record-gst.c
@@ -352,9 +352,12 @@ main (int argc, char **argv)
         {
           if (-1 == ret)
             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Failed to write %u bytes at offset %u (total %u) in phase %d: %s\n",
-                (unsigned int) to_send - offset, (unsigned int) offset,
-                (unsigned int) (to_send + offset), phase, strerror (errno));
+                        "Failed to write %u bytes at offset %u (total %u) in phase %d: %s\n",
+                        (unsigned int) (to_send - offset),
+                        (unsigned int) offset,
+                        (unsigned int) (to_send + offset),
+                        phase,
+                        strerror (errno));
           abort_send = 1;
           break;
         }
diff --git a/src/conversation/gnunet-service-conversation.c b/src/conversation/gnunet-service-conversation.c
index 08458f278..5f43bfe80 100644
--- a/src/conversation/gnunet-service-conversation.c
+++ b/src/conversation/gnunet-service-conversation.c
@@ -1,6 +1,6 @@
 /*
   This file is part of GNUnet.
-  Copyright (C) 2013, 2016 GNUnet e.V.
+  Copyright (C) 2013, 2016, 2017 GNUnet e.V.
 
   GNUnet is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published
@@ -226,9 +226,9 @@ static struct Channel *
 find_channel_by_line (struct Line *line,
                       uint32_t cid)
 {
-  struct Channel *ch;
-
-  for (ch = line->channel_head; NULL != ch; ch = ch->next)
+  for (struct Channel *ch = line->channel_head;
+       NULL != ch;
+       ch = ch->next)
     if (cid == ch->cid)
       return ch;
   return NULL;
@@ -314,11 +314,6 @@ destroy_line_cadet_channels (struct Channel *ch)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Destroying cadet channels\n");
-  if (NULL != ch->mq)
-  {
-    GNUNET_MQ_destroy (ch->mq);
-    ch->mq = NULL;
-  }
   if (NULL != ch->channel)
     GNUNET_CADET_channel_destroy (ch->channel);
 }
@@ -586,59 +581,6 @@ handle_client_resume_message (void *cls,
 
 
 /**
- * Function to handle call request from the client
- *
- * @param cls the `struct Line` the message is about
- * @param msg the message from the client
- */
-static void
-handle_client_call_message (void *cls,
-                            const struct ClientCallMessage *msg)
-{
-  struct Line *line = cls;
-  struct Channel *ch;
-  struct GNUNET_MQ_Envelope *e;
-  struct CadetPhoneRingMessage *ring;
-  struct CadetPhoneRingInfoPS rs;
-
-  line->line_port = msg->line_port;
-  rs.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_CONVERSATION_RING);
-  rs.purpose.size = htonl (sizeof (struct CadetPhoneRingInfoPS));
-  rs.line_port = line->line_port;
-  rs.target_peer = msg->target;
-  rs.expiration_time
-    = GNUNET_TIME_absolute_hton (GNUNET_TIME_relative_to_absolute (RING_TIMEOUT));
-
-  ch = GNUNET_new (struct Channel);
-  ch->line = line;
-  GNUNET_CONTAINER_DLL_insert (line->channel_head,
-                               line->channel_tail,
-                               ch);
-  ch->status = CS_CALLER_CALLING;
-  ch->channel = GNUNET_CADET_channel_create (cadet,
-                                             ch,
-                                             &msg->target,
-                                             &msg->line_port,
-                                             GNUNET_CADET_OPTION_RELIABLE);
-  ch->mq = GNUNET_CADET_mq_create (ch->channel);
-  e = GNUNET_MQ_msg (ring,
-                     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RING);
-  GNUNET_CRYPTO_ecdsa_key_get_public (&msg->caller_id,
-                                      &ring->caller_id);
-  ring->expiration_time = rs.expiration_time;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CRYPTO_ecdsa_sign (&msg->caller_id,
-                                           &rs.purpose,
-                                           &ring->signature));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending RING message via CADET\n");
-  GNUNET_MQ_send (ch->mq,
-                  e);
-  GNUNET_SERVICE_client_continue (line->client);
-}
-
-
-/**
  * Transmission of audio data via cadet channel finished.
  *
  * @param cls the `struct Channel` we are transmitting for
@@ -678,7 +620,7 @@ handle_client_audio_message (void *cls,
                              const struct ClientAudioMessage *msg)
 {
   struct Line *line = cls;
-  struct ClientAudioMessage *mam;
+  struct CadetAudioMessage *mam;
   struct Channel *ch;
   size_t size;
 
@@ -730,6 +672,10 @@ handle_client_audio_message (void *cls,
     ch->env = NULL;
   }
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received %u bytes of AUDIO data from client CID %u\n",
+              (unsigned int) size,
+              msg->cid);
   ch->env = GNUNET_MQ_msg_extra (mam,
                                  size,
                                  GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_AUDIO);
@@ -750,26 +696,18 @@ handle_client_audio_message (void *cls,
  * Function to handle a ring message incoming over cadet
  *
  * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
- * @param message the incoming message
- * @return #GNUNET_OK
+ * @param msg the incoming message
  */
-static int
+static void
 handle_cadet_ring_message (void *cls,
-                           struct GNUNET_CADET_Channel *channel,
-                           void **channel_ctx,
-                           const struct GNUNET_MessageHeader *message)
+                           const struct CadetPhoneRingMessage *msg)
 {
-  struct Channel *ch = *channel_ctx;
+  struct Channel *ch = cls;
   struct Line *line = ch->line;
-  const struct CadetPhoneRingMessage *msg;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhoneRingMessage *cring;
   struct CadetPhoneRingInfoPS rs;
 
-  msg = (const struct CadetPhoneRingMessage *) message;
   rs.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_CONVERSATION_RING);
   rs.purpose.size = htonl (sizeof (struct CadetPhoneRingInfoPS));
   rs.line_port = line->line_port;
@@ -783,7 +721,8 @@ handle_cadet_ring_message (void *cls,
                                   &msg->caller_id))
   {
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    destroy_line_cadet_channels (ch);
+    return;
   }
   if (0 == GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_ntoh (msg->expiration_time)).rel_value_us)
   {
@@ -792,14 +731,16 @@ handle_cadet_ring_message (void *cls,
     /* Note that our reliance on time here is awkward; better would be
        to use a more complex challenge-response protocol against
        replay attacks.  Left for future work ;-). */
-    return GNUNET_SYSERR;
+    destroy_line_cadet_channels (ch);
+    return;
   }
   if (CS_CALLEE_INIT != ch->status)
   {
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    destroy_line_cadet_channels (ch);
+    return;
   }
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   ch->status = CS_CALLEE_RINGING;
   env = GNUNET_MQ_msg (cring,
                        GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_RING);
@@ -810,34 +751,27 @@ handle_cadet_ring_message (void *cls,
               (unsigned int) ch->cid);
   GNUNET_MQ_send (line->mq,
                   env);
-  return GNUNET_OK;
 }
 
 
 /**
  * Function to handle a hangup message incoming over cadet
  *
- * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
+ * @param cls closure, our `struct Channel *`
  * @param message the incoming message
- * @return #GNUNET_OK
  */
-static int
+static void
 handle_cadet_hangup_message (void *cls,
-                             struct GNUNET_CADET_Channel *channel,
-                             void **channel_ctx,
-                             const struct GNUNET_MessageHeader *message)
+                             const struct CadetPhoneHangupMessage *message)
 {
-  struct Channel *ch = *channel_ctx;
+  struct Channel *ch = cls;
   struct Line *line = ch->line;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhoneHangupMessage *hup;
   enum ChannelStatus status;
   uint32_t cid;
 
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   cid = ch->cid;
   status = ch->status;
   destroy_line_cadet_channels (ch);
@@ -845,17 +779,17 @@ handle_cadet_hangup_message (void *cls,
   {
   case CS_CALLEE_INIT:
     GNUNET_break_op (0);
-    return GNUNET_OK;
+    return;
   case CS_CALLEE_RINGING:
   case CS_CALLEE_CONNECTED:
     break;
   case CS_CALLEE_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   case CS_CALLER_CALLING:
   case CS_CALLER_CONNECTED:
     break;
   case CS_CALLER_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Sending HANG UP message to client\n");
@@ -864,33 +798,25 @@ handle_cadet_hangup_message (void *cls,
   hup->cid = cid;
   GNUNET_MQ_send (line->mq,
                   env);
-  return GNUNET_OK;
 }
 
 
 /**
  * Function to handle a pickup message incoming over cadet
  *
- * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
+ * @param cls closure, our `struct Channel *`
  * @param message the incoming message
- * @return #GNUNET_OK if message was OK,
- *         #GNUNET_SYSERR if message violated the protocol
  */
-static int
+static void
 handle_cadet_pickup_message (void *cls,
-                             struct GNUNET_CADET_Channel *channel,
-                             void **channel_ctx,
-                             const struct GNUNET_MessageHeader *message)
+                             const struct CadetPhonePickupMessage *message)
 {
-  struct Channel *ch = *channel_ctx;
+  struct Channel *ch = cls;
   struct Line *line = ch->line;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhonePickedupMessage *pick;
 
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   switch (ch->status)
   {
   case CS_CALLEE_INIT:
@@ -898,21 +824,21 @@ handle_cadet_pickup_message (void *cls,
   case CS_CALLEE_CONNECTED:
     GNUNET_break_op (0);
     destroy_line_cadet_channels (ch);
-    return GNUNET_SYSERR;
+    return;
   case CS_CALLEE_SHUTDOWN:
     GNUNET_break_op (0);
     destroy_line_cadet_channels (ch);
-    return GNUNET_SYSERR;
+    return;
   case CS_CALLER_CALLING:
     ch->status = CS_CALLER_CONNECTED;
     break;
   case CS_CALLER_CONNECTED:
     GNUNET_break_op (0);
-    return GNUNET_OK;
+    return;
   case CS_CALLER_SHUTDOWN:
     GNUNET_break_op (0);
     mq_done_finish_caller_shutdown (ch);
-    return GNUNET_SYSERR;
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Sending PICKED UP message to client\n");
@@ -921,32 +847,25 @@ handle_cadet_pickup_message (void *cls,
   pick->cid = ch->cid;
   GNUNET_MQ_send (line->mq,
                   env);
-  return GNUNET_OK;
 }
 
 
 /**
  * Function to handle a suspend message incoming over cadet
  *
- * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
+ * @param cls closure, our `struct Channel *`
  * @param message the incoming message
- * @return #GNUNET_OK
  */
-static int
+static void
 handle_cadet_suspend_message (void *cls,
-                              struct GNUNET_CADET_Channel *channel,
-                              void **channel_ctx,
-                              const struct GNUNET_MessageHeader *message)
+                              const struct CadetPhoneSuspendMessage *message)
 {
-  struct Channel *ch = *channel_ctx;
+  struct Channel *ch = cls;
   struct Line *line = ch->line;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhoneSuspendMessage *suspend;
 
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Suspending channel CID: %u\n",
               ch->cid);
@@ -962,7 +881,7 @@ handle_cadet_suspend_message (void *cls,
     ch->suspended_remote = GNUNET_YES;
     break;
   case CS_CALLEE_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   case CS_CALLER_CALLING:
     GNUNET_break_op (0);
     break;
@@ -970,51 +889,39 @@ handle_cadet_suspend_message (void *cls,
     ch->suspended_remote = GNUNET_YES;
     break;
   case CS_CALLER_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   }
   env = GNUNET_MQ_msg (suspend,
                        GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_SUSPEND);
   suspend->cid = ch->cid;
   GNUNET_MQ_send (line->mq,
                   env);
-  return GNUNET_OK;
 }
 
 
 /**
  * Function to handle a resume message incoming over cadet
  *
- * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
- * @param message the incoming message
- * @return #GNUNET_OK
+ * @param cls closure, our `struct Channel *`
+ * @param msg the incoming message
  */
-static int
+static void
 handle_cadet_resume_message (void *cls,
-                             struct GNUNET_CADET_Channel *channel,
-                             void **channel_ctx,
-                             const struct GNUNET_MessageHeader *message)
+                             const struct CadetPhoneResumeMessage *msg)
 {
-  struct Channel *ch = *channel_ctx;
+  struct Channel *ch = cls;
   struct Line *line;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhoneResumeMessage *resume;
 
-  if (NULL == ch)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "RESUME message received for non-existing line, dropping channel.\n");
-    return GNUNET_SYSERR;
-  }
   line = ch->line;
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   if (GNUNET_YES != ch->suspended_remote)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "RESUME message received for non-suspended channel, dropping channel.\n");
-    return GNUNET_SYSERR;
+    destroy_line_cadet_channels (ch);
+    return;
   }
   switch (ch->status)
   {
@@ -1028,7 +935,7 @@ handle_cadet_resume_message (void *cls,
     ch->suspended_remote = GNUNET_NO;
     break;
   case CS_CALLEE_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   case CS_CALLER_CALLING:
     GNUNET_break (0);
     break;
@@ -1036,41 +943,47 @@ handle_cadet_resume_message (void *cls,
     ch->suspended_remote = GNUNET_NO;
     break;
   case CS_CALLER_SHUTDOWN:
-    return GNUNET_OK;
+    return;
   }
   env = GNUNET_MQ_msg (resume,
                        GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_RESUME);
   resume->cid = ch->cid;
   GNUNET_MQ_send (line->mq,
                   env);
-  return GNUNET_OK;
 }
 
 
 /**
- * Function to handle an audio message incoming over cadet
+ * Function to check an audio message incoming over cadet
  *
- * @param cls closure, NULL
- * @param channel the channel over which the message arrived
- * @param channel_ctx the channel context, can be NULL
- *                    or point to the `struct Channel`
- * @param message the incoming message
- * @return #GNUNET_OK
+ * @param cls closure, our `struct Channel *`
+ * @param msg the incoming message
+ * @return #GNUNET_OK (always)
  */
 static int
+check_cadet_audio_message (void *cls,
+                           const struct CadetAudioMessage *msg)
+{
+  return GNUNET_OK; /* any payload is fine */
+}
+
+
+/**
+ * Function to handle an audio message incoming over cadet
+ *
+ * @param cls closure, our `struct Channel *`
+ * @param msg the incoming message
+ */
+static void
 handle_cadet_audio_message (void *cls,
-                            struct GNUNET_CADET_Channel *channel,
-                            void **channel_ctx,
-                            const struct GNUNET_MessageHeader *message)
+                            const struct CadetAudioMessage *msg)
 {
-  struct Channel *ch = *channel_ctx;
-  const struct CadetAudioMessage *msg;
-  size_t msize = ntohs (message->size) - sizeof (struct CadetAudioMessage);
+  struct Channel *ch = cls;
+  size_t msize = ntohs (msg->header.size) - sizeof (struct CadetAudioMessage);
   struct GNUNET_MQ_Envelope *env;
   struct ClientAudioMessage *cam;
 
-  msg = (const struct CadetAudioMessage *) message;
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (ch->channel);
   if ( (GNUNET_YES == ch->suspended_local) ||
        (GNUNET_YES == ch->suspended_remote) )
   {
@@ -1078,7 +991,7 @@ handle_cadet_audio_message (void *cls,
                 "Received %u bytes of AUDIO data on suspended channel CID %u; dropping\n",
                 (unsigned int) msize,
                 ch->cid);
-    return GNUNET_OK;
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Forwarding %u bytes of AUDIO data to client CID %u\n",
@@ -1093,44 +1006,6 @@ handle_cadet_audio_message (void *cls,
                  msize);
   GNUNET_MQ_send (ch->line->mq,
                   env);
-  return GNUNET_OK;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- *
- * @param cls the `struct Line` receiving a connection
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port port
- * @param options channel option flags
- * @return initial channel context for the channel
- */
-static void *
-inbound_channel (void *cls,
-                 struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *initiator,
-                 const struct GNUNET_HashCode *port,
-                 enum GNUNET_CADET_ChannelOption options)
-{
-  struct Line *line = cls;
-  struct Channel *ch;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received incoming cadet channel on line %p\n",
-              line);
-  ch = GNUNET_new (struct Channel);
-  ch->status = CS_CALLEE_INIT;
-  ch->line = line;
-  ch->channel = channel;
-  ch->mq = GNUNET_CADET_mq_create (ch->channel);
-  ch->cid = line->cid_gen++;
-  GNUNET_CONTAINER_DLL_insert (line->channel_head,
-                               line->channel_tail,
-                               ch);
-  return ch;
 }
 
 
@@ -1140,26 +1015,16 @@ inbound_channel (void *cls,
  *
  * @param cls closure (set from #GNUNET_CADET_connect)
  * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored;
- *                   may point to the `struct Channel`
  */
 static void
 inbound_end (void *cls,
-             const struct GNUNET_CADET_Channel *channel,
-             void *channel_ctx)
+             const struct GNUNET_CADET_Channel *channel)
 {
-  struct Channel *ch = channel_ctx;
-  struct Line *line;
+  struct Channel *ch = cls;
+  struct Line *line = ch->line;
   struct GNUNET_MQ_Envelope *env;
   struct ClientPhoneHangupMessage *hup;
 
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  line = ch->line;
   GNUNET_assert (channel == ch->channel);
   ch->channel = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1185,7 +1050,6 @@ inbound_end (void *cls,
     }
     break;
   }
-  destroy_line_cadet_channels (ch);
   if (NULL != line)
     GNUNET_CONTAINER_DLL_remove (line->channel_head,
                                  line->channel_tail,
@@ -1195,6 +1059,116 @@ inbound_end (void *cls,
 
 
 /**
+ * Function to handle call request from the client
+ *
+ * @param cls the `struct Line` the message is about
+ * @param msg the message from the client
+ */
+static void
+handle_client_call_message (void *cls,
+                            const struct ClientCallMessage *msg)
+{
+  struct Line *line = cls;
+  struct Channel *ch = GNUNET_new (struct Channel);
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (cadet_hangup_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_HANG_UP,
+                             struct CadetPhoneHangupMessage,
+                             ch),
+    GNUNET_MQ_hd_fixed_size (cadet_pickup_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_PICK_UP,
+                             struct CadetPhonePickupMessage,
+                             ch),
+    GNUNET_MQ_hd_fixed_size (cadet_suspend_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_SUSPEND,
+                             struct CadetPhoneSuspendMessage,
+                             ch),
+    GNUNET_MQ_hd_fixed_size (cadet_resume_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RESUME,
+                             struct CadetPhoneResumeMessage,
+                             ch),
+    GNUNET_MQ_hd_var_size (cadet_audio_message,
+                           GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_AUDIO,
+                           struct CadetAudioMessage,
+                           ch),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MQ_Envelope *e;
+  struct CadetPhoneRingMessage *ring;
+  struct CadetPhoneRingInfoPS rs;
+
+  line->line_port = msg->line_port;
+  rs.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_CONVERSATION_RING);
+  rs.purpose.size = htonl (sizeof (struct CadetPhoneRingInfoPS));
+  rs.line_port = line->line_port;
+  rs.target_peer = msg->target;
+  rs.expiration_time
+    = GNUNET_TIME_absolute_hton (GNUNET_TIME_relative_to_absolute (RING_TIMEOUT));
+  ch->line = line;
+  GNUNET_CONTAINER_DLL_insert (line->channel_head,
+                               line->channel_tail,
+                               ch);
+  ch->status = CS_CALLER_CALLING;
+  ch->channel = GNUNET_CADET_channel_create (cadet,
+                                             ch,
+                                             &msg->target,
+                                             &msg->line_port,
+                                             GNUNET_CADET_OPTION_RELIABLE,
+                                             NULL,
+                                             &inbound_end,
+                                             cadet_handlers);
+  ch->mq = GNUNET_CADET_get_mq (ch->channel);
+  e = GNUNET_MQ_msg (ring,
+                     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RING);
+  GNUNET_CRYPTO_ecdsa_key_get_public (&msg->caller_id,
+                                      &ring->caller_id);
+  ring->expiration_time = rs.expiration_time;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdsa_sign (&msg->caller_id,
+                                           &rs.purpose,
+                                           &ring->signature));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending RING message via CADET\n");
+  GNUNET_MQ_send (ch->mq,
+                  e);
+  GNUNET_SERVICE_client_continue (line->client);
+}
+
+
+/**
+ * Method called whenever another peer has added us to a channel
+ * the other peer initiated.
+ *
+ * @param cls the `struct Line` receiving a connection
+ * @param channel new handle to the channel
+ * @param initiator peer that started the channel
+ * @return initial channel context for the channel
+ */
+static void *
+inbound_channel (void *cls,
+                 struct GNUNET_CADET_Channel *channel,
+                 const struct GNUNET_PeerIdentity *initiator)
+{
+  struct Line *line = cls;
+  struct Channel *ch;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received incoming cadet channel on line %p\n",
+              line);
+  ch = GNUNET_new (struct Channel);
+  ch->status = CS_CALLEE_INIT;
+  ch->line = line;
+  ch->channel = channel;
+  ch->mq = GNUNET_CADET_get_mq (ch->channel);
+  ch->cid = line->cid_gen++;
+  GNUNET_CONTAINER_DLL_insert (line->channel_head,
+                               line->channel_tail,
+                               ch);
+  return ch;
+}
+
+
+/**
  * A client connected.  Initialize the `struct Line` data structure.
  *
  * @param cls closure, NULL
@@ -1260,12 +1234,42 @@ handle_client_register_message (void *cls,
                                 const struct ClientPhoneRegisterMessage *msg)
 {
   struct Line *line = cls;
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (cadet_ring_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RING,
+                             struct CadetPhoneRingMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (cadet_hangup_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_HANG_UP,
+                             struct CadetPhoneHangupMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (cadet_pickup_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_PICK_UP,
+                             struct CadetPhonePickupMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (cadet_suspend_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_SUSPEND,
+                             struct CadetPhoneSuspendMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (cadet_resume_message,
+                             GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RESUME,
+                             struct CadetPhoneResumeMessage,
+                             NULL),
+    GNUNET_MQ_hd_var_size (cadet_audio_message,
+                           GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_AUDIO,
+                           struct CadetAudioMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
 
   line->line_port = msg->line_port;
   line->port = GNUNET_CADET_open_port (cadet,
                                        &msg->line_port,
                                        &inbound_channel,
-                                       line);
+                                       line,
+                                       NULL,
+                                       &inbound_end,
+                                       cadet_handlers);
   GNUNET_SERVICE_client_continue (line->client);
 }
 
@@ -1298,35 +1302,11 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    {&handle_cadet_ring_message,
-     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RING,
-     sizeof (struct CadetPhoneRingMessage)},
-    {&handle_cadet_hangup_message,
-     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_HANG_UP,
-     sizeof (struct CadetPhoneHangupMessage)},
-    {&handle_cadet_pickup_message,
-     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_PICK_UP,
-     sizeof (struct CadetPhonePickupMessage)},
-    {&handle_cadet_suspend_message,
-     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_SUSPEND,
-     sizeof (struct CadetPhoneSuspendMessage)},
-    {&handle_cadet_resume_message,
-     GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_PHONE_RESUME,
-     sizeof (struct CadetPhoneResumeMessage)},
-    {&handle_cadet_audio_message, GNUNET_MESSAGE_TYPE_CONVERSATION_CADET_AUDIO,
-     0},
-    {NULL, 0, 0}
-  };
-
   cfg = c;
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CRYPTO_get_peer_identity (cfg,
                                                   &my_identity));
-  cadet = GNUNET_CADET_connect (cfg,
-                                NULL,
-                                &inbound_end,
-                                cadet_handlers);
+  cadet = GNUNET_CADET_connect (cfg);
   if (NULL == cadet)
   {
     GNUNET_break (0);
@@ -1338,7 +1318,6 @@ run (void *cls,
 }
 
 
-
 /**
  * Define "main" method using service macro.
  */
@@ -1353,26 +1332,26 @@ GNUNET_SERVICE_MAIN
                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_REGISTER,
                           struct ClientPhoneRegisterMessage,
                           NULL),
-  GNUNET_MQ_hd_fixed_size (client_pickup_message,
-                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_PICK_UP,
-                           struct ClientPhonePickupMessage,
-                           NULL),
-  GNUNET_MQ_hd_fixed_size (client_suspend_message,
-                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_SUSPEND,
-                           struct ClientPhoneSuspendMessage,
-                           NULL),
-  GNUNET_MQ_hd_fixed_size (client_resume_message,
-                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_RESUME,
-                           struct ClientPhoneResumeMessage,
-                           NULL),
-  GNUNET_MQ_hd_fixed_size (client_hangup_message,
-                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_HANG_UP,
-                           struct ClientPhoneHangupMessage,
-                           NULL),
-  GNUNET_MQ_hd_fixed_size (client_call_message,
-                           GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_CALL,
-                           struct ClientCallMessage,
-                           NULL),
+ GNUNET_MQ_hd_fixed_size (client_pickup_message,
+                          GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_PICK_UP,
+                          struct ClientPhonePickupMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_suspend_message,
+                          GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_SUSPEND,
+                          struct ClientPhoneSuspendMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_resume_message,
+                          GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_RESUME,
+                          struct ClientPhoneResumeMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_hangup_message,
+                          GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_HANG_UP,
+                          struct ClientPhoneHangupMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_call_message,
+                          GNUNET_MESSAGE_TYPE_CONVERSATION_CS_PHONE_CALL,
+                          struct ClientCallMessage,
+                          NULL),
  GNUNET_MQ_hd_var_size (client_audio_message,
                         GNUNET_MESSAGE_TYPE_CONVERSATION_CS_AUDIO,
                         struct ClientAudioMessage,
diff --git a/src/conversation/gnunet_gst.c b/src/conversation/gnunet_gst.c
index 91c6ddccb..52cb2ccbc 100644
--- a/src/conversation/gnunet_gst.c
+++ b/src/conversation/gnunet_gst.c
@@ -29,6 +29,7 @@
  */
 static struct GNUNET_CONFIGURATION_Handle *cfg;
 
+
 void
 dump_buffer(unsigned n, const unsigned char* buf)
 {
@@ -61,15 +62,13 @@ BREAKOUT:
 /***
  * load gnunet configuration
  */
-  void
+void
 gg_load_configuration(GNUNET_gstData * d)
 {
   char *audiobackend_string;
   cfg =  GNUNET_CONFIGURATION_create();
   GNUNET_CONFIGURATION_load(cfg, "mediahelper.conf");
 
-  char *section = "MEDIAHELPER";
-
   GNUNET_CONFIGURATION_get_value_string(cfg, "MEDIAHELPER", "JACK_PP_IN", &d->jack_pp_in);
   GNUNET_CONFIGURATION_get_value_string(cfg, "MEDIAHELPER", "JACK_PP_OUT", &d->jack_pp_out);
 
@@ -77,19 +76,19 @@ gg_load_configuration(GNUNET_gstData * d)
 
  // printf("abstring: %s \n", audiobackend_string);
 
-  if ( audiobackend_string == "AUTO" )
+  if (0 == strcasecmp (audiobackend_string, "AUTO"))
   {
     d->audiobackend = AUTO;
-  } else if ( audiobackend_string = "JACK" )
+  } else if (0 == strcasecmp (audiobackend_string, "JACK"))
   {
     d->audiobackend = JACK;
-  } else if ( audiobackend_string = "ALSA" )
+  } else if (0 == strcasecmp (audiobackend_string, "ALSA"))
   {
     d->audiobackend = ALSA;
-  } else if ( audiobackend_string = "FAKE" )
+  } else if (0 == strcasecmp (audiobackend_string, "FAKE"))
   {
     d->audiobackend = FAKE;
-  } else if ( audiobackend_string = "TEST" )
+  } else if (0 == strcasecmp (audiobackend_string, "TEST"))
   {
     d->audiobackend = TEST;
   } else
@@ -147,8 +146,6 @@ write_data (const char *ptr, size_t msg_size)
 extern GstFlowReturn
 on_appsink_new_sample (GstElement * element, GNUNET_gstData * d)
 {
-  static unsigned long long toff;
-
   //size of message including gnunet header
   size_t msg_size;
 
@@ -161,7 +158,6 @@ on_appsink_new_sample (GstElement * element, GNUNET_gstData * d)
   GstCaps *s_caps;
   char *caps_str;
 */
-  (d->audio_message)->header.size = htons ((uint16_t) msg_size);
 
   if (gst_app_sink_is_eos(GST_APP_SINK(element)))
     return GST_FLOW_OK;
@@ -197,13 +193,7 @@ on_appsink_new_sample (GstElement * element, GNUNET_gstData * d)
 
   // copy the data into audio_message
   GNUNET_memcpy (((char *) &(d->audio_message)[1]), map.data, len);
-/*
-  toff += msg_size;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending %u bytes of audio data (total: %llu)\n",
-              (unsigned int) msg_size,
-              toff);
-*/
+  (d->audio_message)->header.size = htons ((uint16_t) msg_size);
   if (d->pure_ogg)
     // write the audio_message without the gnunet headers
     write_data ((const char *) &(d->audio_message)[1], len);
@@ -589,14 +579,15 @@ autoaudiosource_child_added (GstChildProxy *child_proxy, GObject *object, gchar
     g_object_set (object, "buffer-time", (gint64) BUFFER_TIME, "latency-time", (gint64) LATENCY_TIME, NULL);
 }
 
+
 GstElement *
 get_pipeline(GstElement *element)
 {
   GstPipeline *p;
 
-  p = gst_object_get_parent(element);
+  p = GST_PIPELINE (gst_object_get_parent(GST_OBJECT (element)));
 
-  return p;
+  return GST_ELEMENT (p);
 }
 
   static void
@@ -617,6 +608,7 @@ decoder_ogg_pad_added (GstElement *element,
   gst_object_unref (sinkpad);
 }
 
+
 int
 gnunet_read (GNUNET_gstData * d)
 {
@@ -646,10 +638,13 @@ gnunet_read (GNUNET_gstData * d)
      else
      {
   //#endif
-  GNUNET_SERVER_mst_receive (d->stdin_mst, NULL,
-      readbuf, ret,
-      GNUNET_NO, GNUNET_NO);
+  GNUNET_MST_from_buffer (d->stdin_mst,
+                          readbuf,
+                          ret,
+                          GNUNET_NO,
+                          GNUNET_NO);
      }
+     return 0;
 }
 
 /**
@@ -657,13 +652,14 @@ gnunet_read (GNUNET_gstData * d)
  */
 static int
 stdin_receiver (void *cls,
-                void *client,
                 const struct GNUNET_MessageHeader *msg)
 {
   struct AudioMessage *audio;
   size_t b_len;
+
   printf("stdin receiver \n ");
-  dump_buffer(sizeof(msg), msg);
+  dump_buffer (sizeof(msg),
+               (const unsigned char *) msg);
 
   switch (ntohs (msg->type))
   {
@@ -711,7 +707,7 @@ get_app(GNUNET_gstData *d, int type)
  //d->audio_message.header.type = htons (GNUNET_MESSAGE_TYPE_CONVERSATION_AUDIO);
 
 
-    d->stdin_mst = GNUNET_SERVER_mst_create (&stdin_receiver, d);
+    d->stdin_mst = GNUNET_MST_create (&stdin_receiver, d);
 
     if ( d->stdin_mst == NULL)
      printf("stdin_mst = NULL");
@@ -768,7 +764,7 @@ get_coder(GNUNET_gstData *d , int type)
 {
   GstBin *bin;
   GstPad *srcpad, *sinkpad, *srcghostpad, *sinkghostpad;
-  GstCaps *caps, *rtpcaps;
+  GstCaps *rtpcaps;
   GstElement *encoder, *muxer, *decoder, *demuxer, *jitterbuffer, *rtpcapsfilter;
 
   if ( d->usertp == TRUE )
@@ -892,7 +888,9 @@ get_coder(GNUNET_gstData *d , int type)
 
   return bin;
 }
-  extern GstBin *
+
+
+extern GstBin *
 get_audiobin(GNUNET_gstData *d , int type)
 {
   GstBin *bin;
diff --git a/src/conversation/gnunet_gst_def.h b/src/conversation/gnunet_gst_def.h
index 9f519b564..980ed2f48 100644
--- a/src/conversation/gnunet_gst_def.h
+++ b/src/conversation/gnunet_gst_def.h
@@ -65,20 +65,23 @@
 #include <gst/app/gstappsink.h>
 
 // sockets
+#ifndef MINGW
+#if HAVE_NETINET_IN_H
 #include <netinet/in.h>
+#endif
 #include <sys/socket.h>
-#include <fcntl.h>
 #include <arpa/inet.h>
+#include <netdb.h>
+#endif
 
 #include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
+#include <fcntl.h>
 
 
 //glib stuff
 //#include <glib.h>
 #include <glib-2.0/glib/gprintf.h>
-#include <glib-unix.h>
+//#include <glib-unix.h>
 
 // static struct AudioMessage *audio_message;
 
@@ -91,7 +94,7 @@ struct GNUNET_gstData {
 
   // things
   struct AudioMessage *audio_message;
-  struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst;
+  struct GNUNET_MessageStreamTokenizer *stdin_mst;
   GstElement *appsrc;
   GstElement *appsink;
   //settings
diff --git a/src/conversation/gnunet_gst_test.c b/src/conversation/gnunet_gst_test.c
index 2ea007583..03f6a4d95 100644
--- a/src/conversation/gnunet_gst_test.c
+++ b/src/conversation/gnunet_gst_test.c
@@ -18,7 +18,7 @@
   Boston, MA 02110-1301, USA.
 */
 /**
- * @file conversation/gnunet_gst.c
+ * @file conversation/gnunet_gst_test.c
  * @brief FIXME
  * @author Hark
  */
@@ -30,8 +30,7 @@ int
 main (int argc, char *argv[])
 {
   struct GNUNET_gstData *gst;
-  GstBus *bus;
-  GstMessage *msg;
+  // GstBus *bus;
   GstElement *gnunetsrc, *gnunetsink, *source, *sink, *encoder, *decoder;
 
 
@@ -137,7 +136,7 @@ main (int argc, char *argv[])
  }
   g_print ("Returned, stopping playback\n");
 
-  gst_object_unref (bus);
+  // gst_object_unref (bus);
   gst_element_set_state (GST_ELEMENT(gst->pipeline), GST_STATE_NULL);
   gst_object_unref (gst->pipeline);
 
diff --git a/src/conversation/microphone.c b/src/conversation/microphone.c
index 94f52f8dc..7871433a3 100644
--- a/src/conversation/microphone.c
+++ b/src/conversation/microphone.c
@@ -64,13 +64,11 @@ struct Microphone
  * Function to process the audio from the record helper
  *
  * @param cls clsoure with our `struct Microphone`
- * @param client NULL
  * @param msg the message from the helper
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
 process_record_messages (void *cls,
-                         void *client,
                          const struct GNUNET_MessageHeader *msg)
 {
   struct Microphone *mic = cls;
diff --git a/src/conversation/test_conversation.conf b/src/conversation/test_conversation.conf
index 46ea95d52..e0fd8200c 100644
--- a/src/conversation/test_conversation.conf
+++ b/src/conversation/test_conversation.conf
@@ -2,6 +2,7 @@
 
 [conversation]
 LINE=1
+PREFIX = valgrind
 
 [nse]
 WORKBITS = 0
diff --git a/src/conversation/test_conversation_api_twocalls.c b/src/conversation/test_conversation_api_twocalls.c
index 2e2a73e3f..7d2705e70 100644
--- a/src/conversation/test_conversation_api_twocalls.c
+++ b/src/conversation/test_conversation_api_twocalls.c
@@ -187,7 +187,8 @@ play (void *cls,
     phone_i++;
   else
   {
-    LOG_DEBUG ("Received unexpected data %.*s\n",
+    LOG_DEBUG ("Received %u bytes of unexpected data `%.*s'\n",
+               (unsigned int) data_size,
                (int) data_size,
                (const char *) data);
   }
diff --git a/src/core/Makefile.am b/src/core/Makefile.am
index ed80bae73..2723151c5 100644
--- a/src/core/Makefile.am
+++ b/src/core/Makefile.am
@@ -71,7 +71,7 @@ check_PROGRAMS = \
  $(TESTING_TESTS)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/core/core_api.c b/src/core/core_api.c
index b6d8f61d2..ed8ce364d 100644
--- a/src/core/core_api.c
+++ b/src/core/core_api.c
@@ -204,6 +204,11 @@ disconnect_and_free_peer_entry (void *cls,
                                                        pr));
   GNUNET_MQ_destroy (pr->mq);
   GNUNET_assert (NULL == pr->mq);
+  if (NULL != pr->env)
+  {
+    GNUNET_MQ_discard (pr->env);
+    pr->env = NULL;
+  }
   GNUNET_free (pr);
   return GNUNET_YES;
 }
@@ -316,7 +321,7 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
 
   /* check message size for sanity */
   msize = ntohs (msg->size);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct SendMessage))
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct SendMessage))
   {
     GNUNET_break (0);
     GNUNET_MQ_impl_send_continue (mq);
@@ -331,7 +336,6 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
   env = GNUNET_MQ_msg (smr,
                        GNUNET_MESSAGE_TYPE_CORE_SEND_REQUEST);
   smr->priority = htonl ((uint32_t) priority);
-  // smr->deadline = GNUNET_TIME_absolute_hton (deadline);
   smr->peer = pr->peer;
   smr->reserved = htonl (0);
   smr->size = htons (msize);
@@ -344,7 +348,6 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
                                      GNUNET_MESSAGE_TYPE_CORE_SEND,
                                      msg);
   sm->priority = htonl ((uint32_t) priority);
-  // sm->deadline = GNUNET_TIME_absolute_hton (deadline);
   sm->peer = pr->peer;
   sm->cork = htonl ((uint32_t) cork);
   sm->reserved = htonl (0);
@@ -781,7 +784,6 @@ GNUNET_CORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
                      const struct GNUNET_MQ_MessageHandler *handlers)
 {
   struct GNUNET_CORE_Handle *h;
-  unsigned int hcnt;
 
   h = GNUNET_new (struct GNUNET_CORE_Handle);
   h->cfg = cfg;
@@ -791,19 +793,10 @@ GNUNET_CORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
   h->disconnects = disconnects;
   h->peers = GNUNET_CONTAINER_multipeermap_create (128,
                                                    GNUNET_NO);
-  hcnt = 0;
-  if (NULL != handlers)
-    while (NULL != handlers[hcnt].cb)
-      hcnt++;
-  h->handlers = GNUNET_new_array (hcnt + 1,
-                                  struct GNUNET_MQ_MessageHandler);
-  if (NULL != handlers)
-    GNUNET_memcpy (h->handlers,
-                   handlers,
-                   hcnt * sizeof (struct GNUNET_MQ_MessageHandler));
-  h->hcnt = hcnt;
-  GNUNET_assert (hcnt <
-                 (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+  h->handlers = GNUNET_MQ_copy_handlers (handlers);
+  h->hcnt = GNUNET_MQ_count_handlers (handlers);
+  GNUNET_assert (h->hcnt <
+                 (GNUNET_MAX_MESSAGE_SIZE -
                   sizeof (struct InitMessage)) / sizeof (uint16_t));
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Connecting to CORE service\n");
@@ -842,7 +835,7 @@ GNUNET_CORE_disconnect (struct GNUNET_CORE_Handle *handle)
     GNUNET_MQ_destroy (handle->mq);
     handle->mq = NULL;
   }
-  GNUNET_free (handle->handlers);
+  GNUNET_free_non_null (handle->handlers);
   GNUNET_free (handle);
 }
 
diff --git a/src/core/gnunet-core.c b/src/core/gnunet-core.c
index d91dc304d..76b7e8ac4 100644
--- a/src/core/gnunet-core.c
+++ b/src/core/gnunet-core.c
@@ -171,10 +171,11 @@ main (int argc,
       char *const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'m', "monitor", NULL,
-     gettext_noop ("provide information about all current connections (continuously)"),
-     0, &GNUNET_GETOPT_set_one, &monitor_connections},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("provide information about all current connections (continuously)"),
+                                  &monitor_connections),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/core/gnunet-service-core.c b/src/core/gnunet-service-core.c
index b0d7b24ba..625bf9655 100644
--- a/src/core/gnunet-service-core.c
+++ b/src/core/gnunet-service-core.c
@@ -32,11 +32,19 @@
 #include "gnunet-service-core_typemap.h"
 
 /**
- * How many messages do we queue up at most for optional
- * notifications to a client?  (this can cause notifications
- * about outgoing messages to be dropped).
+ * How many messages do we queue up at most for any client? This can
+ * cause messages to be dropped if clients do not process them fast
+ * enough!  Note that this is a soft limit; we try
+ * to keep a few larger messages above the limit.
  */
-#define MAX_NOTIFY_QUEUE 1024
+#define SOFT_MAX_QUEUE 128
+
+/**
+ * How many messages do we queue up at most for any client? This can
+ * cause messages to be dropped if clients do not process them fast
+ * enough!  Note that this is the hard limit.
+ */
+#define HARD_MAX_QUEUE 256
 
 
 /**
@@ -696,6 +704,11 @@ GSC_CLIENTS_notify_client_about_neighbour (struct GSC_Client *client,
   new_match = GSC_TYPEMAP_test_match (tmap_new,
                                       client->types,
                                       client->tcnt);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Notifying client about neighbour %s (%d/%d)\n",
+              GNUNET_i2s (neighbour),
+              old_match,
+              new_match);
   if (old_match == new_match)
   {
     GNUNET_assert (old_match ==
@@ -720,7 +733,8 @@ GSC_CLIENTS_notify_client_about_neighbour (struct GSC_Client *client,
                          GNUNET_MESSAGE_TYPE_CORE_NOTIFY_CONNECT);
     cnm->reserved = htonl (0);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sending NOTIFY_CONNECT message to client.\n");
+                "Sending NOTIFY_CONNECT message about peer %s to client.\n",
+                GNUNET_i2s (neighbour));
     cnm->peer = *neighbour;
     GNUNET_MQ_send (client->mq,
                     env);
@@ -740,6 +754,9 @@ GSC_CLIENTS_notify_client_about_neighbour (struct GSC_Client *client,
     env = GNUNET_MQ_msg (dcm,
                          GNUNET_MESSAGE_TYPE_CORE_NOTIFY_DISCONNECT);
     dcm->reserved = htonl (0);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Sending NOTIFY_DISCONNECT message about peer %s to client.\n",
+                GNUNET_i2s (neighbour));
     dcm->peer = *neighbour;
     GNUNET_MQ_send (client->mq,
                     env);
@@ -790,7 +807,7 @@ GSC_CLIENTS_deliver_message (const struct GNUNET_PeerIdentity *sender,
 {
   size_t size = msize + sizeof (struct NotifyTrafficMessage);
 
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -810,6 +827,7 @@ GSC_CLIENTS_deliver_message (const struct GNUNET_PeerIdentity *sender,
     struct GNUNET_MQ_Envelope *env;
     struct NotifyTrafficMessage *ntm;
     uint16_t mtype;
+    unsigned int qlen;
     int tm;
 
     tm = type_match (ntohs (msg->type),
@@ -825,6 +843,45 @@ GSC_CLIENTS_deliver_message (const struct GNUNET_PeerIdentity *sender,
     if ( (0 != (options & GNUNET_CORE_OPTION_SEND_HDR_OUTBOUND)) &&
          (0 != (c->options & GNUNET_CORE_OPTION_SEND_FULL_OUTBOUND)) )
       continue;
+
+    /* Drop messages if:
+       1) We are above the hard limit, or
+       2) We are above the soft limit, and a coin toss limited
+          to the message size (giving larger messages a
+          proportionally higher chance of being queued) falls
+          below the threshold. The threshold is based on where
+          we are between the soft and the hard limit, scaled
+          to match the range of message sizes we usually encounter
+          (i.e. up to 32k); so a 64k message has a 50% chance of
+          being kept if we are just barely below the hard max,
+          and a 99% chance of being kept if we are at the soft max.
+       The reason is to make it more likely to drop control traffic
+       (ACK, queries) which may be cummulative or highly redundant,
+       and cheap to drop than data traffic.  */
+    qlen = GNUNET_MQ_get_length (c->mq);
+    if ( (qlen >= HARD_MAX_QUEUE) ||
+         ( (qlen > SOFT_MAX_QUEUE) &&
+           ( (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                        ntohs (msg->size)) ) <
+             (qlen - SOFT_MAX_QUEUE) * 0x8000 /
+             (HARD_MAX_QUEUE - SOFT_MAX_QUEUE) ) ) )
+    {
+      char buf[1024];
+
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
+                  "Dropping decrypted message of type %u as client is too busy (queue full)\n",
+                  (unsigned int) ntohs (msg->type));
+      GNUNET_snprintf (buf,
+                       sizeof (buf),
+                       gettext_noop ("# messages of type %u discarded (client busy)"),
+                       (unsigned int) ntohs (msg->type));
+      GNUNET_STATISTICS_update (GSC_stats,
+                                buf,
+                                1,
+                                GNUNET_NO);
+      continue;
+    }
+
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sending %u message with %u bytes to client interested in messages of type %u.\n",
                 options,
diff --git a/src/core/gnunet-service-core_kx.c b/src/core/gnunet-service-core_kx.c
index 6743ce215..8a7cada5c 100644
--- a/src/core/gnunet-service-core_kx.c
+++ b/src/core/gnunet-service-core_kx.c
@@ -261,7 +261,12 @@ struct GSC_KeyExchangeInfo
    * Message queue for sending messages to @a peer.
    */
   struct GNUNET_MQ_Handle *mq;
-  
+
+  /**
+   * Our message stream tokenizer (for encrypted payload).
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
   /**
    * PING message we transmit to the other peer.
    */
@@ -319,7 +324,7 @@ struct GSC_KeyExchangeInfo
    * last were received (good for accepting out-of-order packets and
    * estimating reliability of the connection)
    */
-  unsigned int last_packets_bitmap;
+  uint32_t last_packets_bitmap;
 
   /**
    * last sequence number received on this connection (highest)
@@ -370,11 +375,6 @@ static struct GNUNET_CRYPTO_EcdhePrivateKey *my_ephemeral_key;
 static struct EphemeralKeyMessage current_ekm;
 
 /**
- * Our message stream tokenizer (for encrypted payload).
- */
-static struct GNUNET_SERVER_MessageStreamTokenizer *mst;
-
-/**
  * DLL head.
  */
 static struct GSC_KeyExchangeInfo *kx_head;
@@ -702,6 +702,55 @@ setup_fresh_ping (struct GSC_KeyExchangeInfo *kx)
 
 
 /**
+ * Deliver P2P message to interested clients.  Invokes send twice,
+ * once for clients that want the full message, and once for clients
+ * that only want the header
+ *
+ * @param cls the `struct GSC_KeyExchangeInfo`
+ * @param m the message
+ */
+static int
+deliver_message (void *cls,
+                 const struct GNUNET_MessageHeader *m)
+{
+  struct GSC_KeyExchangeInfo *kx = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Decrypted message of type %d from %s\n",
+              ntohs (m->type),
+              GNUNET_i2s (kx->peer));
+  if (GNUNET_CORE_KX_STATE_UP != kx->status)
+  {
+    GNUNET_STATISTICS_update (GSC_stats,
+                              gettext_noop ("# PAYLOAD dropped (out of order)"),
+                              1,
+                              GNUNET_NO);
+    return GNUNET_OK;
+  }
+  switch (ntohs (m->type))
+  {
+  case GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP:
+  case GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP:
+    GSC_SESSIONS_set_typemap (kx->peer, m);
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP:
+    GSC_SESSIONS_confirm_typemap (kx->peer, m);
+    return GNUNET_OK;
+  default:
+    GSC_CLIENTS_deliver_message (kx->peer,
+                                 m,
+                                 ntohs (m->size),
+                                 GNUNET_CORE_OPTION_SEND_FULL_INBOUND);
+    GSC_CLIENTS_deliver_message (kx->peer,
+                                 m,
+                                 sizeof (struct GNUNET_MessageHeader),
+                                 GNUNET_CORE_OPTION_SEND_HDR_INBOUND);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Function called by transport to notify us that
  * a peer connected to us (on the network level).
  * Starts the key exchange with the given peer.
@@ -727,6 +776,8 @@ handle_transport_notify_connect (void *cls,
                             1,
                             GNUNET_NO);
   kx = GNUNET_new (struct GSC_KeyExchangeInfo);
+  kx->mst = GNUNET_MST_create (&deliver_message,
+                               kx);
   kx->mq = mq;
   kx->peer = pid;
   kx->set_key_retry_frequency = INITIAL_SET_KEY_RETRY_FREQUENCY;
@@ -777,7 +828,7 @@ handle_transport_notify_disconnect (void *cls,
                                     void *handler_cls)
 {
   struct GSC_KeyExchangeInfo *kx = handler_cls;
-  
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer `%s' disconnected from us.\n",
               GNUNET_i2s (peer));
@@ -801,6 +852,7 @@ handle_transport_notify_disconnect (void *cls,
   GNUNET_CONTAINER_DLL_remove (kx_head,
                                kx_tail,
                                kx);
+  GNUNET_MST_destroy (kx->mst);
   GNUNET_free (kx);
 }
 
@@ -814,7 +866,7 @@ static void
 send_ping (struct GSC_KeyExchangeInfo *kx)
 {
   struct GNUNET_MQ_Envelope *env;
-  
+
   GNUNET_STATISTICS_update (GSC_stats,
                             gettext_noop ("# PING messages transmitted"),
                             1,
@@ -1332,7 +1384,7 @@ static void
 send_key (struct GSC_KeyExchangeInfo *kx)
 {
   struct GNUNET_MQ_Envelope *env;
-  
+
   GNUNET_assert (GNUNET_CORE_KX_STATE_DOWN != kx->status);
   if (NULL != kx->retry_set_key_task)
   {
@@ -1417,24 +1469,6 @@ GSC_KX_encrypt_and_transmit (struct GSC_KeyExchangeInfo *kx,
 
 
 /**
- * Closure for #deliver_message()
- */
-struct DeliverMessageContext
-{
-
-  /**
-   * Key exchange context.
-   */
-  struct GSC_KeyExchangeInfo *kx;
-
-  /**
-   * Sender of the message.
-   */
-  const struct GNUNET_PeerIdentity *peer;
-};
-
-
-/**
  * We received an encrypted message.  Check that it is
  * well-formed (size-wise).
  *
@@ -1475,7 +1509,6 @@ handle_encrypted (void *cls,
   struct GNUNET_TIME_Absolute t;
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
   struct GNUNET_CRYPTO_AuthKey auth_key;
-  struct DeliverMessageContext dmc;
   uint16_t size = ntohs (m->header.size);
   char buf[size] GNUNET_ALIGN;
 
@@ -1520,7 +1553,7 @@ handle_encrypted (void *cls,
                    sizeof (struct GNUNET_HashCode)))
   {
     /* checksum failed */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Failed checksum validation for a message from `%s'\n",
                 GNUNET_i2s (kx->peer));
     return;
@@ -1536,7 +1569,10 @@ handle_encrypted (void *cls,
                   &m->sequence_number,
                   &buf[ENCRYPTED_HEADER_SIZE],
                   size - ENCRYPTED_HEADER_SIZE))
+  {
+    GNUNET_break_op (0);
     return;
+  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Decrypted %u bytes from %s\n",
               (unsigned int) (size - ENCRYPTED_HEADER_SIZE),
@@ -1570,7 +1606,7 @@ handle_encrypted (void *cls,
   }
   if (kx->last_sequence_number_received > snum)
   {
-    unsigned int rotbit = 1 << (kx->last_sequence_number_received - snum - 1);
+    uint32_t rotbit = 1U << (kx->last_sequence_number_received - snum - 1);
 
     if ((kx->last_packets_bitmap & rotbit) != 0)
     {
@@ -1617,15 +1653,12 @@ handle_encrypted (void *cls,
                             gettext_noop ("# bytes of payload decrypted"),
                             size - sizeof (struct EncryptedMessage),
                             GNUNET_NO);
-  dmc.kx = kx;
-  dmc.peer = kx->peer;
   if (GNUNET_OK !=
-      GNUNET_SERVER_mst_receive (mst,
-                                 &dmc,
-                                 &buf[sizeof (struct EncryptedMessage)],
-                                 size - sizeof (struct EncryptedMessage),
-                                 GNUNET_YES,
-                                 GNUNET_NO))
+      GNUNET_MST_from_buffer (kx->mst,
+                              &buf[sizeof (struct EncryptedMessage)],
+                              size - sizeof (struct EncryptedMessage),
+                              GNUNET_YES,
+                              GNUNET_NO))
     GNUNET_break_op (0);
 }
 
@@ -1642,7 +1675,7 @@ handle_transport_notify_excess_bw (void *cls,
                                    const struct GNUNET_PeerIdentity *pid,
                                    void *connect_cls)
 {
-  struct GSC_KeyExchangeInfo *kx = connect_cls;  
+  struct GSC_KeyExchangeInfo *kx = connect_cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer %s has excess bandwidth available\n",
@@ -1653,53 +1686,6 @@ handle_transport_notify_excess_bw (void *cls,
 
 
 /**
- * Deliver P2P message to interested clients.  Invokes send twice,
- * once for clients that want the full message, and once for clients
- * that only want the header
- *
- * @param cls always NULL
- * @param client who sent us the message (struct GSC_KeyExchangeInfo)
- * @param m the message
- */
-static int
-deliver_message (void *cls,
-                 void *client,
-                 const struct GNUNET_MessageHeader *m)
-{
-  struct DeliverMessageContext *dmc = client;
-
-  if (GNUNET_CORE_KX_STATE_UP != dmc->kx->status)
-  {
-    GNUNET_STATISTICS_update (GSC_stats,
-                              gettext_noop ("# PAYLOAD dropped (out of order)"),
-                              1,
-                              GNUNET_NO);
-    return GNUNET_OK;
-  }
-  switch (ntohs (m->type))
-  {
-  case GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP:
-  case GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP:
-    GSC_SESSIONS_set_typemap (dmc->peer, m);
-    return GNUNET_OK;
-  case GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP:
-    GSC_SESSIONS_confirm_typemap (dmc->peer, m);
-    return GNUNET_OK;
-  default:
-    GSC_CLIENTS_deliver_message (dmc->peer,
-                                 m,
-                                 ntohs (m->size),
-                                 GNUNET_CORE_OPTION_SEND_FULL_INBOUND);
-    GSC_CLIENTS_deliver_message (dmc->peer,
-                                 m,
-                                 sizeof (struct GNUNET_MessageHeader),
-                                 GNUNET_CORE_OPTION_SEND_HDR_INBOUND);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
  * Setup the message that links the ephemeral key to our persistent
  * public key and generate the appropriate signature.
  */
@@ -1822,9 +1808,7 @@ GSC_KX_init (struct GNUNET_CRYPTO_EddsaPrivateKey *pk)
   rekey_task = GNUNET_SCHEDULER_add_delayed (REKEY_FREQUENCY,
                                              &do_rekey,
                                              NULL);
-  mst = GNUNET_SERVER_mst_create (&deliver_message,
-                                  NULL);
-  transport 
+  transport
     = GNUNET_TRANSPORT_core_connect (GSC_cfg,
                                      &GSC_my_identity,
                                      handlers,
@@ -1867,11 +1851,6 @@ GSC_KX_done ()
     GNUNET_free (my_private_key);
     my_private_key = NULL;
   }
-  if (NULL != mst)
-  {
-    GNUNET_SERVER_mst_destroy (mst);
-    mst = NULL;
-  }
   if (NULL != nc)
   {
     GNUNET_notification_context_destroy (nc);
@@ -1927,7 +1906,7 @@ GSC_KX_handle_client_monitor_peers (struct GNUNET_MQ_Handle *mq)
   {
     struct GNUNET_MQ_Envelope *env;
     struct MonitorNotifyMessage *msg;
-    
+
     env = GNUNET_MQ_msg (msg,
                          GNUNET_MESSAGE_TYPE_CORE_MONITOR_NOTIFY);
     msg->state = htonl ((uint32_t) kx->status);
diff --git a/src/core/gnunet-service-core_sessions.c b/src/core/gnunet-service-core_sessions.c
index 6687b4819..034f2e883 100644
--- a/src/core/gnunet-service-core_sessions.c
+++ b/src/core/gnunet-service-core_sessions.c
@@ -57,6 +57,21 @@ struct SessionMessageEntry
   struct SessionMessageEntry *prev;
 
   /**
+   * How important is this message.
+   */
+  enum GNUNET_CORE_Priority priority;
+
+  /**
+   * Flag set to #GNUNET_YES if this is a typemap message.
+   */
+  int is_typemap;
+
+  /**
+   * Flag set to #GNUNET_YES if this is a typemap confirmation message.
+   */
+  int is_typemap_confirm;
+
+  /**
    * Deadline for transmission, 1s after we received it (if we
    * are not corking), otherwise "now".  Note that this message
    * does NOT expire past its deadline.
@@ -70,11 +85,6 @@ struct SessionMessageEntry
    */
   size_t size;
 
-  /**
-   * How important is this message.
-   */
-  enum GNUNET_CORE_Priority priority;
-
 };
 
 
@@ -275,14 +285,19 @@ transmit_typemap_task (void *cls)
   struct GNUNET_MessageHeader *hdr;
   struct GNUNET_TIME_Relative delay;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending TYPEMAP to %s\n",
+              GNUNET_i2s (session->peer));
   session->typemap_delay = GNUNET_TIME_STD_BACKOFF (session->typemap_delay);
   delay = session->typemap_delay;
   /* randomize a bit to avoid spont. sync */
   delay.rel_value_us +=
-      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000 * 1000);
+      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                1000 * 1000);
   session->typemap_task =
       GNUNET_SCHEDULER_add_delayed (delay,
-                                    &transmit_typemap_task, session);
+                                    &transmit_typemap_task,
+                                    session);
   GNUNET_STATISTICS_update (GSC_stats,
                             gettext_noop ("# type map refreshes sent"),
                             1,
@@ -326,7 +341,7 @@ GSC_SESSIONS_create (const struct GNUNET_PeerIdentity *peer,
   struct Session *session;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Creating session for peer `%4s'\n",
+              "Creating session for peer `%s'\n",
               GNUNET_i2s (peer));
   session = GNUNET_new (struct Session);
   session->tmap = GSC_TYPEMAP_create ();
@@ -406,8 +421,14 @@ GSC_SESSIONS_confirm_typemap (const struct GNUNET_PeerIdentity *peer,
                               gettext_noop
                               ("# outdated typemap confirmations received"),
                               1, GNUNET_NO);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Got outdated typemap confirmated from peer `%s'\n",
+                GNUNET_i2s (session->peer));
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Got typemap confirmation from peer `%s'\n",
+              GNUNET_i2s (session->peer));
   if (NULL != session->typemap_task)
   {
     GNUNET_SCHEDULER_cancel (session->typemap_task);
@@ -502,9 +523,9 @@ GSC_SESSIONS_queue_request (struct GSC_ClientActiveRequest *car)
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received client transmission request. queueing\n");
-  GNUNET_CONTAINER_DLL_insert (session->active_client_request_head,
-                               session->active_client_request_tail,
-                               car);
+  GNUNET_CONTAINER_DLL_insert_tail (session->active_client_request_head,
+                                    session->active_client_request_tail,
+                                    car);
   try_transmission (session);
 }
 
@@ -751,7 +772,15 @@ try_transmission (struct Session *session)
     while ( (NULL != (pos = session->sme_head)) &&
             (used + pos->size <= msize) )
     {
-      GNUNET_memcpy (&pbuf[used], &pos[1], pos->size);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Adding message of type %d (%d/%d) to payload for %s\n",
+                  ntohs (((const struct GNUNET_MessageHeader *)&pos[1])->type),
+                  pos->is_typemap,
+                  pos->is_typemap_confirm,
+                  GNUNET_i2s (session->peer));
+      GNUNET_memcpy (&pbuf[used],
+                     &pos[1],
+                     pos->size);
       used += pos->size;
       GNUNET_CONTAINER_DLL_remove (session->sme_head,
                                    session->sme_tail,
@@ -799,8 +828,23 @@ do_restart_typemap_message (void *cls,
   struct SessionMessageEntry *sme;
   uint16_t size;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Restarting sending TYPEMAP to %s\n",
+              GNUNET_i2s (session->peer));
   size = ntohs (hdr->size);
+  for (sme = session->sme_head; NULL != sme; sme = sme->next)
+  {
+    if (GNUNET_YES == sme->is_typemap)
+    {
+      GNUNET_CONTAINER_DLL_remove (session->sme_head,
+                                   session->sme_tail,
+                                   sme);
+      GNUNET_free (sme);
+      break;
+    }
+  }
   sme = GNUNET_malloc (sizeof (struct SessionMessageEntry) + size);
+  sme->is_typemap = GNUNET_YES;
   GNUNET_memcpy (&sme[1],
                  hdr,
                  size);
@@ -924,18 +968,36 @@ GSC_SESSIONS_set_typemap (const struct GNUNET_PeerIdentity *peer,
 
   nmap = GSC_TYPEMAP_get_from_message (msg);
   if (NULL == nmap)
+  {
+    GNUNET_break_op (0);
     return;                     /* malformed */
+  }
   session = find_session (peer);
   if (NULL == session)
   {
     GNUNET_break (0);
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received TYPEMAP from %s\n",
+              GNUNET_i2s (session->peer));
+  for (sme = session->sme_head; NULL != sme; sme = sme->next)
+  {
+    if (GNUNET_YES == sme->is_typemap_confirm)
+    {
+      GNUNET_CONTAINER_DLL_remove (session->sme_head,
+                                   session->sme_tail,
+                                   sme);
+      GNUNET_free (sme);
+      break;
+    }
+  }
   sme = GNUNET_malloc (sizeof (struct SessionMessageEntry) +
                        sizeof (struct TypeMapConfirmationMessage));
   sme->deadline = GNUNET_TIME_absolute_get ();
   sme->size = sizeof (struct TypeMapConfirmationMessage);
   sme->priority = GNUNET_CORE_PRIO_CRITICAL_CONTROL;
+  sme->is_typemap_confirm = GNUNET_YES;
   tmc = (struct TypeMapConfirmationMessage *) &sme[1];
   tmc->header.size = htons (sizeof (struct TypeMapConfirmationMessage));
   tmc->header.type = htons (GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP);
@@ -975,11 +1037,15 @@ GSC_SESSIONS_add_to_typemap (const struct GNUNET_PeerIdentity *peer,
     return;
   session = find_session (peer);
   GNUNET_assert (NULL != session);
-  if (GNUNET_YES == GSC_TYPEMAP_test_match (session->tmap, &type, 1))
+  if (GNUNET_YES == GSC_TYPEMAP_test_match (session->tmap,
+                                            &type, 1))
     return;                     /* already in it */
-  nmap = GSC_TYPEMAP_extend (session->tmap, &type, 1);
+  nmap = GSC_TYPEMAP_extend (session->tmap,
+                             &type,
+                             1);
   GSC_CLIENTS_notify_clients_about_neighbour (peer,
-                                              session->tmap, nmap);
+                                              session->tmap,
+                                              nmap);
   GSC_TYPEMAP_destroy (session->tmap);
   session->tmap = nmap;
 }
diff --git a/src/core/gnunet-service-core_typemap.c b/src/core/gnunet-service-core_typemap.c
index d400c0b8b..0600f59ef 100644
--- a/src/core/gnunet-service-core_typemap.c
+++ b/src/core/gnunet-service-core_typemap.c
@@ -177,8 +177,10 @@ GSC_TYPEMAP_get_from_message (const struct GNUNET_MessageHeader *msg)
     GNUNET_memcpy (ret, &msg[1], sizeof (struct GSC_TypeMap));
     return ret;
   case GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP:
-    GNUNET_STATISTICS_update (GSC_stats, gettext_noop ("# type maps received"),
-                              1, GNUNET_NO);
+    GNUNET_STATISTICS_update (GSC_stats,
+                              gettext_noop ("# type maps received"),
+                              1,
+                              GNUNET_NO);
     ret = GNUNET_new (struct GSC_TypeMap);
     dlen = sizeof (struct GSC_TypeMap);
     if ((Z_OK !=
@@ -207,7 +209,8 @@ broadcast_my_type_map ()
 
   hdr = GSC_TYPEMAP_compute_type_map_message ();
   GNUNET_STATISTICS_update (GSC_stats,
-                            gettext_noop ("# updates to my type map"), 1,
+                            gettext_noop ("# updates to my type map"),
+                            1,
                             GNUNET_NO);
   GSC_SESSIONS_broadcast_typemap (hdr);
   GNUNET_free (hdr);
@@ -238,6 +241,8 @@ GSC_TYPEMAP_add (const uint16_t *types,
   }
   if (GNUNET_YES == changed)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Typemap changed, broadcasting!\n");
     rehash_typemap ();
     broadcast_my_type_map ();
   }
@@ -254,11 +259,10 @@ void
 GSC_TYPEMAP_remove (const uint16_t *types,
                     unsigned int tlen)
 {
-  unsigned int i;
   int changed;
 
   changed = GNUNET_NO;
-  for (i = 0; i < tlen; i++)
+  for (unsigned int i = 0; i < tlen; i++)
   {
     if (0 == --map_counters[types[i]])
     {
@@ -288,13 +292,11 @@ GSC_TYPEMAP_test_match (const struct GSC_TypeMap *tmap,
                         const uint16_t *types,
                         unsigned int tcnt)
 {
-  unsigned int i;
-
   if (NULL == tmap)
     return GNUNET_NO;
   if (0 == tcnt)
     return GNUNET_YES;          /* matches all */
-  for (i = 0; i < tcnt; i++)
+  for (unsigned int i = 0; i < tcnt; i++)
     if (0 != (tmap->bits[types[i] / 32] & (1 << (types[i] % 32))))
       return GNUNET_YES;
   return GNUNET_NO;
@@ -315,12 +317,11 @@ GSC_TYPEMAP_extend (const struct GSC_TypeMap *tmap,
                     unsigned int tcnt)
 {
   struct GSC_TypeMap *ret;
-  unsigned int i;
 
   ret = GNUNET_new (struct GSC_TypeMap);
   if (NULL != tmap)
     GNUNET_memcpy (ret, tmap, sizeof (struct GSC_TypeMap));
-  for (i = 0; i < tcnt; i++)
+  for (unsigned int i = 0; i < tcnt; i++)
     ret->bits[types[i] / 32] |= (1 << (types[i] % 32));
   return ret;
 }
diff --git a/src/datacache/Makefile.am b/src/datacache/Makefile.am
index 9b66636e5..670a64926 100644
--- a/src/datacache/Makefile.am
+++ b/src/datacache/Makefile.am
@@ -53,6 +53,7 @@ libgnunet_plugin_datacache_sqlite_la_SOURCES = \
   plugin_datacache_sqlite.c
 libgnunet_plugin_datacache_sqlite_la_LIBADD = \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
 libgnunet_plugin_datacache_sqlite_la_LDFLAGS = \
@@ -127,7 +128,7 @@ check_PROGRAMS = \
  $(POSTGRES_TESTS)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/datacache/plugin_datacache_sqlite.c b/src/datacache/plugin_datacache_sqlite.c
index 5567077d3..dd79d0125 100644
--- a/src/datacache/plugin_datacache_sqlite.c
+++ b/src/datacache/plugin_datacache_sqlite.c
@@ -26,6 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_datacache_plugin.h"
+#include "gnunet_sq_lib.h"
 #include <sqlite3.h>
 
 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-sqlite", __VA_ARGS__)
@@ -60,6 +61,41 @@ struct Plugin
   char *fn;
 
   /**
+   * Prepared statement for #sqlite_plugin_put.
+   */
+  sqlite3_stmt *insert_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get.
+   */
+  sqlite3_stmt *get_count_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get.
+   */
+  sqlite3_stmt *get_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_del.
+   */
+  sqlite3_stmt *del_select_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_del.
+   */
+  sqlite3_stmt *del_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get_random.
+   */
+  sqlite3_stmt *get_random_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get_closest.
+   */
+  sqlite3_stmt *get_closest_stmt;
+
+  /**
    * Number of key-value pairs in the database.
    */
   unsigned int num_items;
@@ -132,60 +168,47 @@ sqlite_plugin_put (void *cls,
                    const struct GNUNET_PeerIdentity *path_info)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
-  int64_t dval;
+  uint32_t type32 = type;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&discard_time),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_fixed_size (data, size),
+    GNUNET_SQ_query_param_fixed_size (path_info,
+                                      path_info_len * sizeof (struct GNUNET_PeerIdentity)),
+    GNUNET_SQ_query_param_end
+  };
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing PUT of %u bytes with key `%4s' and expiration %s\n",
+       "Processing PUT of %u bytes with key `%s' and expiration %s\n",
        (unsigned int) size,
        GNUNET_h2s (key),
-       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time), GNUNET_YES));
-  dval = (int64_t) discard_time.abs_value_us;
-  if (dval < 0)
-    dval = INT64_MAX;
-  if (sq_prepare
-      (plugin->dbh,
-       "INSERT INTO ds090 (type, expire, key, value, path) VALUES (?, ?, ?, ?, ?)",
-       &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return -1;
-  }
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, dval)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 3,
-                          key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 4,
-                                       data, size,
-                                       SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 5,
-                                       path_info,
-                                       path_info_len * sizeof (struct GNUNET_PeerIdentity),
-                                       SQLITE_TRANSIENT)))
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time),
+                                               GNUNET_YES));
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->insert_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->insert_stmt);
     return -1;
   }
-  if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (plugin->insert_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->insert_stmt);
     return -1;
   }
   plugin->num_items++;
-  if (SQLITE_OK != sqlite3_finalize (stmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_finalize");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->insert_stmt);
   return size + OVERHEAD;
 }
 
@@ -209,120 +232,119 @@ sqlite_plugin_get (void *cls,
                    void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  uint32_t type32 = type;
   struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
+  size_t size;
+  void *dat;
   unsigned int cnt;
-  unsigned int off;
+  uint32_t off;
   unsigned int total;
-  unsigned int psize;
-  char scratch[256];
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
+  size_t psize;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_SQ_QueryParam params_count[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam params_select[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_uint32 (&off),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_end
+  };
 
   now = GNUNET_TIME_absolute_get ();
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing GET for key `%4s'\n",
+       "Processing GET for key `%s'\n",
        GNUNET_h2s (key));
-  if (sq_prepare
-      (plugin->dbh,
-       "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?",
-       &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return 0;
-  }
-  ntime = (int64_t) now.abs_value_us;
-  GNUNET_assert (ntime >= 0);
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_count_stmt,
+                      params_count))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_count_stmt);
     return 0;
   }
-
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->get_count_stmt))
   {
     LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite_step");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_count_stmt);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "No content found when processing GET for key `%4s'\n",
+         "No content found when processing GET for key `%s'\n",
          GNUNET_h2s (key));
     return 0;
   }
-  total = sqlite3_column_int (stmt, 0);
-  sqlite3_finalize (stmt);
-  if ((0 == total) || (NULL == iter))
+  total = sqlite3_column_int (plugin->get_count_stmt,
+                              0);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_count_stmt);
+  if ( (0 == total) ||
+       (NULL == iter) )
   {
     if (0 == total)
       LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "No content found when processing GET for key `%4s'\n",
+           "No content found when processing GET for key `%s'\n",
            GNUNET_h2s (key));
     return total;
   }
 
   cnt = 0;
-  off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
+  off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                  total);
   while (cnt < total)
   {
     off = (off + 1) % total;
-    GNUNET_snprintf (scratch, sizeof (scratch),
-                     "SELECT value,expire,path FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
-                     off);
-    if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
-    {
-      LOG_SQLITE (plugin->dbh,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sq_prepare");
-      return cnt;
-    }
-    if ((SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 1,
-                            key,
-                            sizeof (struct GNUNET_HashCode),
-                            SQLITE_TRANSIENT)) ||
-        (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-        (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->get_stmt,
+                        params_select))
     {
       LOG_SQLITE (plugin->dbh,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_bind_xxx");
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
       return cnt;
     }
-    if (sqlite3_step (stmt) != SQLITE_ROW)
+    if (SQLITE_ROW !=
+        sqlite3_step (plugin->get_stmt))
       break;
-    size = sqlite3_column_bytes (stmt, 0);
-    dat = sqlite3_column_blob (stmt, 0);
-    exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-    psize = sqlite3_column_bytes (stmt, 2);
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->get_stmt,
+                                  rs))
+    {
+      GNUNET_break (0);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
+      break;
+    }
     if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
     {
       GNUNET_break (0);
       psize = 0;
+      path = NULL;
     }
     psize /= sizeof (struct GNUNET_PeerIdentity);
-    if (0 != psize)
-      path = sqlite3_column_blob (stmt, 2);
-    else
-      path = NULL;
-    ntime = (int64_t) exp.abs_value_us;
-    if (ntime == INT64_MAX)
-      exp = GNUNET_TIME_UNIT_FOREVER_ABS;
     cnt++;
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Found %u-byte result when processing GET for key `%4s'\n",
+         "Found %u-byte result when processing GET for key `%s'\n",
          (unsigned int) size,
          GNUNET_h2s (key));
     if (GNUNET_OK != iter (iter_cls,
@@ -334,11 +356,17 @@ sqlite_plugin_get (void *cls,
                            psize,
                            path))
     {
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_cleanup_result (rs);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
       break;
     }
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_cleanup_result (rs);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_stmt);
   }
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_stmt);
   return cnt;
 }
 
@@ -354,79 +382,73 @@ static int
 sqlite_plugin_del (void *cls)
 {
   struct Plugin *plugin = cls;
-  unsigned long long rowid;
-  unsigned int dsize;
-  sqlite3_stmt *stmt;
-  sqlite3_stmt *dstmt;
+  uint64_t rowid;
+  void *data;
+  size_t dsize;
   struct GNUNET_HashCode hc;
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_uint64 (&rowid),
+    GNUNET_SQ_result_spec_auto_from_type (&hc),
+    GNUNET_SQ_result_spec_variable_size ((void **) &data,
+                                         &dsize),
+    GNUNET_SQ_result_spec_end
+  };
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&rowid),
+    GNUNET_SQ_query_param_end
+  };
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Processing DEL\n");
-  stmt = NULL;
-  dstmt = NULL;
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
-                  &stmt))
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    if (stmt != NULL)
-      (void) sqlite3_finalize (stmt);
-    return GNUNET_SYSERR;
-  }
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->del_select_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    (void) sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_select_stmt);
     return GNUNET_SYSERR;
   }
-  rowid = sqlite3_column_int64 (stmt, 0);
-  GNUNET_assert (sqlite3_column_bytes (stmt, 1) == sizeof (struct GNUNET_HashCode));
-  GNUNET_memcpy (&hc, sqlite3_column_blob (stmt, 1), sizeof (struct GNUNET_HashCode));
-  dsize = sqlite3_column_bytes (stmt, 2);
-  if (SQLITE_OK != sqlite3_finalize (stmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_step");
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "DELETE FROM ds090 WHERE _ROWID_=?", &dstmt))
+  if (GNUNET_OK !=
+      GNUNET_SQ_extract_result (plugin->del_select_stmt,
+                                rs))
   {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    if (stmt != NULL)
-      (void) sqlite3_finalize (stmt);
+    GNUNET_break (0);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_select_stmt);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_bind_int64 (dstmt, 1, rowid))
+  GNUNET_SQ_cleanup_result (rs);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->del_select_stmt);
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->del_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
-    (void) sqlite3_finalize (dstmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_stmt);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_DONE != sqlite3_step (dstmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (plugin->del_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    (void) sqlite3_finalize (dstmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_stmt);
     return GNUNET_SYSERR;
   }
   plugin->num_items--;
   plugin->env->delete_notify (plugin->env->cls,
                               &hc,
                               dsize + OVERHEAD);
-  if (SQLITE_OK != sqlite3_finalize (dstmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_finalize");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->del_stmt);
   return GNUNET_OK;
 }
 
@@ -445,17 +467,28 @@ sqlite_plugin_get_random (void *cls,
                           void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
-  unsigned int off;
-  unsigned int psize;
-  unsigned int type;
-  char scratch[256];
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
-  const struct GNUNET_HashCode *key;
+  size_t size;
+  void *dat;
+  uint32_t off;
+  size_t psize;
+  uint32_t type;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_HashCode key;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&off),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_end
+  };
 
   if (0 == plugin->num_items)
     return 0;
@@ -463,60 +496,51 @@ sqlite_plugin_get_random (void *cls,
     return 1;
   off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                   plugin->num_items);
-  GNUNET_snprintf (scratch,
-                   sizeof (scratch),
-                   "SELECT value,expire,path,key,type FROM ds090 ORDER BY key LIMIT 1 OFFSET %u",
-                   off);
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh, scratch, &stmt))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_random_stmt,
+                      params))
   {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
     return 0;
   }
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->get_random_stmt))
+  {
+    GNUNET_break (0);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_random_stmt);
+    return 0;
+  }
+  if (GNUNET_OK !=
+      GNUNET_SQ_extract_result (plugin->get_random_stmt,
+                                rs))
   {
     GNUNET_break (0);
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_random_stmt);
     return 0;
   }
-  size = sqlite3_column_bytes (stmt, 0);
-  dat = sqlite3_column_blob (stmt, 0);
-  exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-  psize = sqlite3_column_bytes (stmt, 2);
   if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
   {
     GNUNET_break (0);
     psize = 0;
+    path = NULL;
   }
   psize /= sizeof (struct GNUNET_PeerIdentity);
-  if (0 != psize)
-    path = sqlite3_column_blob (stmt, 2);
-  else
-    path = NULL;
-
-  GNUNET_assert (sizeof (struct GNUNET_HashCode) ==
-                 sqlite3_column_bytes (stmt, 3));
-  key = sqlite3_column_blob (stmt, 3);
-  type = sqlite3_column_int (stmt, 4);
-
-  ntime = (int64_t) exp.abs_value_us;
-  if (ntime == INT64_MAX)
-    exp = GNUNET_TIME_UNIT_FOREVER_ABS;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Found %u-byte result with key %s when processing GET-RANDOM\n",
        (unsigned int) size,
-       GNUNET_h2s (key));
+       GNUNET_h2s (&key));
   (void) iter (iter_cls,
-               key,
+               &key,
                size,
                dat,
-               type,
+               (enum GNUNET_BLOCK_Type) type,
                exp,
                psize,
                path);
-  sqlite3_finalize (stmt);
+  GNUNET_SQ_cleanup_result (rs);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_random_stmt);
   return 1;
 }
 
@@ -542,83 +566,73 @@ sqlite_plugin_get_closest (void *cls,
                            void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  uint32_t num_results32 = num_results;
   struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
+  size_t size;
+  void *dat;
   unsigned int cnt;
-  unsigned int psize;
-  unsigned int type;
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
+  size_t psize;
+  uint32_t type;
+  struct GNUNET_HashCode hc;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_uint32 (&num_results32),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_auto_from_type (&hc),
+    GNUNET_SQ_result_spec_end
+  };
 
   now = GNUNET_TIME_absolute_get ();
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing GET_CLOSEST for key `%4s'\n",
+       "Processing GET_CLOSEST for key `%s'\n",
        GNUNET_h2s (key));
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "SELECT value,expire,path,type,key FROM ds090 WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?",
-                  &stmt))
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return 0;
-  }
-  ntime = (int64_t) now.abs_value_us;
-  GNUNET_assert (ntime >= 0);
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (stmt,
-                          1,
-                          key,
-                          sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, now.abs_value_us)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 3, num_results)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_closest_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_closest_stmt);
     return 0;
   }
   cnt = 0;
-  while (SQLITE_ROW == sqlite3_step (stmt))
+  while (SQLITE_ROW ==
+         sqlite3_step (plugin->get_closest_stmt))
   {
-    if (sizeof (struct GNUNET_HashCode) !=
-        sqlite3_column_bytes (stmt, 4))
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->get_closest_stmt,
+                                  rs))
     {
       GNUNET_break (0);
       break;
     }
-    size = sqlite3_column_bytes (stmt, 0);
-    dat = sqlite3_column_blob (stmt, 0);
-    exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-    psize = sqlite3_column_bytes (stmt, 2);
-    type = sqlite3_column_int (stmt, 3);
-    key = sqlite3_column_blob (stmt, 4);
     if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
     {
       GNUNET_break (0);
       psize = 0;
+      path = NULL;
     }
     psize /= sizeof (struct GNUNET_PeerIdentity);
-    if (0 != psize)
-      path = sqlite3_column_blob (stmt, 2);
-    else
-      path = NULL;
-    ntime = (int64_t) exp.abs_value_us;
-    if (ntime == INT64_MAX)
-      exp = GNUNET_TIME_UNIT_FOREVER_ABS;
     cnt++;
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Found %u-byte result at %s when processing GET_CLOSE\n",
          (unsigned int) size,
-         GNUNET_h2s (key));
+         GNUNET_h2s (&hc));
     if (GNUNET_OK != iter (iter_cls,
-                           key,
+                           &hc,
                            size,
                            dat,
                            type,
@@ -626,11 +640,13 @@ sqlite_plugin_get_closest (void *cls,
                            psize,
                            path))
     {
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_cleanup_result (rs);
       break;
     }
+    GNUNET_SQ_cleanup_result (rs);
   }
-  sqlite3_finalize (stmt);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_closest_stmt);
   return cnt;
 }
 
@@ -703,6 +719,51 @@ libgnunet_plugin_datacache_sqlite_init (void *cls)
   plugin->env = env;
   plugin->dbh = dbh;
   plugin->fn = fn_utf8;
+
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ds090 (type, expire, key, value, path) "
+                    "VALUES (?, ?, ?, ?, ?)",
+                    &plugin->insert_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT count(*) FROM ds090 "
+                    "WHERE key=? AND type=? AND expire >= ?",
+                    &plugin->get_count_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path FROM ds090 "
+                    "WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET ?",
+                    &plugin->get_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
+                    &plugin->del_select_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ds090 WHERE _ROWID_=?",
+                    &plugin->del_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path,key,type FROM ds090 "
+                    "ORDER BY key LIMIT 1 OFFSET ?",
+                    &plugin->get_random_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path,type,key FROM ds090 "
+                    "WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?",
+                    &plugin->get_closest_stmt))
+       )
+  {
+    LOG_SQLITE (plugin->dbh,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                "sq_prepare");
+    GNUNET_break (SQLITE_OK ==
+                  sqlite3_close (plugin->dbh));
+    GNUNET_free (plugin);
+    return NULL;
+  }
+
   api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
   api->cls = plugin;
   api->get = &sqlite_plugin_get;
@@ -741,6 +802,13 @@ libgnunet_plugin_datacache_sqlite_done (void *cls)
                        plugin->fn);
   GNUNET_free_non_null (plugin->fn);
 #endif
+  sqlite3_finalize (plugin->insert_stmt);
+  sqlite3_finalize (plugin->get_count_stmt);
+  sqlite3_finalize (plugin->get_stmt);
+  sqlite3_finalize (plugin->del_select_stmt);
+  sqlite3_finalize (plugin->del_stmt);
+  sqlite3_finalize (plugin->get_random_stmt);
+  sqlite3_finalize (plugin->get_closest_stmt);
   result = sqlite3_close (plugin->dbh);
 #if SQLITE_VERSION_NUMBER >= 3007000
   if (SQLITE_BUSY == result)
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am
index 1e7619d82..9b8cf365f 100644
--- a/src/datastore/Makefile.am
+++ b/src/datastore/Makefile.am
@@ -115,6 +115,7 @@ noinst_LTLIBRARIES = \
 libgnunet_plugin_datastore_sqlite_la_SOURCES = \
   plugin_datastore_sqlite.c
 libgnunet_plugin_datastore_sqlite_la_LIBADD = \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
@@ -175,7 +176,7 @@ check_PROGRAMS = \
   $(POSTGRES_TESTS)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h
index dc3d9d1f2..98f8b82ef 100644
--- a/src/datastore/datastore.h
+++ b/src/datastore/datastore.h
@@ -119,9 +119,14 @@ struct GetKeyMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
    */
-  uint64_t offset GNUNET_PACKED;
+  uint64_t next_uid GNUNET_PACKED;
+
+  /**
+   * If true return a random result
+   */
+  uint32_t random GNUNET_PACKED;
 
   /**
    * Desired key.
@@ -148,9 +153,14 @@ struct GetMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
    */
-  uint64_t offset GNUNET_PACKED;
+  uint64_t next_uid GNUNET_PACKED;
+
+  /**
+   * If true return a random result
+   */
+  uint32_t random GNUNET_PACKED;
 
 };
 
@@ -172,38 +182,9 @@ struct GetZeroAnonymityMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
    */
-  uint64_t offset GNUNET_PACKED;
-
-};
-
-
-/**
- * Message to the datastore service requesting an update
- * to the priority or expiration for some content.
- */
-struct UpdateMessage
-{
-  /**
-   * Type is GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE.
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Desired priority increase.
-   */
-  int32_t priority GNUNET_PACKED;
-
-  /**
-   * Desired new expiration time.
-   */
-  struct GNUNET_TIME_AbsoluteNBO expiration;
-
-  /**
-   * Unique ID for the content.
-   */
-  uint64_t uid;
+  uint64_t next_uid GNUNET_PACKED;
 
 };
 
@@ -248,7 +229,7 @@ struct DataMessage
   uint32_t anonymity GNUNET_PACKED;
 
   /**
-   * Desired replication level. 0 from service to API.
+   * Desired replication level.
    */
   uint32_t replication GNUNET_PACKED;
 
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c
index db485364e..31f7a997f 100644
--- a/src/datastore/datastore_api.c
+++ b/src/datastore/datastore_api.c
@@ -33,6 +33,8 @@
 
 #define LOG(kind,...) GNUNET_log_from (kind, "datastore-api",__VA_ARGS__)
 
+#define DELAY_WARN_TIMEOUT GNUNET_TIME_UNIT_MINUTES
+
 /**
  * Collect an instane number of statistics?  May cause excessive IPC.
  */
@@ -138,6 +140,12 @@ struct GNUNET_DATASTORE_QueueEntry
   struct GNUNET_MQ_Envelope *env;
 
   /**
+   * Task we run if this entry stalls the queue and we
+   * need to warn the user.
+   */
+  struct GNUNET_SCHEDULER_Task *delay_warn_task;
+
+  /**
    * Priority in the queue.
    */
   unsigned int priority;
@@ -269,11 +277,36 @@ free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
   h->queue_size--;
   if (NULL != qe->env)
     GNUNET_MQ_discard (qe->env);
+  if (NULL != qe->delay_warn_task)
+    GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
   GNUNET_free (qe);
 }
 
 
 /**
+ * Task that logs an error after some time.
+ *
+ * @param qe `struct GNUNET_DATASTORE_QueueEntry` about which the error is
+ */
+static void
+delay_warning (void *cls)
+{
+  struct GNUNET_DATASTORE_QueueEntry *qe = cls;
+
+  qe->delay_warn_task = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Request %p of type %u at head of datastore queue for more than %s\n",
+              qe,
+              (unsigned int) qe->response_type,
+              GNUNET_STRINGS_relative_time_to_string (DELAY_WARN_TIMEOUT,
+                                                      GNUNET_YES));
+  qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
+                                                      &delay_warning,
+                                                      qe);
+}
+
+
+/**
  * Handle error in sending drop request to datastore.
  *
  * @param cls closure with the datastore handle
@@ -290,8 +323,14 @@ mq_error_handler (void *cls,
        "MQ error, reconnecting to DATASTORE\n");
   do_disconnect (h);
   qe = h->queue_head;
-  if ( (NULL != qe) &&
-       (NULL == qe->env) )
+  if (NULL == qe)
+    return;
+  if (NULL != qe->delay_warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
+    qe->delay_warn_task = NULL;
+  }
+  if (NULL == qe->env)
   {
     union QueueContext qc = qe->qc;
     uint16_t rt = qe->response_type;
@@ -313,7 +352,11 @@ mq_error_handler (void *cls,
         qc.rc.proc (qc.rc.proc_cls,
                     NULL,
                     0,
-                    NULL, 0, 0, 0,
+                    NULL,
+                    0,
+                    0,
+                    0,
+                    0,
                     GNUNET_TIME_UNIT_ZERO_ABS,
                     0);
       break;
@@ -429,7 +472,11 @@ GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h,
         qe->qc.rc.proc (qe->qc.rc.proc_cls,
                         NULL,
                         0,
-                        NULL, 0, 0, 0,
+                        NULL,
+                        0,
+                        0,
+                        0,
+                        0,
                         GNUNET_TIME_UNIT_ZERO_ABS,
                         0);
       break;
@@ -498,8 +545,17 @@ make_queue_entry (struct GNUNET_DATASTORE_Handle *h,
   struct GNUNET_DATASTORE_QueueEntry *pos;
   unsigned int c;
 
-  c = 0;
-  pos = h->queue_head;
+  if ( (NULL != h->queue_tail) &&
+       (h->queue_tail->priority >= queue_priority) )
+  {
+    c = h->queue_size;
+    pos = NULL;
+  }
+  else
+  {
+    c = 0;
+    pos = h->queue_head;
+  }
   while ( (NULL != pos) &&
           (c < max_queue_size) &&
           (pos->priority >= queue_priority) )
@@ -585,6 +641,10 @@ process_queue (struct GNUNET_DATASTORE_Handle *h)
          "Not connected\n");
     return;
   }
+  GNUNET_assert (NULL == qe->delay_warn_task);
+  qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
+                                                      &delay_warning,
+                                                      qe);
   GNUNET_MQ_send (h->mq,
                   qe->env);
   qe->env = NULL;
@@ -773,6 +833,7 @@ handle_data (void *cls,
              ntohl (dm->type),
              ntohl (dm->priority),
              ntohl (dm->anonymity),
+             ntohl (dm->replication),
              GNUNET_TIME_absolute_ntoh (dm->expiration),
              GNUNET_ntohll (dm->uid));
 }
@@ -835,6 +896,7 @@ handle_data_end (void *cls,
              0,
              0,
              0,
+             0,
              GNUNET_TIME_UNIT_ZERO_ABS,
              0);
 }
@@ -949,7 +1011,7 @@ GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
   struct DataMessage *dm;
   union QueueContext qc;
 
-  if (size + sizeof (*dm) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size + sizeof (*dm) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
@@ -970,8 +1032,6 @@ GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
   dm->priority = htonl (priority);
   dm->anonymity = htonl (anonymity);
   dm->replication = htonl (replication);
-  dm->reserved = htonl (0);
-  dm->uid = GNUNET_htonll (0);
   dm->expiration = GNUNET_TIME_absolute_hton (expiration);
   dm->key = *key;
   GNUNET_memcpy (&dm[1],
@@ -1126,72 +1186,6 @@ GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
 
 
 /**
- * Update a value in the datastore.
- *
- * @param h handle to the datastore
- * @param uid identifier for the value
- * @param priority how much to increase the priority of the value
- * @param expiration new expiration value should be MAX of existing and this argument
- * @param queue_priority ranking of this request in the priority queue
- * @param max_queue_size at what queue size should this request be dropped
- *        (if other requests of higher priority are in the queue)
- * @param cont continuation to call when done
- * @param cont_cls closure for @a cont
- * @return NULL if the entry was not queued, otherwise a handle that can be used to
- *         cancel; note that even if NULL is returned, the callback will be invoked
- *         (or rather, will already have been invoked)
- */
-struct GNUNET_DATASTORE_QueueEntry *
-GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
-                         uint64_t uid,
-                         uint32_t priority,
-                         struct GNUNET_TIME_Absolute expiration,
-                         unsigned int queue_priority,
-                         unsigned int max_queue_size,
-                         GNUNET_DATASTORE_ContinuationWithStatus cont,
-                         void *cont_cls)
-{
-  struct GNUNET_DATASTORE_QueueEntry *qe;
-  struct GNUNET_MQ_Envelope *env;
-  struct UpdateMessage *um;
-  union QueueContext qc;
-
-  if (NULL == cont)
-    cont = &drop_status_cont;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Asked to update entry %llu raising priority by %u and expiration to %s\n",
-       uid,
-       (unsigned int) priority,
-       GNUNET_STRINGS_absolute_time_to_string (expiration));
-  env = GNUNET_MQ_msg (um,
-                       GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE);
-  um->priority = htonl (priority);
-  um->expiration = GNUNET_TIME_absolute_hton (expiration);
-  um->uid = GNUNET_htonll (uid);
-
-  qc.sc.cont = cont;
-  qc.sc.cont_cls = cont_cls;
-  qe = make_queue_entry (h,
-                         env,
-                         queue_priority,
-                         max_queue_size,
-                         GNUNET_MESSAGE_TYPE_DATASTORE_STATUS,
-                         &qc);
-  if (NULL == qe)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Could not create queue entry for UPDATE\n");
-    return NULL;
-  }
-  GNUNET_STATISTICS_update (h->stats,
-                            gettext_noop ("# UPDATE requests executed"), 1,
-                            GNUNET_NO);
-  process_queue (h);
-  return qe;
-}
-
-
-/**
  * Explicitly remove some content from the database.
  * The @a cont continuation will be called with `status`
  * #GNUNET_OK" if content was removed, #GNUNET_NO
@@ -1226,7 +1220,7 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
   struct GNUNET_MQ_Envelope *env;
   union QueueContext qc;
 
-  if (sizeof (*dm) + size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (sizeof (*dm) + size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
@@ -1240,13 +1234,7 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
   env = GNUNET_MQ_msg_extra (dm,
                              size,
                              GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE);
-  dm->rid = htonl (0);
   dm->size = htonl (size);
-  dm->type = htonl (0);
-  dm->priority = htonl (0);
-  dm->anonymity = htonl (0);
-  dm->uid = GNUNET_htonll (0);
-  dm->expiration = GNUNET_TIME_absolute_hton (GNUNET_TIME_UNIT_ZERO_ABS);
   dm->key = *key;
   GNUNET_memcpy (&dm[1],
           data,
@@ -1339,10 +1327,7 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
  * Get a single zero-anonymity value from the datastore.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param queue_priority ranking of this request in the priority queue
  * @param max_queue_size at what queue size should this request be dropped
  *        (if other requests of higher priority are in the queue)
@@ -1356,7 +1341,7 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
-                                     uint64_t offset,
+                                     uint64_t next_uid,
                                      unsigned int queue_priority,
                                      unsigned int max_queue_size,
                                      enum GNUNET_BLOCK_Type type,
@@ -1371,13 +1356,12 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
   GNUNET_assert (NULL != proc);
   GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Asked to get %llu-th zero-anonymity entry of type %d\n",
-       (unsigned long long) offset,
+       "Asked to get a zero-anonymity entry of type %d\n",
        type);
   env = GNUNET_MQ_msg (m,
                        GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
   m->type = htonl ((uint32_t) type);
-  m->offset = GNUNET_htonll (offset);
+  m->next_uid = GNUNET_htonll (next_uid);
   qc.rc.proc = proc;
   qc.rc.proc_cls = proc_cls;
   qe = make_queue_entry (h,
@@ -1406,10 +1390,8 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
  * will only be called once.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param type desired type, 0 for any
  * @param queue_priority ranking of this request in the priority queue
@@ -1423,7 +1405,8 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
-                          uint64_t offset,
+                          uint64_t next_uid,
+                          bool random,
                           const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           unsigned int queue_priority,
@@ -1447,14 +1430,16 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
     env = GNUNET_MQ_msg (gm,
                          GNUNET_MESSAGE_TYPE_DATASTORE_GET);
     gm->type = htonl (type);
-    gm->offset = GNUNET_htonll (offset);
+    gm->next_uid = GNUNET_htonll (next_uid);
+    gm->random = random;
   }
   else
   {
     env = GNUNET_MQ_msg (gkm,
                          GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
     gkm->type = htonl (type);
-    gkm->offset = GNUNET_htonll (offset);
+    gkm->next_uid = GNUNET_htonll (next_uid);
+    gkm->random = random;
     gkm->key = *key;
   }
   qc.rc.proc = proc;
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index b3d14c43c..9e0ee205e 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -130,20 +130,23 @@ do_finish (void *cls,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
  */
 static void
 do_put (void *cls,
-        const struct GNUNET_HashCode *key,
-        size_t size,
+        const struct GNUNET_HashCode *key,
+        size_t size,
         const void *data,
-        enum GNUNET_BLOCK_Type type,
-        uint32_t priority,
-        uint32_t anonymity,
-        struct GNUNET_TIME_Absolute
-        expiration, uint64_t uid)
+        enum GNUNET_BLOCK_Type type,
+        uint32_t priority,
+        uint32_t anonymity,
+        uint32_t replication,
+        struct GNUNET_TIME_Absolute
+        expiration,
+        uint64_t uid)
 {
   qe = NULL;
   if ( (0 != offset) &&
@@ -154,13 +157,20 @@ do_put (void *cls,
   }
   if (0 == offset)
     first_uid = uid;
-  qe = GNUNET_DATASTORE_put (db_dst, 0,
-                             key, size, data, type,
-                             priority, anonymity,
-                             0 /* FIXME: replication is lost... */,
-                             expiration,
-                             0, 1,
-                             &do_finish, NULL);
+  qe = GNUNET_DATASTORE_put (db_dst,
+                             0,
+                             key,
+                             size,
+                             data,
+                             type,
+                             priority,
+                             anonymity,
+                             replication,
+                             expiration,
+                             0,
+                             1,
+                             &do_finish,
+                             NULL);
 }
 
 
@@ -171,7 +181,7 @@ static void
 do_get ()
 {
   qe = GNUNET_DATASTORE_get_key (db_src,
-                                 offset,
+                                 0, false,
                                  NULL, GNUNET_BLOCK_TYPE_ANY,
                                  0, 1,
                                  &do_put, NULL);
@@ -239,10 +249,12 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 's', "sourcecfg", "FILENAME",
-      gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"),
-      1, &GNUNET_GETOPT_set_filename, &alternative_cfg },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_filename ('s',
+                                   "sourcecfg",
+                                   "FILENAME",
+                                   gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"),
+                                       &alternative_cfg),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index e632e33e0..277530843 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -286,6 +286,7 @@ delete_expired (void *cls);
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -302,12 +303,13 @@ expired_processor (void *cls,
                    enum GNUNET_BLOCK_Type type,
                    uint32_t priority,
                    uint32_t anonymity,
+                   uint32_t replication,
                    struct GNUNET_TIME_Absolute expiration,
                    uint64_t uid)
 {
   struct GNUNET_TIME_Absolute now;
 
-  if (key == NULL)
+  if (NULL == key)
   {
     expired_kill_task =
         GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
@@ -374,6 +376,7 @@ delete_expired (void *cls)
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -389,6 +392,7 @@ quota_processor (void *cls,
                  enum GNUNET_BLOCK_Type type,
                  uint32_t priority,
                  uint32_t anonymity,
+                 uint32_t replication,
                  struct GNUNET_TIME_Absolute expiration,
                  uint64_t uid)
 {
@@ -495,6 +499,7 @@ transmit_status (struct GNUNET_SERVICE_Client *client,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -509,6 +514,7 @@ transmit_item (void *cls,
                enum GNUNET_BLOCK_Type type,
                uint32_t priority,
                uint32_t anonymity,
+               uint32_t replication,
                struct GNUNET_TIME_Absolute expiration,
                uint64_t uid)
 {
@@ -529,7 +535,7 @@ transmit_item (void *cls,
     return GNUNET_OK;
   }
   GNUNET_assert (sizeof (struct DataMessage) + size <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (dm,
                              size,
                              GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
@@ -538,8 +544,7 @@ transmit_item (void *cls,
   dm->type = htonl (type);
   dm->priority = htonl (priority);
   dm->anonymity = htonl (anonymity);
-  dm->replication = htonl (0);
-  dm->reserved = htonl (0);
+  dm->replication = htonl (replication);
   dm->expiration = GNUNET_TIME_absolute_hton (expiration);
   dm->uid = GNUNET_htonll (uid);
   dm->key = *key;
@@ -848,6 +853,7 @@ check_present_continuation (void *cls,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -856,13 +862,14 @@ check_present_continuation (void *cls,
  */
 static int
 check_present (void *cls,
-               const struct GNUNET_HashCode *key,
-               uint32_t size,
+               const struct GNUNET_HashCode *key,
+               uint32_t size,
                const void *data,
-               enum GNUNET_BLOCK_Type type,
-               uint32_t priority,
+               enum GNUNET_BLOCK_Type type,
+               uint32_t priority,
                uint32_t anonymity,
-               struct GNUNET_TIME_Absolute expiration,
+               uint32_t replication,
+               struct GNUNET_TIME_Absolute expiration,
                uint64_t uid)
 {
   struct PutContext *pc = cls;
@@ -883,16 +890,17 @@ check_present (void *cls,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Result already present in datastore\n");
-    /* FIXME: change API to allow increasing 'replication' counter */
-    if ((ntohl (dm->priority) > 0) ||
-        (GNUNET_TIME_absolute_ntoh (dm->expiration).abs_value_us >
-         expiration.abs_value_us))
+    if ( (ntohl (dm->priority) > 0) ||
+         (ntohl (dm->replication) > 0) ||
+         (GNUNET_TIME_absolute_ntoh (dm->expiration).abs_value_us >
+          expiration.abs_value_us) )
       plugin->api->update (plugin->api->cls,
-                           uid,
-                           (int32_t) ntohl (dm->priority),
+                           uid,
+                           ntohl (dm->priority),
+                           ntohl (dm->replication),
                            GNUNET_TIME_absolute_ntoh (dm->expiration),
                            &check_present_continuation,
-                           pc->client);
+                           pc->client);
     else
     {
       transmit_status (pc->client,
@@ -949,7 +957,7 @@ handle_put (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing PUT request for `%s' of type %u\n",
               GNUNET_h2s (&dm->key),
-              ntohl (dm->type));
+              (uint32_t) ntohl (dm->type));
   rid = ntohl (dm->rid);
   size = ntohl (dm->size);
   if (rid > 0)
@@ -984,12 +992,13 @@ handle_put (void *cls,
                         size,
                         &vhash);
     plugin->api->get_key (plugin->api->cls,
-                          0,
-                          &dm->key,
-                          &vhash,
+                          0,
+                          false,
+                          &dm->key,
+                          &vhash,
                           ntohl (dm->type),
-                          &check_present,
-                          pc);
+                          &check_present,
+                          pc);
     GNUNET_SERVICE_client_continue (client);
     return;
   }
@@ -1012,13 +1021,14 @@ handle_get (void *cls,
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing GET request of type %u\n",
-              ntohl (msg->type));
+              (uint32_t) ntohl (msg->type));
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# GET requests received"),
                             1,
                             GNUNET_NO);
   plugin->api->get_key (plugin->api->cls,
-                        GNUNET_ntohll (msg->offset),
+                        GNUNET_ntohll (msg->next_uid),
+                        msg->random,
                         NULL,
                         NULL,
                         ntohl (msg->type),
@@ -1043,7 +1053,7 @@ handle_get_key (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing GET request for `%s' of type %u\n",
               GNUNET_h2s (&msg->key),
-              ntohl (msg->type));
+              (uint32_t) ntohl (msg->type));
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# GET KEY requests received"),
                             1,
@@ -1062,14 +1072,15 @@ handle_get_key (void *cls,
                               1,
                               GNUNET_NO);
     transmit_item (client,
-                   NULL, 0, NULL, 0, 0, 0,
+                   NULL, 0, NULL, 0, 0, 0, 0,
                    GNUNET_TIME_UNIT_ZERO_ABS,
                    0);
     GNUNET_SERVICE_client_continue (client);
     return;
   }
   plugin->api->get_key (plugin->api->cls,
-                        GNUNET_ntohll (msg->offset),
+                        GNUNET_ntohll (msg->next_uid),
+                        msg->random,
                         &msg->key,
                         NULL,
                         ntohl (msg->type),
@@ -1080,55 +1091,6 @@ handle_get_key (void *cls,
 
 
 /**
- * Function called with the result of an update operation.
- *
- * @param cls closure
- * @param status #GNUNET_OK or #GNUNET_SYSERR
- * @param msg error message on error
- */
-static void
-update_continuation (void *cls,
-                     int status,
-                     const char *msg)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  transmit_status (client,
-                   status,
-                   msg);
-}
-
-
-/**
- * Handle UPDATE-message.
- *
- * @param cls client identification of the client
- * @param message the actual message
- */
-static void
-handle_update (void *cls,
-               const struct UpdateMessage *msg)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# UPDATE requests received"),
-                            1,
-                            GNUNET_NO);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Processing UPDATE request for %llu\n",
-              (unsigned long long) GNUNET_ntohll (msg->uid));
-  plugin->api->update (plugin->api->cls,
-                       GNUNET_ntohll (msg->uid),
-                       (int32_t) ntohl (msg->priority),
-                       GNUNET_TIME_absolute_ntoh (msg->expiration),
-                       &update_continuation,
-                       client);
-  GNUNET_SERVICE_client_continue (client);
-}
-
-
-/**
  * Handle GET_REPLICATION-message.
  *
  * @param cls identification of the client
@@ -1180,7 +1142,7 @@ handle_get_zero_anonymity (void *cls,
                             1,
                             GNUNET_NO);
   plugin->api->get_zero_anonymity (plugin->api->cls,
-                                   GNUNET_ntohll (msg->offset),
+                                   GNUNET_ntohll (msg->next_uid),
                                    type,
                                    &transmit_item,
                                    client);
@@ -1199,6 +1161,7 @@ handle_get_zero_anonymity (void *cls,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum
  * @return #GNUNET_OK to keep the item
@@ -1212,6 +1175,7 @@ remove_callback (void *cls,
                  enum GNUNET_BLOCK_Type type,
                  uint32_t priority,
                  uint32_t anonymity,
+                 uint32_t replication,
                  struct GNUNET_TIME_Absolute expiration,
                  uint64_t uid)
 {
@@ -1287,9 +1251,10 @@ handle_remove (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing REMOVE request for `%s' of type %u\n",
               GNUNET_h2s (&dm->key),
-              ntohl (dm->type));
+              (uint32_t) ntohl (dm->type));
   plugin->api->get_key (plugin->api->cls,
                         0,
+                        false,
                         &dm->key,
                         &vhash,
                         (enum GNUNET_BLOCK_Type) ntohl (dm->type),
@@ -1857,10 +1822,6 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
                         struct DataMessage,
                         NULL),
- GNUNET_MQ_hd_fixed_size (update,
-                          GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE,
-                          struct UpdateMessage,
-                          NULL),
  GNUNET_MQ_hd_fixed_size (get,
                           GNUNET_MESSAGE_TYPE_DATASTORE_GET,
                           struct GetMessage,
diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c
index 4f1f99a5c..10d9cf72e 100644
--- a/src/datastore/perf_datastore_api.c
+++ b/src/datastore/perf_datastore_api.c
@@ -303,6 +303,7 @@ remove_next (void *cls,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -315,6 +316,7 @@ delete_value (void *cls,
               enum GNUNET_BLOCK_Type type,
               uint32_t priority,
               uint32_t anonymity,
+              uint32_t replication,
               struct GNUNET_TIME_Absolute expiration,
               uint64_t uid)
 {
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
index 1e43b74b4..2f9502989 100644
--- a/src/datastore/perf_plugin_datastore.c
+++ b/src/datastore/perf_plugin_datastore.c
@@ -197,13 +197,14 @@ do_put (struct CpsRunContext *crc)
 
 static int
 iterate_zeros (void *cls,
-               const struct GNUNET_HashCode *key,
-               uint32_t size,
+               const struct GNUNET_HashCode *key,
+               uint32_t size,
                const void *data,
-               enum GNUNET_BLOCK_Type type,
-               uint32_t priority,
+               enum GNUNET_BLOCK_Type type,
+               uint32_t priority,
                uint32_t anonymity,
-               struct GNUNET_TIME_Absolute expiration,
+               uint32_t replication,
+               struct GNUNET_TIME_Absolute expiration,
                uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
@@ -253,14 +254,15 @@ iterate_zeros (void *cls,
 
 static int
 expiration_get (void *cls,
-                const struct GNUNET_HashCode *key,
-                uint32_t size,
+                const struct GNUNET_HashCode *key,
+                uint32_t size,
                 const void *data,
-                enum GNUNET_BLOCK_Type type,
+                enum GNUNET_BLOCK_Type type,
                 uint32_t priority,
-                uint32_t anonymity,
+                uint32_t anonymity,
+                uint32_t replication,
                 struct GNUNET_TIME_Absolute expiration,
-                uint64_t uid)
+                uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
   int i;
@@ -305,14 +307,15 @@ expiration_get (void *cls,
 
 static int
 replication_get (void *cls,
-                 const struct GNUNET_HashCode *key,
-                 uint32_t size,
+                 const struct GNUNET_HashCode *key,
+                 uint32_t size,
                  const void *data,
-                 enum GNUNET_BLOCK_Type type,
+                 enum GNUNET_BLOCK_Type type,
                  uint32_t priority,
-                 uint32_t anonymity,
+                 uint32_t anonymity,
+                 uint32_t replication,
                  struct GNUNET_TIME_Absolute expiration,
-                 uint64_t uid)
+                 uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
   int i;
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c
index 977d599d2..d04c1cf60 100644
--- a/src/datastore/plugin_datastore_heap.c
+++ b/src/datastore/plugin_datastore_heap.c
@@ -323,19 +323,19 @@ struct GetContext
 {
 
   /**
-   * Desired result offset / number of results.
+   * Lowest uid to consider.
    */
-  uint64_t offset;
+  uint64_t next_uid;
 
   /**
-   * The plugin.
+   * Value with lowest uid >= next_uid found so far.
    */
-  struct Plugin *plugin;
+  struct Value *value;
 
   /**
    * Requested value hash.
    */
-  const struct GNUNET_HashCode * vhash;
+  const struct GNUNET_HashCode *vhash;
 
   /**
    * Requested type.
@@ -343,68 +343,15 @@ struct GetContext
   enum GNUNET_BLOCK_Type type;
 
   /**
-   * Function to call with the result.
+   * If true, return a random value
    */
-  PluginDatumProcessor proc;
+  bool random;
 
-  /**
-   * Closure for 'proc'.
-   */
-  void *proc_cls;
 };
 
 
 /**
- * Test if a value matches the specification from the 'get' context
- *
- * @param gc query
- * @param value the value to check against the query
- * @return GNUNET_YES if the value matches
- */
-static int
-match (const struct GetContext *gc,
-       struct Value *value)
-{
-  struct GNUNET_HashCode vh;
-
-  if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) &&
-       (gc->type != value->type) )
-    return GNUNET_NO;
-  if (NULL != gc->vhash)
-  {
-    GNUNET_CRYPTO_hash (&value[1], value->size, &vh);
-    if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode)))
-      return GNUNET_NO;
-  }
-  return GNUNET_YES;
-}
-
-
-/**
- * Count number of matching values.
- *
- * @param cls the 'struct GetContext'
- * @param key unused
- * @param val the 'struct Value'
- * @return GNUNET_YES (continue iteration)
- */
-static int
-count_iterator (void *cls,
-                const struct GNUNET_HashCode *key,
-                void *val)
-{
-  struct GetContext *gc = cls;
-  struct Value *value = val;
-
-  if (GNUNET_NO == match (gc, value))
-    return GNUNET_OK;
-  gc->offset++;
-  return GNUNET_OK;
-}
-
-
-/**
- * Obtain matching value at 'offset'.
+ * Obtain the matching value with the lowest uid >= next_uid.
  *
  * @param cls the 'struct GetContext'
  * @param key unused
@@ -418,23 +365,29 @@ get_iterator (void *cls,
 {
   struct GetContext *gc = cls;
   struct Value *value = val;
+  struct GNUNET_HashCode vh;
 
-  if (GNUNET_NO == match (gc, value))
+  if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) &&
+       (gc->type != value->type) )
     return GNUNET_OK;
-  if (0 != gc->offset--)
+  if (NULL != gc->vhash)
+  {
+    GNUNET_CRYPTO_hash (&value[1], value->size, &vh);
+    if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode)))
+      return GNUNET_OK;
+  }
+  if (gc->random)
+  {
+    gc->value = value;
+    return GNUNET_NO;
+  }
+  if ( (uint64_t) (intptr_t) value < gc->next_uid)
     return GNUNET_OK;
-  if (GNUNET_NO ==
-      gc->proc (gc->proc_cls,
-                key,
-                value->size,
-                &value[1],
-                value->type,
-                value->priority,
-                value->anonymity,
-            value->expiration,
-                (uint64_t) (long) value))
-    delete_value (gc->plugin, value);
-  return GNUNET_NO;
+  if ( (NULL != gc->value) &&
+       (value > gc->value) )
+    return GNUNET_OK;
+  gc->value = value;
+  return GNUNET_OK;
 }
 
 
@@ -442,8 +395,8 @@ get_iterator (void *cls,
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -457,7 +410,7 @@ get_iterator (void *cls,
  * @param proc_cls closure for proc
  */
 static void
-heap_plugin_get_key (void *cls, uint64_t offset,
+heap_plugin_get_key (void *cls, uint64_t next_uid, bool random,
                      const struct GNUNET_HashCode *key,
                      const struct GNUNET_HashCode *vhash,
                      enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc,
@@ -466,25 +419,14 @@ heap_plugin_get_key (void *cls, uint64_t offset,
   struct Plugin *plugin = cls;
   struct GetContext gc;
 
-  gc.plugin = plugin;
-  gc.offset = 0;
+  gc.value = NULL;
+  gc.next_uid = next_uid;
+  gc.random = random;
   gc.vhash = vhash;
   gc.type = type;
-  gc.proc = proc;
-  gc.proc_cls = proc_cls;
   if (NULL == key)
   {
     GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
-                                           &count_iterator,
-                                           &gc);
-    if (0 == gc.offset)
-    {
-      proc (proc_cls,
-            NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-      return;
-    }
-    gc.offset = offset % gc.offset;
-    GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
                                            &get_iterator,
                                            &gc);
   }
@@ -492,20 +434,28 @@ heap_plugin_get_key (void *cls, uint64_t offset,
   {
     GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
                                                 key,
-                                                &count_iterator,
-                                                &gc);
-    if (0 == gc.offset)
-    {
-      proc (proc_cls,
-            NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-      return;
-    }
-    gc.offset = offset % gc.offset;
-    GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
-                                                key,
                                                 &get_iterator,
                                                 &gc);
   }
+  if (NULL == gc.value)
+  {
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    return;
+  }
+  if (GNUNET_NO ==
+      proc (proc_cls,
+            &gc.value->key,
+            gc.value->size,
+            &gc.value[1],
+            gc.value->type,
+            gc.value->priority,
+            gc.value->anonymity,
+            gc.value->replication,
+            gc.value->expiration,
+            (uint64_t) (intptr_t) gc.value))
+  {
+    delete_value (plugin, gc.value);
+  }
 }
 
 
@@ -531,8 +481,7 @@ heap_plugin_get_replication (void *cls,
   value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication);
   if (NULL == value)
   {
-    proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
   if (value->replication > 0)
@@ -552,14 +501,15 @@ heap_plugin_get_replication (void *cls,
   }
   if (GNUNET_NO ==
       proc (proc_cls,
-            &value->key,
-            value->size,
-            &value[1],
-            value->type,
-            value->priority,
-            value->anonymity,
-            value->expiration,
-            (uint64_t) (long) value))
+            &value->key,
+            value->size,
+            &value[1],
+            value->type,
+            value->priority,
+            value->anonymity,
+            value->replication,
+            value->expiration,
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
@@ -582,38 +532,37 @@ heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
   value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration);
   if (NULL == value)
   {
-    proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
   if (GNUNET_NO ==
       proc (proc_cls,
-            &value->key,
-            value->size,
-            &value[1],
-            value->type,
-            value->priority,
-            value->anonymity,
-            value->expiration,
-            (uint64_t) (long) value))
+            &value->key,
+            value->size,
+            &value[1],
+            value->type,
+            value->priority,
+            value->anonymity,
+            value->replication,
+            value->expiration,
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls our `struct Plugin *`
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
@@ -622,15 +571,16 @@ heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
  */
 static void
 heap_plugin_update (void *cls,
-                    uint64_t uid,
-                    int delta,
-                    struct GNUNET_TIME_Absolute expire,
-                    PluginUpdateCont cont,
-                    void *cont_cls)
+                    uint64_t uid,
+                    uint32_t priority,
+                    uint32_t replication,
+                    struct GNUNET_TIME_Absolute expire,
+                    PluginUpdateCont cont,
+                    void *cont_cls)
 {
   struct Value *value;
 
-  value = (struct Value*) (long) uid;
+  value = (struct Value*) (intptr_t) uid;
   GNUNET_assert (NULL != value);
   if (value->expiration.abs_value_us != expire.abs_value_us)
   {
@@ -638,10 +588,15 @@ heap_plugin_update (void *cls,
     GNUNET_CONTAINER_heap_update_cost (value->expire_heap,
                                        expire.abs_value_us);
   }
-  if ( (delta < 0) && (value->priority < - delta) )
-    value->priority = 0;
+  /* Saturating adds, don't overflow */
+  if (value->priority > UINT32_MAX - priority)
+    value->priority = UINT32_MAX;
+  else
+    value->priority += priority;
+  if (value->replication > UINT32_MAX - replication)
+    value->replication = UINT32_MAX;
   else
-    value->priority += delta;
+    value->replication += replication;
   cont (cont_cls, GNUNET_OK, NULL);
 }
 
@@ -650,63 +605,53 @@ heap_plugin_update (void *cls,
  * Call the given processor on an item with zero anonymity.
  *
  * @param cls our "struct Plugin*"
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
+ *        Must not be zero (ANY).
  * @param proc function to call on each matching value;
- *        will be called  with NULL if no value matches
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for proc
  */
 static void
-heap_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
                                 enum GNUNET_BLOCK_Type type,
                                 PluginDatumProcessor proc, void *proc_cls)
 {
   struct Plugin *plugin = cls;
   struct ZeroAnonByType *zabt;
-  struct Value *value;
-  uint64_t count;
+  struct Value *value = NULL;
 
-  count = 0;
   for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
   {
     if ( (type != GNUNET_BLOCK_TYPE_ANY) &&
-         (type != zabt->type) )
+         (type != zabt->type) )
       continue;
-    count += zabt->array_pos;
+    for (int i = 0; i < zabt->array_pos; ++i)
+    {
+      if ( (uint64_t) (intptr_t) zabt->array[i] < next_uid)
+        continue;
+      if ( (NULL != value) &&
+           (zabt->array[i] > value) )
+        continue;
+      value = zabt->array[i];
+    }
   }
-  if (0 == count)
+  if (NULL == value)
   {
-    proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  offset = offset % count;
-  for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
-  {
-    if ( (type != GNUNET_BLOCK_TYPE_ANY) &&
-         (type != zabt->type) )
-      continue;
-    if (offset >= zabt->array_pos)
-    {
-      offset -= zabt->array_pos;
-      continue;
-    }
-    break;
-  }
-  GNUNET_assert (NULL != zabt);
-  value = zabt->array[offset];
   if (GNUNET_NO ==
       proc (proc_cls,
-            &value->key,
-            value->size,
-            &value[1],
-            value->type,
-            value->priority,
-            value->anonymity,
-            value->expiration,
-            (uint64_t) (long) value))
+            &value->key,
+            value->size,
+            &value[1],
+            value->type,
+            value->priority,
+            value->anonymity,
+            value->replication,
+            value->expiration,
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index d76b4ccb4..6f2a76499 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -150,31 +150,56 @@ struct Plugin
 #define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
   struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid;
 
-#define COUNT_ENTRY_BY_HASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash) WHERE hash=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash;
-
-#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash) WHERE hash=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid"
+
+#define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "WHERE uid >= ? AND "\
+  "(rvalue >= ? OR 0 = ?) "\
+  "ORDER BY uid LIMIT 1"
+  struct GNUNET_MYSQL_StatementHandle *select_entry;
+
+#define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_hash) "\
+  "WHERE hash=? AND "\
+  "uid >= ? AND "\
+  "(rvalue >= ? OR 0 = ?) "\
+  "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash;
 
-#define COUNT_ENTRY_BY_HASH_AND_VHASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_and_vhash;
-
-#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_hash_vhash) "\
+  "WHERE hash = ? AND "\
+  "vhash = ? AND "\
+  "uid >= ? AND "\
+  "(rvalue >= ? OR 0 = ?) "\
+  "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_vhash;
 
-#define COUNT_ENTRY_BY_HASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_and_type;
-
-#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_hash_type_uid) "\
+  "WHERE hash = ? AND "\
+  "type = ? AND "\
+  "uid >= ? AND "\
+  "(rvalue >= ? OR 0 = ?) "\
+  "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type;
 
-#define COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_vhash_and_type;
-
-#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=? ORDER BY uid ASC LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT " RESULT_COLUMNS " "\
+  "FROM gn090 "\
+  "FORCE INDEX (idx_hash_vhash) "\
+  "WHERE hash = ? AND "\
+  "vhash = ? AND "\
+  "type = ? AND "\
+  "uid >= ? AND "\
+  "(rvalue >= ? OR 0 = ?) "\
+  "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_vhash_and_type;
 
-#define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE uid=?"
+#define UPDATE_ENTRY "UPDATE gn090 SET "\
+  "prio = prio + ?, "\
+  "repl = repl + ?, "\
+  "expire = IF(expire >= ?, expire, ?) "\
+  "WHERE uid = ?"
   struct GNUNET_MYSQL_StatementHandle *update_entry;
 
 #define DEC_REPL "UPDATE gn090 SET repl=GREATEST (1, repl) - 1 WHERE uid=?"
@@ -183,22 +208,27 @@ struct Plugin
 #define SELECT_SIZE "SELECT SUM(LENGTH(value)+256) FROM gn090"
   struct GNUNET_MYSQL_StatementHandle *get_size;
 
-#define SELECT_IT_NON_ANONYMOUS "SELECT type,prio,anonLevel,expire,hash,value,uid "\
-   "FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) "\
-   "WHERE anonLevel=0 AND type=? AND "\
-   "(rvalue >= ? OR"\
-   "  NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) WHERE anonLevel=0 AND type=? AND rvalue>=?)) "\
-   "ORDER BY rvalue ASC LIMIT 1"
+#define SELECT_IT_NON_ANONYMOUS "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_anonLevel_type_rvalue) "\
+  "WHERE anonLevel=0 AND "\
+  "type=? AND "\
+  "uid >= ? "\
+  "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *zero_iter;
 
-#define SELECT_IT_EXPIRATION "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_expire) WHERE expire < ? ORDER BY expire ASC LIMIT 1"
+#define SELECT_IT_EXPIRATION "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_expire) "\
+  "WHERE expire < ? "\
+  "ORDER BY expire ASC LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_expiration;
 
-#define SELECT_IT_PRIORITY "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_prio) ORDER BY prio ASC LIMIT 1"
+#define SELECT_IT_PRIORITY "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_prio) "\
+  "ORDER BY prio ASC LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_priority;
 
-#define SELECT_IT_REPLICATION "SELECT type,prio,anonLevel,expire,hash,value,uid "\
-  "FROM gn090 FORCE INDEX (idx_repl_rvalue) "\
+#define SELECT_IT_REPLICATION "SELECT " RESULT_COLUMNS " FROM gn090 "\
+  "FORCE INDEX (idx_repl_rvalue) "\
   "WHERE repl=? AND "\
   " (rvalue>=? OR"\
   "  NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_repl_rvalue) WHERE repl=? AND rvalue>=?)) "\
@@ -382,22 +412,18 @@ mysql_plugin_put (void *cls,
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
- *
- * Note that it is possible for multiple values to match this put.
- * In that case, all of the respective values are updated.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls our "struct Plugin*"
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
@@ -407,24 +433,26 @@ mysql_plugin_put (void *cls,
 static void
 mysql_plugin_update (void *cls,
                      uint64_t uid,
-                     int delta,
+                     uint32_t priority,
+                     uint32_t replication,
                      struct GNUNET_TIME_Absolute expire,
                      PluginUpdateCont cont,
                      void *cont_cls)
 {
   struct Plugin *plugin = cls;
-  uint32_t idelta = (uint32_t) delta;
   uint64_t lexpire = expire.abs_value_us;
   int ret;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Updating value %llu adding %d to priority and maxing exp at %s\n",
+              "Updating value %llu adding %d to priority %d to replication and maxing exp at %s\n",
               (unsigned long long) uid,
-              delta,
-              GNUNET_STRINGS_absolute_time_to_string (expire));
+              priority,
+              replication,
+              GNUNET_STRINGS_absolute_time_to_string (expire));
 
   struct GNUNET_MY_QueryParam params_update[] = {
-    GNUNET_MY_query_param_uint32 (&idelta),
+    GNUNET_MY_query_param_uint32 (&priority),
+    GNUNET_MY_query_param_uint32 (&replication),
     GNUNET_MY_query_param_uint64 (&lexpire),
     GNUNET_MY_query_param_uint64 (&lexpire),
     GNUNET_MY_query_param_uint64 (&uid),
@@ -471,6 +499,7 @@ execute_select (struct Plugin *plugin,
                 struct GNUNET_MY_QueryParam *params_select)
 {
   int ret;
+  uint32_t replication;
   uint32_t type;
   uint32_t priority;
   uint32_t anonymity;
@@ -480,6 +509,7 @@ execute_select (struct Plugin *plugin,
   struct GNUNET_HashCode key;
   struct GNUNET_TIME_Absolute expiration;
   struct GNUNET_MY_ResultSpec results_select[] = {
+    GNUNET_MY_result_spec_uint32 (&replication),
     GNUNET_MY_result_spec_uint32 (&type),
     GNUNET_MY_result_spec_uint32 (&priority),
     GNUNET_MY_result_spec_uint32 (&anonymity),
@@ -496,7 +526,7 @@ execute_select (struct Plugin *plugin,
   if (GNUNET_OK != ret)
   {
     proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+          NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
 
@@ -505,7 +535,7 @@ execute_select (struct Plugin *plugin,
   if (GNUNET_OK != ret)
   {
     proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+          NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
 
@@ -527,6 +557,7 @@ execute_select (struct Plugin *plugin,
               type,
               priority,
               anonymity,
+              replication,
               expiration,
               uid);
   GNUNET_MY_cleanup_result (results_select);
@@ -544,8 +575,8 @@ execute_select (struct Plugin *plugin,
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key key to match, never NULL
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -560,7 +591,8 @@ execute_select (struct Plugin *plugin,
  */
 static void
 mysql_plugin_get_key (void *cls,
-                      uint64_t offset,
+                      uint64_t next_uid,
+                      bool random,
                       const struct GNUNET_HashCode *key,
                       const struct GNUNET_HashCode *vhash,
                       enum GNUNET_BLOCK_Type type,
@@ -568,121 +600,33 @@ mysql_plugin_get_key (void *cls,
                       void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  int ret;
-  uint64_t total;
-  struct GNUNET_MY_ResultSpec results_get[] = {
-    GNUNET_MY_result_spec_uint64 (&total),
-    GNUNET_MY_result_spec_end
-  };
+  uint64_t rvalue;
 
-  total = UINT64_MAX;
-  if (0 != type)
+  if (random)
   {
-    if (NULL != vhash)
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_vhash_and_type,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_vhash_and_type,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_vhash_and_type,
-                                                NULL));
-    }
-    else
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_and_type,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_type,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_type,
-                                                NULL));
-    }
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
   else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_end
-      };
+    rvalue = 0;
 
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_and_vhash,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_vhash,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_vhash,
-                                                NULL));
-    }
-    else
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash,
-                                                NULL));
-    }
-  }
-  if ( (GNUNET_OK != ret) ||
-       (0 >= total) )
+  if (NULL == key)
   {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
+    struct GNUNET_MY_QueryParam params_select[] = {
+      GNUNET_MY_query_param_uint64 (&next_uid),
+      GNUNET_MY_query_param_uint64 (&rvalue),
+      GNUNET_MY_query_param_uint64 (&rvalue),
+      GNUNET_MY_query_param_end
+    };
+
+    execute_select (plugin,
+                    plugin->select_entry,
+                    proc,
+                    proc_cls,
+                    params_select);
   }
-  offset = offset % total;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Obtaining %llu/%lld result for GET `%s'\n",
-              (unsigned long long) offset,
-              (unsigned long long) total,
-              GNUNET_h2s (key));
-  if (type != GNUNET_BLOCK_TYPE_ANY)
+  else if (type != GNUNET_BLOCK_TYPE_ANY)
   {
     if (NULL != vhash)
     {
@@ -690,7 +634,9 @@ mysql_plugin_get_key (void *cls,
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_auto_from_type (vhash),
         GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -705,7 +651,9 @@ mysql_plugin_get_key (void *cls,
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -723,7 +671,9 @@ mysql_plugin_get_key (void *cls,
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -737,7 +687,9 @@ mysql_plugin_get_key (void *cls,
     {
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -756,28 +708,26 @@ mysql_plugin_get_key (void *cls,
  * Get a zero-anonymity datum from the datastore.
  *
  * @param cls our `struct Plugin *`
- * @param offset offset of the result
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on a matching value or NULL
+ *        Must not be zero (ANY).
+ * @param proc function to call on a matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
 mysql_plugin_get_zero_anonymity (void *cls,
-                                 uint64_t offset,
+                                 uint64_t next_uid,
                                  enum GNUNET_BLOCK_Type type,
                                  PluginDatumProcessor proc,
                                  void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t typei = (uint32_t) type;
-  uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                              UINT64_MAX);
+
   struct GNUNET_MY_QueryParam params_zero_iter[] = {
     GNUNET_MY_query_param_uint32 (&typei),
-    GNUNET_MY_query_param_uint64 (&rvalue),
-    GNUNET_MY_query_param_uint32 (&typei),
-    GNUNET_MY_query_param_uint64 (&rvalue),
+    GNUNET_MY_query_param_uint64 (&next_uid),
     GNUNET_MY_query_param_end
   };
 
@@ -824,6 +774,7 @@ struct ReplCtx
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -839,6 +790,7 @@ repl_proc (void *cls,
            enum GNUNET_BLOCK_Type type,
            uint32_t priority,
            uint32_t anonymity,
+           uint32_t replication,
            struct GNUNET_TIME_Absolute expiration,
            uint64_t uid)
 {
@@ -854,6 +806,7 @@ repl_proc (void *cls,
                   type,
                   priority,
                   anonymity,
+                  replication,
                   expiration,
                   uid);
   if (NULL != key)
@@ -921,7 +874,7 @@ mysql_plugin_get_replication (void *cls,
                                plugin->max_repl,
                                params_get))
   {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
 
@@ -929,7 +882,7 @@ mysql_plugin_get_replication (void *cls,
       GNUNET_MY_extract_result (plugin->max_repl,
                                 results_get))
   {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
   GNUNET_break (GNUNET_NO ==
@@ -1071,6 +1024,7 @@ struct ExpiCtx
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -1086,6 +1040,7 @@ expi_proc (void *cls,
            enum GNUNET_BLOCK_Type type,
            uint32_t priority,
            uint32_t anonymity,
+           uint32_t replication,
            struct GNUNET_TIME_Absolute expiration,
            uint64_t uid)
 {
@@ -1111,6 +1066,7 @@ expi_proc (void *cls,
                    type,
                    priority,
                    anonymity,
+                   replication,
                    expiration,
                    uid);
 }
@@ -1212,6 +1168,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
        ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") ||
       PINIT (plugin->insert_entry, INSERT_ENTRY) ||
       PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) ||
+      PINIT (plugin->select_entry, SELECT_ENTRY) ||
       PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) ||
       PINIT (plugin->select_entry_by_hash_and_vhash,
              SELECT_ENTRY_BY_HASH_AND_VHASH) ||
@@ -1219,13 +1176,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
              SELECT_ENTRY_BY_HASH_AND_TYPE) ||
       PINIT (plugin->select_entry_by_hash_vhash_and_type,
              SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) ||
-      PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH) ||
       PINIT (plugin->get_size, SELECT_SIZE) ||
-      PINIT (plugin->count_entry_by_hash_and_vhash,
-             COUNT_ENTRY_BY_HASH_AND_VHASH) ||
-      PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE)
-      || PINIT (plugin->count_entry_by_hash_vhash_and_type,
-                COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE) ||
       PINIT (plugin->update_entry, UPDATE_ENTRY) ||
       PINIT (plugin->dec_repl, DEC_REPL) ||
       PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) ||
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index 994118bfa..87a7acbdc 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -76,6 +76,12 @@ init_connection (struct Plugin *plugin)
   if (NULL == plugin->dbh)
     return GNUNET_SYSERR;
 
+  /* FIXME: PostgreSQL does not have unsigned integers! This is ok for the type column because
+   * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
+   * we do math or inequality tests, so we can't handle the entire range of uint32_t.
+   * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
+   * PostgreSQL also recommends against using WITH OIDS.
+   */
   ret =
       PQexec (plugin->dbh,
               "CREATE TABLE IF NOT EXISTS gn090 ("
@@ -109,13 +115,17 @@ init_connection (struct Plugin *plugin)
   if (PQresultStatus (ret) == PGRES_COMMAND_OK)
   {
     if ((GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)")) ||
         (GNUNET_OK !=
          GNUNET_POSTGRES_exec (plugin->dbh,
                                "CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel)")) ||
@@ -123,9 +133,11 @@ init_connection (struct Plugin *plugin)
          GNUNET_POSTGRES_exec (plugin->dbh,
                                "CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)")))
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)")))
     {
       PQclear (ret);
       PQfinish (plugin->dbh);
@@ -164,64 +176,49 @@ init_connection (struct Plugin *plugin)
     return GNUNET_SYSERR;
   }
   PQclear (ret);
+#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, oid"
   if ((GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "getvt",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND vhash=$2 AND type=$3 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $4", 4)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "gett",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND type=$2 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "getv",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND vhash=$2 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3)) ||
-      (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "get",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 " "ORDER BY oid ASC LIMIT 1 OFFSET $2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_getvt",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2 AND type=$3", 3)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_gett",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND type=$2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_getv",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_get",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1", 1)) ||
+                   "SELECT " RESULT_COLUMNS " FROM gn090 "
+                   "WHERE oid >= $1::bigint AND "
+                   "(rvalue >= $2 OR 0 = $3::smallint) AND "
+                   "(hash = $4 OR 0 = $5::smallint) AND "
+                   "(vhash = $6 OR 0 = $7::smallint) AND "
+                   "(type = $8 OR 0 = $9::smallint) "
+                   "ORDER BY oid ASC LIMIT 1", 9)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "put",
                    "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
-                   "VALUES ($1, $2, $3, $4, $5, RANDOM(), $6, $7, $8)", 9)) ||
+                   "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", 9)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "update",
-                   "UPDATE gn090 SET prio = prio + $1, expire = CASE WHEN expire < $2 THEN $2 ELSE expire END "
-                   "WHERE oid = $3", 3)) ||
+                   "UPDATE gn090 "
+                   "SET prio = prio + $1, "
+                   "repl = repl + $2, "
+                   "expire = CASE WHEN expire < $3 THEN $3 ELSE expire END "
+                   "WHERE oid = $4", 4)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "decrepl",
                    "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) "
                    "WHERE oid = $1", 1)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "select_non_anonymous",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE anonLevel = 0 AND type = $1 ORDER BY oid DESC LIMIT 1 OFFSET $2",
-                   1)) ||
+                   "SELECT " RESULT_COLUMNS " FROM gn090 "
+                   "WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint "
+                   "ORDER BY oid ASC LIMIT 1",
+                   2)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "select_expiration_order",
-                   "(SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE expire < $1 ORDER BY prio ASC LIMIT 1) " "UNION "
-                   "(SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "ORDER BY prio ASC LIMIT 1) " "ORDER BY expire ASC LIMIT 1",
+                   "(SELECT " RESULT_COLUMNS " FROM gn090 "
+                    "WHERE expire < $1 ORDER BY prio ASC LIMIT 1) "
+                   "UNION "
+                   "(SELECT " RESULT_COLUMNS " FROM gn090 "
+                    "ORDER BY prio ASC LIMIT 1) "
+                   "ORDER BY expire ASC LIMIT 1",
                    1)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "select_replication_order",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
+                   "SELECT " RESULT_COLUMNS " FROM gn090 "
                    "ORDER BY repl DESC,RANDOM() LIMIT 1", 0)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "delrow", "DELETE FROM gn090 " "WHERE oid=$1", 1)) ||
@@ -317,6 +314,8 @@ postgres_plugin_put (void *cls,
   struct Plugin *plugin = cls;
   uint32_t utype = type;
   struct GNUNET_HashCode vhash;
+  uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                              UINT64_MAX);
   PGresult *ret;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_uint32 (&replication),
@@ -324,6 +323,7 @@ postgres_plugin_put (void *cls,
     GNUNET_PQ_query_param_uint32 (&priority),
     GNUNET_PQ_query_param_uint32 (&anonymity),
     GNUNET_PQ_query_param_absolute_time (&expiration),
+    GNUNET_PQ_query_param_uint64 (&rvalue),
     GNUNET_PQ_query_param_auto_from_type (key),
     GNUNET_PQ_query_param_auto_from_type (&vhash),
     GNUNET_PQ_query_param_fixed_size (data, size),
@@ -377,19 +377,21 @@ process_result (struct Plugin *plugin,
   uint32_t rowid;
   uint32_t utype;
   uint32_t anonymity;
+  uint32_t replication;
   uint32_t priority;
   size_t size;
   void *data;
   struct GNUNET_TIME_Absolute expiration_time;
   struct GNUNET_HashCode key;
   struct GNUNET_PQ_ResultSpec rs[] = {
+    GNUNET_PQ_result_spec_uint32 ("repl", &replication),
     GNUNET_PQ_result_spec_uint32 ("type", &utype),
     GNUNET_PQ_result_spec_uint32 ("prio", &priority),
     GNUNET_PQ_result_spec_uint32 ("anonLevel", &anonymity),
-    GNUNET_PQ_result_spec_uint32 ("oid", &rowid),
     GNUNET_PQ_result_spec_absolute_time ("expire", &expiration_time),
     GNUNET_PQ_result_spec_auto_from_type ("hash", &key),
     GNUNET_PQ_result_spec_variable_size ("value", &data, &size),
+    GNUNET_PQ_result_spec_uint32 ("oid", &rowid),
     GNUNET_PQ_result_spec_end
   };
 
@@ -404,8 +406,7 @@ process_result (struct Plugin *plugin,
     GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                      "datastore-postgres",
                      "Ending iteration (postgres error)\n");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
 
@@ -415,16 +416,14 @@ process_result (struct Plugin *plugin,
     GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                      "datastore-postgres",
                      "Ending iteration (no more results)\n");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     PQclear (res);
     return;
   }
   if (1 != PQntuples (res))
   {
     GNUNET_break (0);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     PQclear (res);
     return;
   }
@@ -438,8 +437,7 @@ process_result (struct Plugin *plugin,
     GNUNET_POSTGRES_delete_by_rowid (plugin->dbh,
                                      "delrow",
                                      rowid);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
 
@@ -449,14 +447,15 @@ process_result (struct Plugin *plugin,
                    (unsigned int) size,
                    (unsigned int) utype);
   iret = proc (proc_cls,
-               &key,
-               size,
-               data,
-               (enum GNUNET_BLOCK_Type) utype,
-               priority,
-               anonymity,
-               expiration_time,
-               rowid);
+               &key,
+               size,
+               data,
+               (enum GNUNET_BLOCK_Type) utype,
+               priority,
+               anonymity,
+               replication,
+               expiration_time,
+               rowid);
   PQclear (res);
   if (iret == GNUNET_NO)
   {
@@ -484,12 +483,11 @@ process_result (struct Plugin *plugin,
 
 
 /**
- * Iterate over the results for a particular key
- * in the datastore.
+ * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure with the 'struct Plugin'
- * @param offset offset of the result (modulo num-results);
- *        specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -499,160 +497,52 @@ process_result (struct Plugin *plugin,
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
  * @param proc function to call on the matching value;
- *        will be called once with a NULL if no value matches
- * @param proc_cls closure for iter
+ *        will be called with NULL if nothing matches
+ * @param proc_cls closure for @a proc
  */
 static void
 postgres_plugin_get_key (void *cls,
-                         uint64_t offset,
+                         uint64_t next_uid,
+                         bool random,
                          const struct GNUNET_HashCode *key,
                          const struct GNUNET_HashCode *vhash,
                          enum GNUNET_BLOCK_Type type,
-                         PluginDatumProcessor proc,
+                         PluginDatumProcessor proc,
                          void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t utype = type;
+  uint16_t use_rvalue = random;
+  uint16_t use_key = NULL != key;
+  uint16_t use_vhash = NULL != vhash;
+  uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
+  uint64_t rvalue;
+  struct GNUNET_PQ_QueryParam params[] = {
+    GNUNET_PQ_query_param_uint64 (&next_uid),
+    GNUNET_PQ_query_param_uint64 (&rvalue),
+    GNUNET_PQ_query_param_uint16 (&use_rvalue),
+    GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_uint16 (&use_key),
+    GNUNET_PQ_query_param_auto_from_type (vhash),
+    GNUNET_PQ_query_param_uint16 (&use_vhash),
+    GNUNET_PQ_query_param_uint32 (&utype),
+    GNUNET_PQ_query_param_uint16 (&use_type),
+    GNUNET_PQ_query_param_end
+  };
   PGresult *ret;
-  uint64_t total;
-  uint64_t limit_off;
 
-  if (0 != type)
+  if (random)
   {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_getvt",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_gett",
-                                     params);
-    }
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
   else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_getv",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_get",
-                                     params);
-    }
-  }
+    rvalue = 0;
 
-  if (GNUNET_OK !=
-      GNUNET_POSTGRES_check_result (plugin->dbh,
-                                    ret,
-                                    PGRES_TUPLES_OK,
-                                    "PQexecParams",
-                                    "count"))
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  if ( (PQntuples (ret) != 1) ||
-       (PQnfields (ret) != 1) ||
-       (PQgetlength (ret, 0, 0) != sizeof (uint64_t)))
-  {
-    GNUNET_break (0);
-    PQclear (ret);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  total = GNUNET_ntohll (*(const uint64_t *) PQgetvalue (ret, 0, 0));
-  PQclear (ret);
-  if (0 == total)
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  limit_off = offset % total;
-
-  if (0 != type)
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "getvt",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "gett",
-                                     params);
-    }
-  }
-  else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "getv",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "get",
-                                     params);
-    }
-  }
+  ret = GNUNET_PQ_exec_prepared (plugin->dbh,
+                                 "get",
+                                 params);
   process_result (plugin,
                   proc,
                   proc_cls,
@@ -666,26 +556,25 @@ postgres_plugin_get_key (void *cls,
  * the given iterator for each of them.
  *
  * @param cls our `struct Plugin *`
- * @param offset offset of the result (modulo num-results);
- *        specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
+ *        Must not be zero (ANY).
  * @param proc function to call on the matching value;
- *        will be called with a NULL if no value matches
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
 postgres_plugin_get_zero_anonymity (void *cls,
-                                    uint64_t offset,
+                                    uint64_t next_uid,
                                     enum GNUNET_BLOCK_Type type,
                                     PluginDatumProcessor proc,
-                                    void *proc_cls)
+                                    void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t utype = type;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_uint32 (&utype),
-    GNUNET_PQ_query_param_uint64 (&offset),
+    GNUNET_PQ_query_param_uint64 (&next_uid),
     GNUNET_PQ_query_param_end
   };
   PGresult *ret;
@@ -736,6 +625,7 @@ struct ReplCtx
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -746,13 +636,14 @@ struct ReplCtx
  */
 static int
 repl_proc (void *cls,
-           const struct GNUNET_HashCode *key,
-           uint32_t size,
+           const struct GNUNET_HashCode *key,
+           uint32_t size,
            const void *data,
-           enum GNUNET_BLOCK_Type type,
-           uint32_t priority,
+           enum GNUNET_BLOCK_Type type,
+           uint32_t priority,
            uint32_t anonymity,
-           struct GNUNET_TIME_Absolute expiration,
+           uint32_t replication,
+           struct GNUNET_TIME_Absolute expiration,
            uint64_t uid)
 {
   struct ReplCtx *rc = cls;
@@ -766,12 +657,15 @@ repl_proc (void *cls,
   PGresult *qret;
 
   ret = rc->proc (rc->proc_cls,
-                  key,
-                  size, data,
-                  type,
-                  priority,
-                  anonymity,
-                  expiration, uid);
+                  key,
+                  size,
+                  data,
+                  type,
+                  priority,
+                  anonymity,
+                  replication,
+                  expiration,
+                  uid);
   if (NULL == key)
     return ret;
   qret = GNUNET_PQ_exec_prepared (plugin->dbh,
@@ -856,22 +750,18 @@ postgres_plugin_get_expiration (void *cls,
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
- *
- * Note that it is possible for multiple values to match this put.
- * In that case, all of the respective values are updated.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls our `struct Plugin *`
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
@@ -880,17 +770,18 @@ postgres_plugin_get_expiration (void *cls,
  */
 static void
 postgres_plugin_update (void *cls,
-                        uint64_t uid,
-                        int delta,
+                        uint64_t uid,
+                        uint32_t priority,
+                        uint32_t replication,
                         struct GNUNET_TIME_Absolute expire,
                         PluginUpdateCont cont,
-                        void *cont_cls)
+                        void *cont_cls)
 {
   struct Plugin *plugin = cls;
-  uint32_t idelta = delta;
   uint32_t oid = (uint32_t) uid;
   struct GNUNET_PQ_QueryParam params[] = {
-    GNUNET_PQ_query_param_uint32 (&idelta),
+    GNUNET_PQ_query_param_uint32 (&priority),
+    GNUNET_PQ_query_param_uint32 (&replication),
     GNUNET_PQ_query_param_absolute_time (&expire),
     GNUNET_PQ_query_param_uint32 (&oid),
     GNUNET_PQ_query_param_end
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index 18a3aa4ac..1f874e190 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -1,6 +1,6 @@
  /*
   * This file is part of GNUnet
-  * Copyright (C) 2009, 2011 GNUnet e.V.
+  * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
   *
   * GNUnet is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published
@@ -26,6 +26,7 @@
 
 #include "platform.h"
 #include "gnunet_datastore_plugin.h"
+#include "gnunet_sq_lib.h"
 #include <sqlite3.h>
 
 
@@ -127,6 +128,11 @@ struct Plugin
   sqlite3_stmt *insertContent;
 
   /**
+   * Precompiled SQL for selection
+   */
+  sqlite3_stmt *get;
+
+  /**
    * Should the database be dropped on shutdown?
    */
   int drop_on_shutdown;
@@ -150,11 +156,17 @@ sq_prepare (sqlite3 *dbh,
   char *dummy;
   int result;
 
-  result =
-      sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
-                          (const char **) &dummy);
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
+  result = sqlite3_prepare_v2 (dbh,
+                               zSql,
+                               strlen (zSql),
+                               ppStmt,
+                               (const char **) &dummy);
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
+                   "Prepared `%s' / %p: %d\n",
+                   zSql,
+                   *ppStmt,
+                   result);
   return result;
 }
 
@@ -229,35 +241,41 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
 {
   sqlite3_stmt *stmt;
   char *afsdir;
-
 #if ENULL_DEFINED
   char *e;
 #endif
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (cfg, "datastore-sqlite",
-                                               "FILENAME", &afsdir))
+      GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                               "datastore-sqlite",
+                                               "FILENAME",
+                                               &afsdir))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
-                               "datastore-sqlite", "FILENAME");
+                               "datastore-sqlite",
+                               "FILENAME");
     return GNUNET_SYSERR;
   }
   if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
   {
-    if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir))
+    if (GNUNET_OK !=
+        GNUNET_DISK_directory_create_for_file (afsdir))
     {
       GNUNET_break (0);
       GNUNET_free (afsdir);
       return GNUNET_SYSERR;
     }
     /* database is new or got deleted, reset payload to zero! */
-    plugin->env->duc (plugin->env->cls, 0);
+    if (NULL != plugin->env->duc)
+      plugin->env->duc (plugin->env->cls,
+                        0);
   }
   /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
   plugin->fn = afsdir;
 
   /* Open database and precompile statements */
-  if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK)
+  if (SQLITE_OK !=
+      sqlite3_open (plugin->fn, &plugin->dbh))
   {
     GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite",
                      _("Unable to initialize SQLite: %s.\n"),
@@ -265,25 +283,32 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
     return GNUNET_SYSERR;
   }
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA temp_store=MEMORY", NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA synchronous=OFF", NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA legacy_file_format=OFF", NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL", NULL,
                        NULL, ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA page_size=4092", NULL, NULL,
                        ENULL));
 
-  CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
+  CHECK (SQLITE_OK ==
+         sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
 
 
   /* We have to do it here, because otherwise precompiling SQL might fail */
@@ -291,80 +316,120 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
          sq_prepare (plugin->dbh,
                      "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn090'",
                      &stmt));
-  if ((sqlite3_step (stmt) == SQLITE_DONE) &&
-      (sqlite3_exec
-       (plugin->dbh,
-        "CREATE TABLE gn090 (" "  repl INT4 NOT NULL DEFAULT 0,"
-        "  type INT4 NOT NULL DEFAULT 0," "  prio INT4 NOT NULL DEFAULT 0,"
-        "  anonLevel INT4 NOT NULL DEFAULT 0,"
-        "  expire INT8 NOT NULL DEFAULT 0," "  rvalue INT8 NOT NULL,"
-        "  hash TEXT NOT NULL DEFAULT ''," "  vhash TEXT NOT NULL DEFAULT '',"
-        "  value BLOB NOT NULL DEFAULT '')", NULL, NULL, NULL) != SQLITE_OK))
+
+  /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
+   * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
+   * we do math or inequality tests, so we can't handle the entire range of uint32_t.
+   * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
+   */
+  if ( (SQLITE_DONE ==
+        sqlite3_step (stmt)) &&
+       (SQLITE_OK !=
+        sqlite3_exec (plugin->dbh,
+                      "CREATE TABLE gn090 ("
+                      "  repl INT4 NOT NULL DEFAULT 0,"
+                      "  type INT4 NOT NULL DEFAULT 0,"
+                      "  prio INT4 NOT NULL DEFAULT 0,"
+                      "  anonLevel INT4 NOT NULL DEFAULT 0,"
+                      "  expire INT8 NOT NULL DEFAULT 0,"
+                      "  rvalue INT8 NOT NULL,"
+                      "  hash TEXT NOT NULL DEFAULT '',"
+                      "  vhash TEXT NOT NULL DEFAULT '',"
+                      "  value BLOB NOT NULL DEFAULT '')",
+                      NULL,
+                      NULL,
+                      NULL)) )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_exec");
     sqlite3_finalize (stmt);
     return GNUNET_SYSERR;
   }
   sqlite3_finalize (stmt);
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "UPDATE gn090 "
-        "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
-        &plugin->updPrio) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
-        &plugin->updRepl) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+#define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "UPDATE gn090 "
+                    "SET prio = prio + ?, "
+                    "repl = repl + ?, "
+                    "expire = MAX(expire, ?) "
+                    "WHERE _ROWID_ = ?",
+                    &plugin->updPrio)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
+                    &plugin->updRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT " RESULT_COLUMNS " FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_repl_rvalue "
+                    "INDEXED BY idx_repl_rvalue "
 #endif
-        "WHERE repl=?2 AND " " (rvalue>=?1 OR "
-        "  NOT EXISTS (SELECT 1 FROM gn090 "
+                    "WHERE repl=?2 AND " " (rvalue>=?1 OR "
+                    "  NOT EXISTS (SELECT 1 FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_repl_rvalue "
+                    "INDEXED BY idx_repl_rvalue "
 #endif
-        "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
-        "ORDER BY rvalue ASC LIMIT 1", &plugin->selRepl) != SQLITE_OK) ||
-      (sq_prepare (plugin->dbh, "SELECT MAX(repl) FROM gn090"
+                    "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
+                    "ORDER BY rvalue ASC LIMIT 1",
+                    &plugin->selRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT MAX(repl) FROM gn090"
 #if SQLITE_VERSION_NUMBER >= 3007000
-                   " INDEXED BY idx_repl_rvalue"
+                    " INDEXED BY idx_repl_rvalue"
 #endif
-                   "", &plugin->maxRepl) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+                    "",
+                    &plugin->maxRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT " RESULT_COLUMNS " FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_expire "
+                    "INDEXED BY idx_expire "
 #endif
-        "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
-        "ORDER BY expire ASC LIMIT 1", &plugin->selExpi) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+                    "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
+                    "ORDER BY expire ASC LIMIT 1",
+                    &plugin->selExpi)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT " RESULT_COLUMNS " FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_anon_type_hash "
+                    "INDEXED BY idx_anon_type_hash "
 #endif
-        "WHERE (anonLevel = 0 AND type=?1) "
-        "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
-        &plugin->selZeroAnon) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
-        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
-        &plugin->insertContent) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?",
-        &plugin->delRow) != SQLITE_OK))
+                    "WHERE _ROWID_ >= ? AND "
+                    "anonLevel = 0 AND "
+                    "type = ? "
+                    "ORDER BY _ROWID_ ASC LIMIT 1",
+                    &plugin->selZeroAnon)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    &plugin->insertContent)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT " RESULT_COLUMNS " FROM gn090 "
+                    "WHERE _ROWID_ >= ? AND "
+                    "(rvalue >= ? OR 0 = ?) AND "
+                    "(hash = ? OR 0 = ?) AND "
+                    "(vhash = ? OR 0 = ?) AND "
+                    "(type = ? OR 0 = ?) "
+                    "ORDER BY _ROWID_ ASC LIMIT 1",
+                    &plugin->get)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM gn090 WHERE _ROWID_ = ?",
+                    &plugin->delRow))
+       )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
 }
 
@@ -379,51 +444,60 @@ static void
 database_shutdown (struct Plugin *plugin)
 {
   int result;
-
 #if SQLITE_VERSION_NUMBER >= 3007000
   sqlite3_stmt *stmt;
 #endif
 
-  if (plugin->delRow != NULL)
+  if (NULL != plugin->delRow)
     sqlite3_finalize (plugin->delRow);
-  if (plugin->updPrio != NULL)
+  if (NULL != plugin->updPrio)
     sqlite3_finalize (plugin->updPrio);
-  if (plugin->updRepl != NULL)
+  if (NULL != plugin->updRepl)
     sqlite3_finalize (plugin->updRepl);
-  if (plugin->selRepl != NULL)
+  if (NULL != plugin->selRepl)
     sqlite3_finalize (plugin->selRepl);
-  if (plugin->maxRepl != NULL)
+  if (NULL != plugin->maxRepl)
     sqlite3_finalize (plugin->maxRepl);
-  if (plugin->selExpi != NULL)
+  if (NULL != plugin->selExpi)
     sqlite3_finalize (plugin->selExpi);
-  if (plugin->selZeroAnon != NULL)
+  if (NULL != plugin->selZeroAnon)
     sqlite3_finalize (plugin->selZeroAnon);
-  if (plugin->insertContent != NULL)
+  if (NULL != plugin->insertContent)
     sqlite3_finalize (plugin->insertContent);
+  if (NULL != plugin->get)
+    sqlite3_finalize (plugin->get);
   result = sqlite3_close (plugin->dbh);
 #if SQLITE_VERSION_NUMBER >= 3007000
   if (result == SQLITE_BUSY)
   {
-    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                     _
-                     ("Tried to close sqlite without finalizing all prepared statements.\n"));
-    stmt = sqlite3_next_stmt (plugin->dbh, NULL);
-    while (stmt != NULL)
+    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                     "sqlite",
+                     _("Tried to close sqlite without finalizing all prepared statements.\n"));
+    stmt = sqlite3_next_stmt (plugin->dbh,
+                              NULL);
+    while (NULL != stmt)
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                       "Closing statement %p\n", stmt);
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Closing statement %p\n",
+                       stmt);
       result = sqlite3_finalize (stmt);
       if (result != SQLITE_OK)
-        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                         "Failed to close statement %p: %d\n", stmt, result);
-      stmt = sqlite3_next_stmt (plugin->dbh, NULL);
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sqlite",
+                         "Failed to close statement %p: %d\n",
+                         stmt,
+                         result);
+      stmt = sqlite3_next_stmt (plugin->dbh,
+                                NULL);
     }
     result = sqlite3_close (plugin->dbh);
   }
 #endif
   if (SQLITE_OK != result)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
-
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
   GNUNET_free_non_null (plugin->fn);
 }
 
@@ -437,31 +511,27 @@ database_shutdown (struct Plugin *plugin)
  */
 static int
 delete_by_rowid (struct Plugin *plugin,
-                 unsigned long long rid)
+                 uint64_t rid)
 {
-  if (SQLITE_OK != sqlite3_bind_int64 (plugin->delRow, 1, rid))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&rid),
+    GNUNET_SQ_query_param_end
+  };
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->delRow,
+                      params))
     return GNUNET_SYSERR;
-  }
   if (SQLITE_DONE != sqlite3_step (plugin->delRow))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delRow);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->delRow);
   return GNUNET_OK;
 }
 
@@ -494,12 +564,25 @@ sqlite_plugin_put (void *cls,
                    PluginPutCont cont,
                    void *cont_cls)
 {
+  uint64_t rvalue;
+  struct GNUNET_HashCode vhash;
+  uint32_t type32 = (uint32_t) type;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&replication),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_uint32 (&priority),
+    GNUNET_SQ_query_param_uint32 (&anonymity),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_auto_from_type (&vhash),
+    GNUNET_SQ_query_param_fixed_size (data, size),
+    GNUNET_SQ_query_param_end
+  };
   struct Plugin *plugin = cls;
   int n;
   int ret;
   sqlite3_stmt *stmt;
-  struct GNUNET_HashCode vhash;
-  uint64_t rvalue;
   char *msg = NULL;
 
   if (size > MAX_ITEM_SIZE)
@@ -518,26 +601,10 @@ sqlite_plugin_put (void *cls,
   GNUNET_CRYPTO_hash (data, size, &vhash);
   stmt = plugin->insertContent;
   rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value_us)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 7, key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 8, &vhash, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 9, data, size, SQLITE_TRANSIENT)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     cont (cont_cls, key, size, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
@@ -546,7 +613,9 @@ sqlite_plugin_put (void *cls,
   switch (n)
   {
   case SQLITE_DONE:
-    plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
+    if (NULL != plugin->env->duc)
+      plugin->env->duc (plugin->env->cls,
+                        size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
     GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
                      "Stored new entry (%u bytes)\n",
                      size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
@@ -561,77 +630,71 @@ sqlite_plugin_put (void *cls,
   default:
     LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                     "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     database_shutdown (plugin);
     database_setup (plugin->env->cfg, plugin);
     cont (cont_cls, key, size, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   cont (cont_cls, key, size, ret, msg);
   GNUNET_free_non_null(msg);
 }
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
- *
- * Note that it is possible for multiple values to match this put.
- * In that case, all of the respective values are updated.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls the plugin context (state for this module)
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
  * @param cont continuation called with success or failure status
- * @param cons_cls continuation closure
+ * @param cons_cls closure for @a cont
  */
 static void
 sqlite_plugin_update (void *cls,
                       uint64_t uid,
-                      int delta,
+                      uint32_t priority,
+                      uint32_t replication,
                       struct GNUNET_TIME_Absolute expire,
                       PluginUpdateCont cont,
                       void *cont_cls)
 {
   struct Plugin *plugin = cls;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&priority),
+    GNUNET_SQ_query_param_uint32 (&replication),
+    GNUNET_SQ_query_param_absolute_time (&expire),
+    GNUNET_SQ_query_param_uint64 (&uid),
+    GNUNET_SQ_query_param_end
+  };
   int n;
   char *msg = NULL;
 
-  if ((SQLITE_OK != sqlite3_bind_int (plugin->updPrio, 1, delta)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value_us))
-      || (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 3, uid)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->updPrio,
+                      params))
   {
-    LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     cont (cont_cls, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
   }
   n = sqlite3_step (plugin->updPrio);
-  if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->updPrio);
   switch (n)
   {
   case SQLITE_DONE:
@@ -672,73 +735,88 @@ execute_get (struct Plugin *plugin,
 {
   int n;
   struct GNUNET_TIME_Absolute expiration;
-  unsigned long long rowid;
-  unsigned int size;
+  uint32_t replication;
+  uint32_t type;
+  uint32_t priority;
+  uint32_t anonymity;
+  uint64_t rowid;
+  void *value;
+  size_t value_size;
+  struct GNUNET_HashCode key;
   int ret;
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_uint32 (&replication),
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_uint32 (&priority),
+    GNUNET_SQ_result_spec_uint32 (&anonymity),
+    GNUNET_SQ_result_spec_absolute_time (&expiration),
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_variable_size (&value,
+                                         &value_size),
+    GNUNET_SQ_result_spec_uint64 (&rowid),
+    GNUNET_SQ_result_spec_end
+  };
 
   n = sqlite3_step (stmt);
   switch (n)
   {
   case SQLITE_ROW:
-    size = sqlite3_column_bytes (stmt, 5);
-    rowid = sqlite3_column_int64 (stmt, 6);
-    if (sqlite3_column_bytes (stmt, 4) != sizeof (struct GNUNET_HashCode))
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (stmt,
+                                  rs))
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                       _("Invalid data in database.  Trying to fix (by deletion).\n"));
-      if (SQLITE_OK != sqlite3_reset (stmt))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
-      if (GNUNET_OK == delete_by_rowid (plugin, rowid))
-        plugin->env->duc (plugin->env->cls,
-                          -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
+      GNUNET_break (0);
       break;
     }
-    expiration.abs_value_us = sqlite3_column_int64 (stmt, 3);
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
                      "Found reply in database with expiration %s\n",
                      GNUNET_STRINGS_absolute_time_to_string (expiration));
-    ret = proc (proc_cls, sqlite3_column_blob (stmt, 4) /* key */ ,
-                size, sqlite3_column_blob (stmt, 5) /* data */ ,
-                sqlite3_column_int (stmt, 0) /* type */ ,
-                sqlite3_column_int (stmt, 1) /* priority */ ,
-                sqlite3_column_int (stmt, 2) /* anonymity */ ,
-                expiration, rowid);
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)))
+    ret = proc (proc_cls,
+                &key,
+                value_size,
+                value,
+                type,
+                priority,
+                anonymity,
+                replication,
+                expiration,
+                rowid);
+    GNUNET_SQ_cleanup_result (rs);
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
+    if ( (GNUNET_NO == ret) &&
+         (GNUNET_OK == delete_by_rowid (plugin,
+                                        rowid)) &&
+         (NULL != plugin->env->duc) )
       plugin->env->duc (plugin->env->cls,
-                        -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
+                        -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
     return;
   case SQLITE_DONE:
     /* database must be empty */
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     break;
   case SQLITE_BUSY:
   case SQLITE_ERROR:
   case SQLITE_MISUSE:
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (stmt))
+    if (SQLITE_OK !=
+        sqlite3_reset (stmt))
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_reset");
     GNUNET_break (0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     database_shutdown (plugin);
-    database_setup (plugin->env->cfg, plugin);
-    break;
+    database_setup (plugin->env->cfg,
+                    plugin);
+    return;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-  proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
+  proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
 }
 
 
@@ -747,37 +825,36 @@ execute_get (struct Plugin *plugin,
  * the given processor for the item.
  *
  * @param cls our plugin context
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on each matching value;
- *        will be called once with a NULL value at the end
+ *        Must not be zero (ANY).
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
-sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+sqlite_plugin_get_zero_anonymity (void *cls,
+                                  uint64_t next_uid,
                                   enum GNUNET_BLOCK_Type type,
-                                  PluginDatumProcessor proc, void *proc_cls)
+                                  PluginDatumProcessor proc,
+                                  void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&next_uid),
+    GNUNET_SQ_query_param_uint32 (&type),
+    GNUNET_SQ_query_param_end
+  };
 
   GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
-  stmt = plugin->selZeroAnon;
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->selZeroAnon,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  execute_get (plugin, stmt, proc, proc_cls);
+  execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
 }
 
 
@@ -785,8 +862,9 @@ sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
  * Get results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset (mod count).
- * @param key key to match, never NULL
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
+ * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
  *        Note that for DBlocks there is no difference
@@ -800,7 +878,8 @@ sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
  */
 static void
 sqlite_plugin_get_key (void *cls,
-                       uint64_t offset,
+                       uint64_t next_uid,
+                       bool random,
                        const struct GNUNET_HashCode *key,
                        const struct GNUNET_HashCode *vhash,
                        enum GNUNET_BLOCK_Type type,
@@ -808,97 +887,45 @@ sqlite_plugin_get_key (void *cls,
                        void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  int ret;
-  int total;
-  int limit_off;
-  unsigned int sqoff;
-  sqlite3_stmt *stmt;
-  char scratch[256];
-
-  GNUNET_assert (proc != NULL);
-  GNUNET_assert (key != NULL);
-  GNUNET_snprintf (scratch, sizeof (scratch),
-                   "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
-                   vhash == NULL ? "" : " AND vhash=?",
-                   type == 0 ? "" : " AND type=?");
-  if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_prepare");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  sqoff = 1;
-  ret =
-      sqlite3_bind_blob (stmt, sqoff++, key, sizeof (struct GNUNET_HashCode),
-                         SQLITE_TRANSIENT);
-  if ((vhash != NULL) && (ret == SQLITE_OK))
-    ret =
-        sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (struct GNUNET_HashCode),
-                           SQLITE_TRANSIENT);
-  if ((type != 0) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_int (stmt, sqoff++, type);
-  if (SQLITE_OK != ret)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
-    sqlite3_finalize (stmt);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  ret = sqlite3_step (stmt);
-  if (ret != SQLITE_ROW)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_step");
-    sqlite3_finalize (stmt);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  total = sqlite3_column_int (stmt, 0);
-  sqlite3_finalize (stmt);
-  if (0 == total)
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  limit_off = (int) (offset % total);
-  if (limit_off < 0)
-    limit_off += total;
-  GNUNET_snprintf (scratch, sizeof (scratch),
-                   "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
-                   "FROM gn090 WHERE hash=?%s%s "
-                   "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
-                   vhash == NULL ? "" : " AND vhash=?",
-                   type == 0 ? "" : " AND type=?");
-  if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
+  uint64_t rvalue;
+  uint16_t use_rvalue = random;
+  uint32_t type32 = (uint32_t) type;
+  uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
+  uint16_t use_key = NULL != key;
+  uint16_t use_vhash = NULL != vhash;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&next_uid),
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_uint16 (&use_rvalue),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint16 (&use_key),
+    GNUNET_SQ_query_param_auto_from_type (vhash),
+    GNUNET_SQ_query_param_uint16 (&use_vhash),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_uint16 (&use_type),
+    GNUNET_SQ_query_param_end
+  };
+
+  if (random)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_prepare");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
-  sqoff = 1;
-  ret = sqlite3_bind_blob (stmt, sqoff++, key,
-                           sizeof (struct GNUNET_HashCode),
-                           SQLITE_TRANSIENT);
-  if ((vhash != NULL) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_blob (stmt, sqoff++, vhash,
-                             sizeof (struct GNUNET_HashCode),
-                             SQLITE_TRANSIENT);
-  if ((type != 0) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_int (stmt, sqoff++, type);
-  if (ret == SQLITE_OK)
-    ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
-  if (ret != SQLITE_OK)
+  else
+    rvalue = 0;
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get,
+                      params))
   {
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_bind");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  execute_get (plugin, stmt, proc, proc_cls);
-  sqlite3_finalize (stmt);
+  execute_get (plugin,
+               plugin->get,
+               proc,
+               proc_cls);
 }
 
 
@@ -942,6 +969,7 @@ struct ReplCtx
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -956,19 +984,25 @@ repl_proc (void *cls,
            enum GNUNET_BLOCK_Type type,
            uint32_t priority,
            uint32_t anonymity,
+           uint32_t replication,
            struct GNUNET_TIME_Absolute expiration,
            uint64_t uid)
 {
   struct ReplCtx *rc = cls;
   int ret;
 
+  if (GNUNET_SYSERR == rc->have_uid)
+    rc->have_uid = GNUNET_NO;
   ret = rc->proc (rc->proc_cls,
                   key,
-                  size, data,
+                  size,
+                  data,
                   type,
                   priority,
                   anonymity,
-                  expiration, uid);
+                  replication,
+                  expiration,
+                  uid);
   if (NULL != key)
   {
     rc->uid = uid;
@@ -989,81 +1023,77 @@ repl_proc (void *cls,
  * @param proc_cls closure for @a proc
  */
 static void
-sqlite_plugin_get_replication (void *cls, PluginDatumProcessor proc,
+sqlite_plugin_get_replication (void *cls,
+                               PluginDatumProcessor proc,
                                void *proc_cls)
 {
   struct Plugin *plugin = cls;
   struct ReplCtx rc;
   uint64_t rvalue;
   uint32_t repl;
-  sqlite3_stmt *stmt;
-
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  struct GNUNET_SQ_QueryParam params_sel_repl[] = {
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_uint32 (&repl),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam params_upd_repl[] = {
+    GNUNET_SQ_query_param_uint64 (&rc.uid),
+    GNUNET_SQ_query_param_end
+  };
+
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "datastore-sqlite",
                    "Getting random block based on replication order.\n");
-  rc.have_uid = GNUNET_NO;
-  rc.proc = proc;
-  rc.proc_cls = proc_cls;
-  stmt = plugin->maxRepl;
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->maxRepl))
   {
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->maxRepl);
     /* DB empty */
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  repl = sqlite3_column_int (stmt, 0);
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-  stmt = plugin->selRepl;
-  rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
-  if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl))
+  repl = sqlite3_column_int (plugin->maxRepl,
+                             0);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->maxRepl);
+  rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                     UINT64_MAX);
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->selRepl,
+                      params_sel_repl))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  execute_get (plugin, stmt, &repl_proc, &rc);
+  rc.have_uid = GNUNET_SYSERR;
+  rc.proc = proc;
+  rc.proc_cls = proc_cls;
+  execute_get (plugin,
+               plugin->selRepl,
+               &repl_proc,
+               &rc);
   if (GNUNET_YES == rc.have_uid)
   {
-    if (SQLITE_OK != sqlite3_bind_int64 (plugin->updRepl, 1, rc.uid))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->updRepl,
+                        params_upd_repl))
     {
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_bind_XXXX");
-      if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
       return;
     }
-    if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
+    if (SQLITE_DONE !=
+        sqlite3_step (plugin->updRepl))
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->updRepl);
+  }
+  if (GNUNET_SYSERR == rc.have_uid)
+  {
+    /* proc was not called at all so far, do it now. */
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
   }
 }
 
@@ -1083,20 +1113,21 @@ sqlite_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
   struct GNUNET_TIME_Absolute now;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
 
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
                    "Getting random block based on expiration and priority order.\n");
   now = GNUNET_TIME_absolute_get ();
   stmt = plugin->selExpi;
-  if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
   execute_get (plugin, stmt, proc, proc_cls);
@@ -1116,30 +1147,47 @@ sqlite_plugin_get_keys (void *cls,
                         void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  const struct GNUNET_HashCode *key;
+  struct GNUNET_HashCode key;
+  struct GNUNET_SQ_ResultSpec results[] = {
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_end
+  };
   sqlite3_stmt *stmt;
   int ret;
 
-  GNUNET_assert (proc != NULL);
-  if (sq_prepare (plugin->dbh, "SELECT hash FROM gn090", &stmt) != SQLITE_OK)
+  GNUNET_assert (NULL != proc);
+  if (SQLITE_OK !=
+      sq_prepare (plugin->dbh,
+                  "SELECT hash FROM gn090",
+                  &stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite_prepare");
-    proc (proc_cls, NULL, 0);
+    proc (proc_cls,
+          NULL,
+          0);
     return;
   }
   while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
   {
-    key = sqlite3_column_blob (stmt, 0);
-    if (sizeof (struct GNUNET_HashCode) == sqlite3_column_bytes (stmt, 0))
-      proc (proc_cls, key, 1);
+    if (GNUNET_OK ==
+        GNUNET_SQ_extract_result (stmt,
+                                  results))
+      proc (proc_cls,
+            &key,
+            1);
     else
       GNUNET_break (0);
   }
   if (SQLITE_DONE != ret)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite_step");
   sqlite3_finalize (stmt);
-  proc (proc_cls, NULL, 0);
+  proc (proc_cls,
+        NULL,
+        0);
 }
 
 
@@ -1165,7 +1213,8 @@ sqlite_plugin_drop (void *cls)
  * @return the size of the database on disk (estimate)
  */
 static void
-sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
+sqlite_plugin_estimate_size (void *cls,
+                             unsigned long long *estimate)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
@@ -1180,30 +1229,45 @@ sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
     return;
   if (SQLITE_VERSION_NUMBER < 3006000)
   {
-    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "datastore-sqlite",
-                     _
-                     ("sqlite version to old to determine size, assuming zero\n"));
+    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                     "datastore-sqlite",
+                     _("sqlite version to old to determine size, assuming zero\n"));
     *estimate = 0;
     return;
   }
-  CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
+         sqlite3_exec (plugin->dbh,
+                       "VACUUM",
+                       NULL,
+                       NULL,
+                       ENULL));
+  CHECK (SQLITE_OK ==
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL",
+                       NULL,
                        NULL, ENULL));
-  CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt));
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_count",
+                     &stmt));
   if (SQLITE_ROW == sqlite3_step (stmt))
-    pages = sqlite3_column_int64 (stmt, 0);
+    pages = sqlite3_column_int64 (stmt,
+                                  0);
   else
     pages = 0;
   sqlite3_finalize (stmt);
-  CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt));
-  CHECK (SQLITE_ROW == sqlite3_step (stmt));
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_size",
+                     &stmt));
+  CHECK (SQLITE_ROW ==
+         sqlite3_step (stmt));
   page_size = sqlite3_column_int64 (stmt, 0);
   sqlite3_finalize (stmt);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              _
-              ("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
-              (unsigned long long) pages, (unsigned long long) page_size);
+              _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
+              (unsigned long long) pages,
+              (unsigned long long) page_size);
   *estimate = pages * page_size;
 }
 
@@ -1221,9 +1285,11 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
   struct GNUNET_DATASTORE_PluginFunctions *api;
 
-  if (plugin.env != NULL)
+  if (NULL != plugin.env)
     return NULL;                /* can only initialize once! */
-  memset (&plugin, 0, sizeof (struct Plugin));
+  memset (&plugin,
+          0,
+          sizeof (struct Plugin));
   plugin.env = env;
   if (GNUNET_OK != database_setup (env->cfg, &plugin))
   {
@@ -1241,7 +1307,8 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
   api->get_keys = &sqlite_plugin_get_keys;
   api->drop = &sqlite_plugin_drop;
-  GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
+                   "sqlite",
                    _("Sqlite database running\n"));
   return api;
 }
@@ -1260,24 +1327,23 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
   struct GNUNET_DATASTORE_PluginFunctions *api = cls;
   struct Plugin *plugin = api->cls;
 
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
                    "sqlite plugin is done\n");
   fn = NULL;
   if (plugin->drop_on_shutdown)
     fn = GNUNET_strdup (plugin->fn);
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "Shutting down database\n");
   database_shutdown (plugin);
   plugin->env = NULL;
   GNUNET_free (api);
-  if (fn != NULL)
+  if (NULL != fn)
   {
     if (0 != UNLINK (fn))
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
+                                fn);
     GNUNET_free (fn);
   }
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "sqlite plugin is finished\n");
   return NULL;
 }
 
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
index fdd4fb157..8e44f020d 100644
--- a/src/datastore/plugin_datastore_template.c
+++ b/src/datastore/plugin_datastore_template.c
@@ -89,8 +89,8 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -104,7 +104,7 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
  * @param proc_cls closure for proc
  */
 static void
-template_plugin_get_key (void *cls, uint64_t offset,
+template_plugin_get_key (void *cls, uint64_t next_uid, bool random,
                          const struct GNUNET_HashCode * key,
                          const struct GNUNET_HashCode * vhash,
                          enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc,
@@ -151,22 +151,18 @@ template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
- *
- * Note that it is possible for multiple values to match this put.
- * In that case, all of the respective values are updated.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls our "struct Plugin*"
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
@@ -174,9 +170,13 @@ template_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
  * @param cons_cls continuation closure
  */
 static void
-template_plugin_update (void *cls, uint64_t uid, int delta,
+template_plugin_update (void *cls,
+                        uint64_t uid,
+                        uint32_t priority,
+                        uint32_t replication,
                         struct GNUNET_TIME_Absolute expire,
-                        PluginUpdateCont cont, void *cont_cls)
+                        PluginUpdateCont cont,
+                        void *cont_cls)
 {
   GNUNET_break (0);
   cont (cont_cls, GNUNET_SYSERR, "not implemented");
@@ -187,16 +187,15 @@ template_plugin_update (void *cls, uint64_t uid, int delta,
  * Call the given processor on an item with zero anonymity.
  *
  * @param cls our "struct Plugin*"
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on each matching value;
- *        will be called  with NULL if no value matches
+ *        Must not be zero (ANY).
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for proc
  */
 static void
-template_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
                                     enum GNUNET_BLOCK_Type type,
                                     PluginDatumProcessor proc, void *proc_cls)
 {
diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c
index 9f89d4087..63927a364 100644
--- a/src/datastore/test_datastore_api.c
+++ b/src/datastore/test_datastore_api.c
@@ -130,8 +130,6 @@ enum RunPhase
   RP_PUT_MULTIPLE_NEXT = 8,
   RP_GET_MULTIPLE = 9,
   RP_GET_MULTIPLE_NEXT = 10,
-  RP_UPDATE = 11,
-  RP_UPDATE_VALIDATE = 12,
 
   /**
    * Execution failed with some kind of error.
@@ -158,8 +156,6 @@ struct CpsRunContext
   void *data;
   size_t size;
 
-  uint64_t uid;
-  uint64_t offset;
   uint64_t first_uid;
 };
 
@@ -235,6 +231,7 @@ check_value (void *cls,
              enum GNUNET_BLOCK_Type type,
              uint32_t priority,
              uint32_t anonymity,
+             uint32_t replication,
              struct GNUNET_TIME_Absolute expiration,
              uint64_t uid)
 {
@@ -269,7 +266,6 @@ check_value (void *cls,
   GNUNET_assert (priority == get_priority (i));
   GNUNET_assert (anonymity == get_anonymity (i));
   GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
-  crc->offset++;
   if (crc->i == 0)
   {
     crc->phase = RP_DEL;
@@ -288,6 +284,7 @@ delete_value (void *cls,
               enum GNUNET_BLOCK_Type type,
               uint32_t priority,
               uint32_t anonymity,
+              uint32_t replication,
               struct GNUNET_TIME_Absolute expiration,
               uint64_t uid)
 {
@@ -313,6 +310,7 @@ check_nothing (void *cls,
                enum GNUNET_BLOCK_Type type,
                uint32_t priority,
                uint32_t anonymity,
+               uint32_t replication,
                struct GNUNET_TIME_Absolute expiration,
                uint64_t uid)
 {
@@ -334,6 +332,7 @@ check_multiple (void *cls,
                 enum GNUNET_BLOCK_Type type,
                 uint32_t priority,
                 uint32_t anonymity,
+                uint32_t replication,
                 struct GNUNET_TIME_Absolute expiration,
                 uint64_t uid)
 {
@@ -345,45 +344,16 @@ check_multiple (void *cls,
   case RP_GET_MULTIPLE:
     crc->phase = RP_GET_MULTIPLE_NEXT;
     crc->first_uid = uid;
-    crc->offset++;
     break;
   case RP_GET_MULTIPLE_NEXT:
     GNUNET_assert (uid != crc->first_uid);
-    crc->phase = RP_UPDATE;
+    crc->phase = RP_DONE;
     break;
   default:
     GNUNET_break (0);
     crc->phase = RP_ERROR;
     break;
   }
-  if (priority == get_priority (42))
-    crc->uid = uid;
-  GNUNET_SCHEDULER_add_now (&run_continuation, crc);
-}
-
-
-static void
-check_update (void *cls,
-              const struct GNUNET_HashCode *key,
-              size_t size,
-              const void *data,
-              enum GNUNET_BLOCK_Type type,
-              uint32_t priority,
-              uint32_t anonymity,
-              struct GNUNET_TIME_Absolute expiration,
-              uint64_t uid)
-{
-  struct CpsRunContext *crc = cls;
-
-  GNUNET_assert (key != NULL);
-  if ((anonymity == get_anonymity (42)) && (size == get_size (42)) &&
-      (priority == get_priority (42) + 100))
-    crc->phase = RP_DONE;
-  else
-  {
-    GNUNET_assert (size == get_size (43));
-    crc->offset++;
-  }
   GNUNET_SCHEDULER_add_now (&run_continuation, crc);
 }
 
@@ -428,7 +398,8 @@ run_continuation (void *cls)
                         sizeof (int),
                         &crc->key);
     GNUNET_DATASTORE_get_key (datastore,
-                              crc->offset,
+                              0,
+                              false,
                               &crc->key,
                               get_type (crc->i),
                               1,
@@ -445,7 +416,8 @@ run_continuation (void *cls)
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             0,
+                                             false,
                                              &crc->key,
                                              get_type (crc->i),
                                              1,
@@ -478,9 +450,15 @@ run_continuation (void *cls)
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
     GNUNET_assert (NULL !=
-                   GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key,
-                                             get_type (crc->i), 1, 1,
-                                             &check_nothing, crc));
+                   GNUNET_DATASTORE_get_key (datastore,
+                                             0,
+                                             false,
+                                             &crc->key,
+                                             get_type (crc->i),
+                                             1,
+                                             1,
+                                             &check_nothing,
+                                             crc));
     break;
   case RP_RESERVE:
     crc->phase = RP_PUT_MULTIPLE;
@@ -511,37 +489,26 @@ run_continuation (void *cls)
   case RP_GET_MULTIPLE:
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
-                                             &crc->key,
-                                             get_type (42), 1, 1,
-                                             &check_multiple, crc));
-    break;
-  case RP_GET_MULTIPLE_NEXT:
-    GNUNET_assert (NULL !=
-                   GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             0,
+                                             false,
                                              &crc->key,
                                              get_type (42),
-                                             1, 1,
-                                             &check_multiple, crc));
-    break;
-  case RP_UPDATE:
-    GNUNET_assert (crc->uid > 0);
-    crc->phase = RP_UPDATE_VALIDATE;
-    GNUNET_DATASTORE_update (datastore,
-                             crc->uid, 100,
-                             get_expiration (42), 1,
-                             1,
-                             &check_success, crc);
+                                             1,
+                                             1,
+                                             &check_multiple,
+                                             crc));
     break;
-  case RP_UPDATE_VALIDATE:
+  case RP_GET_MULTIPLE_NEXT:
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             crc->first_uid + 1,
+                                             false,
                                              &crc->key,
                                              get_type (42),
-                                             1, 1,
-                                             &check_update, crc));
+                                             1,
+                                             1,
+                                             &check_multiple,
+                                             crc));
     break;
   case RP_DONE:
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c
index 5e536d6c5..e50b98909 100644
--- a/src/datastore/test_datastore_api_management.c
+++ b/src/datastore/test_datastore_api_management.c
@@ -58,7 +58,6 @@ struct CpsRunContext
   const struct GNUNET_CONFIGURATION_Handle *cfg;
   void *data;
   enum RunPhase phase;
-  uint64_t offset;
 };
 
 
@@ -138,9 +137,15 @@ check_success (void *cls, int success, struct GNUNET_TIME_Absolute min_expiratio
 
 
 static void
-check_value (void *cls, const struct GNUNET_HashCode * key, size_t size,
-             const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority,
-             uint32_t anonymity, struct GNUNET_TIME_Absolute expiration,
+check_value (void *cls,
+             const struct GNUNET_HashCode *key,
+             size_t size,
+             const void *data,
+             enum GNUNET_BLOCK_Type type,
+             uint32_t priority,
+             uint32_t anonymity,
+             uint32_t replication,
+             struct GNUNET_TIME_Absolute expiration,
              uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
@@ -159,7 +164,6 @@ check_value (void *cls, const struct GNUNET_HashCode * key, size_t size,
   GNUNET_assert (priority == get_priority (i));
   GNUNET_assert (anonymity == get_anonymity (i));
   GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
-  crc->offset++;
   crc->i--;
   if (crc->i == 0)
     crc->phase = RP_DONE;
@@ -168,9 +172,15 @@ check_value (void *cls, const struct GNUNET_HashCode * key, size_t size,
 
 
 static void
-check_nothing (void *cls, const struct GNUNET_HashCode * key, size_t size,
-               const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority,
-               uint32_t anonymity, struct GNUNET_TIME_Absolute expiration,
+check_nothing (void *cls,
+               const struct GNUNET_HashCode *key,
+               size_t size,
+               const void *data,
+               enum GNUNET_BLOCK_Type type,
+               uint32_t priority,
+               uint32_t anonymity,
+               uint32_t replication,
+               struct GNUNET_TIME_Absolute expiration,
                uint64_t uid)
 {
   struct CpsRunContext *crc = cls;
@@ -221,8 +231,13 @@ run_continuation (void *cls)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET",
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
-    GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key,
-                              get_type (crc->i), 1, 1,
+    GNUNET_DATASTORE_get_key (datastore,
+                              0,
+                              false,
+                              &crc->key,
+                              get_type (crc->i),
+                              1,
+                              1,
                               &check_value,
                               crc);
     break;
@@ -230,8 +245,13 @@ run_continuation (void *cls)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)",
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
-    GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key,
-                              get_type (crc->i), 1, 1,
+    GNUNET_DATASTORE_get_key (datastore,
+                              0,
+                              false,
+                              &crc->key,
+                              get_type (crc->i),
+                              1,
+                              1,
                               &check_nothing,
                               crc);
     break;
@@ -298,7 +318,21 @@ run (void *cls,
 
 
 /**
- * check if plugin is actually working 
+ * Function called when disk utilization changes, does nothing.
+ *
+ * @param cls closure
+ * @param delta change in utilization
+ */
+static void
+ignore_payload_cb (void *cls,
+                   int delta)
+{
+  /* do nothing */
+}
+
+
+/**
+ * check if plugin is actually working
  */
 static int
 test_plugin (const char *cfg_name)
@@ -307,7 +341,7 @@ test_plugin (const char *cfg_name)
   struct GNUNET_CONFIGURATION_Handle *cfg;
   struct GNUNET_DATASTORE_PluginFunctions *api;
   struct GNUNET_DATASTORE_PluginEnvironment env;
-  
+
   cfg = GNUNET_CONFIGURATION_create ();
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_load (cfg,
@@ -321,6 +355,7 @@ test_plugin (const char *cfg_name)
   }
   memset (&env, 0, sizeof (env));
   env.cfg = cfg;
+  env.duc = &ignore_payload_cb;
   GNUNET_snprintf (libname,
                    sizeof (libname),
                    "libgnunet_plugin_datastore_%s",
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
index 9b85d57da..1867d6755 100644
--- a/src/datastore/test_plugin_datastore.c
+++ b/src/datastore/test_plugin_datastore.c
@@ -64,7 +64,6 @@ struct CpsRunContext
   enum RunPhase phase;
   unsigned int cnt;
   unsigned int i;
-  uint64_t offset;
 };
 
 
@@ -194,6 +193,7 @@ iterate_one_shot (void *cls,
                   enum GNUNET_BLOCK_Type type,
                   uint32_t priority,
                   uint32_t anonymity,
+                  uint32_t replication,
                   struct GNUNET_TIME_Absolute expiration,
                   uint64_t uid)
 {
@@ -308,7 +308,8 @@ test (void *cls)
                 "Looking for %s\n",
                 GNUNET_h2s (&key));
     crc->api->get_key (crc->api->cls,
-                       crc->offset++,
+                       0,
+                       false,
                        &key,
                        NULL,
                        GNUNET_BLOCK_TYPE_ANY,
@@ -317,11 +318,12 @@ test (void *cls)
     break;
   case RP_UPDATE:
     crc->api->update (crc->api->cls,
-                      guid,
-                      1,
-                      GNUNET_TIME_UNIT_ZERO_ABS,
+                      guid,
+                      1,
+                      1,
+                      GNUNET_TIME_UNIT_ZERO_ABS,
                       &update_continuation,
-                      crc);
+                      crc);
     break;
 
   case RP_ITER_ZERO:
diff --git a/src/dht/.gitignore b/src/dht/.gitignore
index e7f3c2a86..25b1daf28 100644
--- a/src/dht/.gitignore
+++ b/src/dht/.gitignore
@@ -1,10 +1,8 @@
-gnunet-service-dht-xvine
 gnunet-dht-get
 gnunet-dht-monitor
 gnunet-dht-profiler
 gnunet-dht-put
 gnunet-service-dht
-gnunet-service-dht-whanau
 test_dht_2dtorus
 test_dht_api
 test_dht_line
diff --git a/src/dht/Makefile.am b/src/dht/Makefile.am
index 1a174165c..f712890ac 100644
--- a/src/dht/Makefile.am
+++ b/src/dht/Makefile.am
@@ -40,6 +40,7 @@ libgnunet_plugin_block_dht_la_SOURCES = \
 libgnunet_plugin_block_dht_la_LIBADD = \
   $(top_builddir)/src/hello/libgnunethello.la \
   $(top_builddir)/src/block/libgnunetblock.la \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(LTLIBINTL)
 libgnunet_plugin_block_dht_la_LDFLAGS = \
@@ -49,12 +50,6 @@ libgnunet_plugin_block_dht_la_LDFLAGS = \
 libexec_PROGRAMS = \
  gnunet-service-dht
 
-if HAVE_EXPERIMENTAL
-libexec_PROGRAMS += \
- gnunet-service-dht-xvine \
- gnunet-service-dht-whanau
-endif
-
 noinst_PROGRAMS = \
  gnunet-dht-monitor \
  gnunet-dht-get \
@@ -85,44 +80,6 @@ gnunet_service_dht_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   -lm
 
-gnunet_service_dht_xvine_SOURCES = \
- gnunet-service-xdht.c gnunet-service-dht.h \
- gnunet-service-dht_datacache.c gnunet-service-dht_datacache.h \
- gnunet-service-dht_nse.c gnunet-service-dht_nse.h \
- gnunet-service-xdht_neighbours.c gnunet-service-xdht_neighbours.h \
- gnunet-service-xdht_routing.c gnunet-service-xdht_routing.h
-
-gnunet_service_dht_xvine_LDADD = \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
-  $(top_builddir)/src/nse/libgnunetnse.la \
-  $(top_builddir)/src/ats/libgnunetats.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
-  $(top_builddir)/src/hello/libgnunethello.la \
-  $(top_builddir)/src/block/libgnunetblock.la \
-  $(top_builddir)/src/datacache/libgnunetdatacache.la \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -lm
-
-gnunet_service_dht_whanau_SOURCES = \
- gnunet-service-wdht.c gnunet-service-wdht.h \
- gnunet-service-dht_datacache.c gnunet-service-dht_datacache.h \
- gnunet-service-dht_nse.c gnunet-service-dht_nse.h \
- gnunet-service-wdht_neighbours.c gnunet-service-dht_neighbours.h
-gnunet_service_dht_whanau_LDADD = \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
-  $(top_builddir)/src/nse/libgnunetnse.la \
-  $(top_builddir)/src/ats/libgnunetats.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
-  $(top_builddir)/src/hello/libgnunethello.la \
-  $(top_builddir)/src/block/libgnunetblock.la \
-  $(top_builddir)/src/datacache/libgnunetdatacache.la \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  -lm
-
 gnunet_dht_get_SOURCES = \
  gnunet-dht-get.c
 gnunet_dht_get_LDADD = \
@@ -180,7 +137,7 @@ if HAVE_EXPERIMENTAL
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = test_dht_api $(check_SCRIPTS) \
  test_dht_twopeer \
  test_dht_line \
@@ -254,7 +211,7 @@ check_SCRIPTS = \
   test_dht_tools.py
 endif
 
-do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g'
+do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -e 's,[@]bindir[@],$(bindir),g'
 
 %.py: %.py.in Makefile
         $(do_subst) < $(srcdir)/$< > $@
diff --git a/src/dht/dht_api.c b/src/dht/dht_api.c
index 66eaf1064..42ddc7b60 100644
--- a/src/dht/dht_api.c
+++ b/src/dht/dht_api.c
@@ -319,7 +319,7 @@ send_get_known_results (struct GNUNET_DHT_GetHandle *gh,
   unsigned int max;
   unsigned int transmission_offset;
 
-  max = (GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*msg))
+  max = (GNUNET_MAX_MESSAGE_SIZE - sizeof (*msg))
     / sizeof (struct GNUNET_HashCode);
   transmission_offset = transmission_offset_start;
   while (transmission_offset < gh->seen_results_end)
@@ -503,7 +503,7 @@ check_monitor_get (void *cls,
   uint16_t msize = ntohs (msg->header.size) - sizeof (*msg);
 
   if ( (plen > UINT16_MAX) ||
-       (plen * sizeof (struct GNUNET_HashCode) != msize) )
+       (plen * sizeof (struct GNUNET_PeerIdentity) != msize) )
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
@@ -704,9 +704,9 @@ check_client_result (void *cls,
     sizeof (struct GNUNET_PeerIdentity) * (get_path_length + put_path_length);
   if ( (msize < meta_length) ||
        (get_path_length >
-        GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+        GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
        (put_path_length >
-        GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) )
+        GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) )
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
@@ -754,12 +754,25 @@ process_client_result (void *cls,
   meta_length =
       sizeof (struct GNUNET_PeerIdentity) * (get_path_length + put_path_length);
   data_length = msize - meta_length;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Giving %u byte reply for %s to application\n",
-       (unsigned int) data_length,
-       GNUNET_h2s (key));
   put_path = (const struct GNUNET_PeerIdentity *) &crm[1];
   get_path = &put_path[put_path_length];
+  {
+    char *pp;
+    char *gp;
+
+    gp = GNUNET_STRINGS_pp2s (get_path,
+                              get_path_length);
+    pp = GNUNET_STRINGS_pp2s (put_path,
+                              put_path_length);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Giving %u byte reply for %s to application (GP: %s, PP: %s)\n",
+         (unsigned int) data_length,
+         GNUNET_h2s (key),
+         gp,
+         pp);
+    GNUNET_free (gp);
+    GNUNET_free (pp);
+  }
   data = &get_path[get_path_length];
   /* remember that we've seen this result */
   GNUNET_CRYPTO_hash (data,
@@ -985,8 +998,8 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle,
   struct GNUNET_DHT_PutHandle *ph;
 
   msize = sizeof (struct GNUNET_DHT_ClientPutMessage) + size;
-  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  if ((msize >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (size >= GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return NULL;
@@ -1077,8 +1090,8 @@ GNUNET_DHT_get_start (struct GNUNET_DHT_Handle *handle,
   size_t msize;
 
   msize = sizeof (struct GNUNET_DHT_ClientGetMessage) + xquery_size;
-  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (xquery_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  if ((msize >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (xquery_size >= GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/dht/gnunet-dht-get.c b/src/dht/gnunet-dht-get.c
index 8a1b836f6..e361df336 100644
--- a/src/dht/gnunet-dht-get.c
+++ b/src/dht/gnunet-dht-get.c
@@ -50,7 +50,7 @@ static struct GNUNET_TIME_Relative timeout_request = { 60000 };
 /**
  * Be verbose
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Use DHT demultixplex_everywhere
@@ -148,13 +148,36 @@ get_result_iterator (void *cls, struct GNUNET_TIME_Absolute exp,
                      const struct GNUNET_PeerIdentity *get_path,
                      unsigned int get_path_length,
                      const struct GNUNET_PeerIdentity *put_path,
-                     unsigned int put_path_length, enum GNUNET_BLOCK_Type type,
-                     size_t size, const void *data)
+                     unsigned int put_path_length,
+                     enum GNUNET_BLOCK_Type type,
+                     size_t size,
+                     const void *data)
 {
   FPRINTF (stdout,
            _("Result %d, type %d:\n%.*s\n"),
-           result_count, type,
-           (unsigned int) size, (char *) data);
+           result_count,
+           type,
+           (unsigned int) size,
+           (char *) data);
+  if (verbose)
+  {
+    FPRINTF (stdout,
+             "  GET path: ");
+    for (unsigned int i=0;i<get_path_length;i++)
+      FPRINTF (stdout,
+               "%s%s",
+               (0 == i) ? "" : "-",
+               GNUNET_i2s (&get_path[i]));
+    FPRINTF (stdout,
+             "\n  PUT path: ");
+    for (unsigned int i=0;i<put_path_length;i++)
+      FPRINTF (stdout,
+               "%s%s",
+               (0 == i) ? "" : "-",
+               GNUNET_i2s (&put_path[i]));
+    FPRINTF (stdout,
+             "\n");
+  }
   result_count++;
 }
 
@@ -203,33 +226,6 @@ run (void *cls, char *const *args, const char *cfgfile,
 
 }
 
-
-/**
- * gnunet-dht-get command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'r', "replication", "LEVEL",
-   gettext_noop ("how many parallel requests (replicas) to create"),
-   1, &GNUNET_GETOPT_set_uint, &replication},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type of data to look for"),
-   1, &GNUNET_GETOPT_set_uint, &query_type},
-  {'T', "timeout", "TIMEOUT",
-   gettext_noop ("how long to execute this query before giving up?"),
-   1, &GNUNET_GETOPT_set_relative_time, &timeout_request},
-  {'x', "demultiplex", NULL,
-    gettext_noop ("use DHT's demultiplex everywhere option"),
-    0, &GNUNET_GETOPT_set_one, &demultixplex_everywhere},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-get
  *
@@ -240,6 +236,45 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_option_string ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_option_uint ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("how many parallel requests (replicas) to create"),
+                                   &replication),
+  
+  
+    GNUNET_GETOPT_option_uint ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type of data to look for"),
+                                   &query_type),
+  
+    GNUNET_GETOPT_option_relative_time ('T',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("how long to execute this query before giving up?"),
+                                            &timeout_request),
+  
+    GNUNET_GETOPT_option_flag ('x',
+                                  "demultiplex",
+                                  gettext_noop ("use DHT's demultiplex everywhere option"),
+                                  &demultixplex_everywhere),
+  
+    GNUNET_GETOPT_option_verbose (&verbose),
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
     return 2;
   return (GNUNET_OK ==
diff --git a/src/dht/gnunet-dht-monitor.c b/src/dht/gnunet-dht-monitor.c
index 7f14255d3..b7360bbab 100644
--- a/src/dht/gnunet-dht-monitor.c
+++ b/src/dht/gnunet-dht-monitor.c
@@ -280,27 +280,6 @@ run (void *cls, char *const *args, const char *cfgfile,
                                              NULL);
 }
 
-
-/**
- * gnunet-dht-monitor command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type of data to look for"),
-   1, &GNUNET_GETOPT_set_uint, &block_type},
-  {'T', "timeout", "TIMEOUT",
-   gettext_noop ("how long should the monitor command run"),
-   1, &GNUNET_GETOPT_set_relative_time, &timeout_request},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-monitor
  *
@@ -311,6 +290,35 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_option_string ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_option_uint ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type of data to look for"),
+                                   &block_type),
+  
+    GNUNET_GETOPT_option_relative_time ('T',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("how long should the monitor command run"),
+                                            &timeout_request),
+  
+    GNUNET_GETOPT_option_flag ('V',
+                                  "verbose",
+                                  gettext_noop ("be verbose (print progress information)"),
+                                  &verbose),
+  
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
     return 2;
 
diff --git a/src/dht/gnunet-dht-put.c b/src/dht/gnunet-dht-put.c
index 1f3df1d35..f183fe588 100644
--- a/src/dht/gnunet-dht-put.c
+++ b/src/dht/gnunet-dht-put.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 GNUnet e.V.
+     Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -44,7 +44,7 @@ static char *query_key;
 /**
  * User supplied expiration value
  */
-static unsigned long long expiration_seconds = 3600;
+static struct GNUNET_TIME_Relative expiration;
 
 /**
  * Desired replication level.
@@ -54,14 +54,19 @@ static unsigned int replication = 5;
 /**
  * Be verbose
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
- * Use DHT demultixplex_everywhere
+ * Use #GNUNET_DHT_DEMULTIPLEX_EVERYWHERE.
  */
 static int demultixplex_everywhere;
 
 /**
+ * Use #GNUNET_DHT_RO_RECORD_ROUTE.
+ */
+static int record_route;
+
+/**
  * Handle to the DHT
  */
 static struct GNUNET_DHT_Handle *dht_handle;
@@ -143,7 +148,7 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  struct GNUNET_TIME_Absolute expiration;
+  enum GNUNET_DHT_RouteOption ro;
 
   cfg = c;
   if ((NULL == query_key) || (NULL == data))
@@ -164,55 +169,28 @@ run (void *cls,
 
   GNUNET_CRYPTO_hash (query_key, strlen (query_key), &key);
 
-  expiration =
-      GNUNET_TIME_relative_to_absolute (GNUNET_TIME_relative_multiply
-                                        (GNUNET_TIME_UNIT_SECONDS,
-                                         expiration_seconds));
   if (verbose)
-    FPRINTF (stderr, _("Issuing put request for `%s' with data `%s'!\n"),
-             query_key, data);
+    FPRINTF (stderr,
+             _("Issuing put request for `%s' with data `%s'!\n"),
+             query_key,
+             data);
+  ro = GNUNET_DHT_RO_NONE;
+  if (demultixplex_everywhere)
+    ro |= GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE;
+  if (record_route)
+    ro |= GNUNET_DHT_RO_RECORD_ROUTE;
   GNUNET_DHT_put (dht_handle,
                   &key,
                   replication,
-                  (demultixplex_everywhere) ? GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE : GNUNET_DHT_RO_NONE,
+                  ro,
                   query_type,
                   strlen (data),
                   data,
-                  expiration,
+                  GNUNET_TIME_relative_to_absolute (expiration),
                   &message_sent_cont,
                   NULL);
 }
 
-
-/**
- * gnunet-dht-put command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'d', "data", "DATA",
-   gettext_noop ("the data to insert under the key"),
-   1, &GNUNET_GETOPT_set_string, &data},
-  {'e', "expiration", "EXPIRATION",
-   gettext_noop ("how long to store this entry in the dht (in seconds)"),
-   1, &GNUNET_GETOPT_set_ulong, &expiration_seconds},
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'x', "demultiplex", NULL,
-   gettext_noop ("use DHT's demultiplex everywhere option"),
-   0, &GNUNET_GETOPT_set_one, &demultixplex_everywhere},
-  {'r', "replication", "LEVEL",
-   gettext_noop ("how many replicas to create"),
-   1, &GNUNET_GETOPT_set_uint, &replication},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type to insert data as"),
-   1, &GNUNET_GETOPT_set_uint, &query_type},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-put
  *
@@ -223,14 +201,69 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_option_string ('d',
+                                 "data",
+                                 "DATA",
+                                 gettext_noop ("the data to insert under the key"),
+                                 &data),
+  
+    GNUNET_GETOPT_option_relative_time ('e',
+                                            "expiration",
+                                            "EXPIRATION",
+                                            gettext_noop ("how long to store this entry in the dht (in seconds)"),
+                                            &expiration),
+  
+    GNUNET_GETOPT_option_string ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_option_flag ('x',
+                                  "demultiplex",
+                                  gettext_noop ("use DHT's demultiplex everywhere option"),
+                                  &demultixplex_everywhere),
+  
+    GNUNET_GETOPT_option_uint ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("how many replicas to create"),
+                                   &replication),
+  
+    GNUNET_GETOPT_option_flag ('R',
+                                  "record",
+                                  gettext_noop ("use DHT's record route option"),
+                                  &record_route),
+  
+    GNUNET_GETOPT_option_uint ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type to insert data as"),
+                                   &query_type),
+  
+    GNUNET_GETOPT_option_verbose (&verbose),
+  
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv,
                                                  &argc, &argv))
     return 2;
+  expiration = GNUNET_TIME_UNIT_HOURS;
   return (GNUNET_OK ==
-          GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-put",
+          GNUNET_PROGRAM_run (argc,
+                              argv,
+                              "gnunet-dht-put",
                               gettext_noop
                               ("Issue a PUT request to the GNUnet DHT insert DATA under KEY."),
-                              options, &run, NULL)) ? ret : 1;
+                              options,
+                              &run,
+                              NULL))
+    ? ret : 1;
 }
 
 /* end of gnunet-dht-put.c */
diff --git a/src/dht/gnunet-service-dht_clients.c b/src/dht/gnunet-service-dht_clients.c
index 5ba4e5820..cb155c484 100644
--- a/src/dht/gnunet-service-dht_clients.c
+++ b/src/dht/gnunet-service-dht_clients.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011, 2016 GNUnet e.V.
+     Copyright (C) 2009, 2010, 2011, 2016, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -362,21 +362,25 @@ client_disconnect_cb (void *cls,
 static void
 transmit_request (struct ClientQueryRecord *cqr)
 {
-  int32_t reply_bf_mutator;
-  struct GNUNET_CONTAINER_BloomFilter *reply_bf;
+  struct GNUNET_BLOCK_Group *bg;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
 
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# GET requests from clients injected"),
                             1,
                             GNUNET_NO);
-  reply_bf_mutator =
-      (int32_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                          UINT32_MAX);
-  reply_bf
-    = GNUNET_BLOCK_construct_bloomfilter (reply_bf_mutator,
-                                          cqr->seen_replies,
-                                          cqr->seen_replies_count);
+  bg = GNUNET_BLOCK_group_create (GDS_block_context,
+                                  cqr->type,
+                                  GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                                            UINT32_MAX),
+                                  NULL,
+                                  0,
+                                  "seen-set-size",
+                                  cqr->seen_replies_count,
+                                  NULL);
+  GNUNET_BLOCK_group_set_seen (bg,
+                               cqr->seen_replies,
+                               cqr->seen_replies_count);
   peer_bf
     = GNUNET_CONTAINER_bloomfilter_init (NULL,
                                          DHT_BLOOM_SIZE,
@@ -393,10 +397,9 @@ transmit_request (struct ClientQueryRecord *cqr)
                              &cqr->key,
                              cqr->xquery,
                              cqr->xquery_size,
-                             reply_bf,
-                             reply_bf_mutator,
+                             bg,
                              peer_bf);
-  GNUNET_CONTAINER_bloomfilter_free (reply_bf);
+  GNUNET_BLOCK_group_destroy (bg);
   GNUNET_CONTAINER_bloomfilter_free (peer_bf);
 
   /* exponential back-off for retries.
@@ -668,7 +671,6 @@ handle_dht_local_get (void *cls,
                             cqr->xquery,
                             xquery_size,
                             NULL,
-                            0,
                             &handle_local_result,
                             ch);
   GNUNET_SERVICE_client_continue (ch->client);
@@ -1052,10 +1054,9 @@ forward_reply (void *cls,
   eval
     = GNUNET_BLOCK_evaluate (GDS_block_context,
                              record->type,
+                             NULL,
                              GNUNET_BLOCK_EO_NONE,
                              key,
-                             NULL,
-                             0,
                              record->xquery,
                              record->xquery_size,
                              frc->data,
@@ -1165,7 +1166,7 @@ GDS_CLIENTS_handle_reply (struct GNUNET_TIME_Absolute expiration,
 
   msize = sizeof (struct GNUNET_DHT_ClientResultMessage) + data_size +
     (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/dht/gnunet-service-dht_datacache.c b/src/dht/gnunet-service-dht_datacache.c
index 9b4dace67..36047d561 100644
--- a/src/dht/gnunet-service-dht_datacache.c
+++ b/src/dht/gnunet-service-dht_datacache.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011, 2015 GNUnet e.V.
+     Copyright (C) 2009, 2010, 2011, 2015, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -72,7 +72,7 @@ GDS_DATACACHE_handle_put (struct GNUNET_TIME_Absolute expiration,
                 _("%s request received, but have no datacache!\n"), "PUT");
     return;
   }
-  if (data_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (data_size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -109,39 +109,34 @@ struct GetRequestContext
   const void *xquery;
 
   /**
-   * Bloomfilter to filter out duplicate replies (updated)
-   */
-  struct GNUNET_CONTAINER_BloomFilter **reply_bf;
-
-  /**
    * The key this request was about
    */
   struct GNUNET_HashCode key;
 
   /**
-   * Number of bytes in xquery.
+   * Block group to use to evaluate replies (updated)
    */
-  size_t xquery_size;
+  struct GNUNET_BLOCK_Group *bg;
 
   /**
-   * Mutator value for the @e reply_bf, see gnunet_block_lib.h
+   * Function to call on results.
    */
-  uint32_t reply_bf_mutator;
+  GDS_DATACACHE_GetCallback gc;
 
   /**
-   * Return value to give back.
+   * Closure for @e gc.
    */
-  enum GNUNET_BLOCK_EvaluationResult eval;
+  void *gc_cls;
 
   /**
-   * Function to call on results.
+   * Number of bytes in xquery.
    */
-  GDS_DATACACHE_GetCallback gc;
+  size_t xquery_size;
 
   /**
-   * Closure for @e gc.
+   * Return value to give back.
    */
-  void *gc_cls;
+  enum GNUNET_BLOCK_EvaluationResult eval;
 
 };
 
@@ -170,16 +165,20 @@ datacache_get_iterator (void *cls,
                         unsigned int put_path_length,
                         const struct GNUNET_PeerIdentity *put_path)
 {
+  static char non_null;
   struct GetRequestContext *ctx = cls;
   enum GNUNET_BLOCK_EvaluationResult eval;
 
+  if ( (NULL == data) &&
+       (0 == data_size) )
+    data = &non_null; /* point anywhere, but not to NULL */
+
   eval
     = GNUNET_BLOCK_evaluate (GDS_block_context,
                              type,
+                             ctx->bg,
                              GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO,
                              key,
-                             ctx->reply_bf,
-                             ctx->reply_bf_mutator,
                              ctx->xquery,
                              ctx->xquery_size,
                              data,
@@ -251,8 +250,7 @@ datacache_get_iterator (void *cls,
  * @param type requested data type
  * @param xquery extended query
  * @param xquery_size number of bytes in @a xquery
- * @param reply_bf where the reply bf is (to be) stored, possibly updated, can be NULL
- * @param reply_bf_mutator mutation value for @a reply_bf
+ * @param bg block group to use for reply evaluation
  * @param gc function to call on the results
  * @param gc_cls closure for @a gc
  * @return evaluation result for the local replies
@@ -262,8 +260,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           const void *xquery,
                           size_t xquery_size,
-                          struct GNUNET_CONTAINER_BloomFilter **reply_bf,
-                          uint32_t reply_bf_mutator,
+                          struct GNUNET_BLOCK_Group *bg,
                           GDS_DATACACHE_GetCallback gc,
                           void *gc_cls)
 {
@@ -280,8 +277,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
   ctx.key = *key;
   ctx.xquery = xquery;
   ctx.xquery_size = xquery_size;
-  ctx.reply_bf = reply_bf;
-  ctx.reply_bf_mutator = reply_bf_mutator;
+  ctx.bg = bg;
   ctx.gc = gc;
   ctx.gc_cls = gc_cls;
   r = GNUNET_DATACACHE_get (datacache,
diff --git a/src/dht/gnunet-service-dht_datacache.h b/src/dht/gnunet-service-dht_datacache.h
index 5069883c7..ff6ae23da 100644
--- a/src/dht/gnunet-service-dht_datacache.h
+++ b/src/dht/gnunet-service-dht_datacache.h
@@ -87,8 +87,7 @@ typedef void
  * @param type requested data type
  * @param xquery extended query
  * @param xquery_size number of bytes in xquery
- * @param reply_bf where the reply bf is (to be) stored, possibly updated!, can be NULL
- * @param reply_bf_mutator mutation value for reply_bf
+ * @param bg block group to use for evaluation of replies
  * @param gc function to call on the results
  * @param gc_cls closure for @a gc
  * @return evaluation result for the local replies
@@ -98,8 +97,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           const void *xquery,
                           size_t xquery_size,
-                          struct GNUNET_CONTAINER_BloomFilter **reply_bf,
-                          uint32_t reply_bf_mutator,
+                          struct GNUNET_BLOCK_Group *bg,
                           GDS_DATACACHE_GetCallback gc,
                           void *gc_cls);
 
diff --git a/src/dht/gnunet-service-dht_neighbours.c b/src/dht/gnunet-service-dht_neighbours.c
index 20bdc0ce4..0309bea88 100644
--- a/src/dht/gnunet-service-dht_neighbours.c
+++ b/src/dht/gnunet-service-dht_neighbours.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2016 GNUnet e.V.
+     Copyright (C) 2009-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -49,6 +49,11 @@
 #define LOG_TRAFFIC(kind,...) GNUNET_log_from (kind, "dht-traffic",__VA_ARGS__)
 
 /**
+ * Enable slow sanity checks to debug issues.
+ */
+#define SANITY_CHECKS 1
+
+/**
  * How many buckets will we allow total.
  */
 #define MAX_BUCKETS sizeof (struct GNUNET_HashCode) * 8
@@ -597,27 +602,10 @@ update_connect_preferences ()
 
 
 /**
- * Closure for #add_known_to_bloom().
- */
-struct BloomConstructorContext
-{
-  /**
-   * Bloom filter under construction.
-   */
-  struct GNUNET_CONTAINER_BloomFilter *bloom;
-
-  /**
-   * Mutator to use.
-   */
-  uint32_t bf_mutator;
-};
-
-
-/**
  * Add each of the peers we already know to the bloom filter of
  * the request so that we don't get duplicate HELLOs.
  *
- * @param cls the 'struct BloomConstructorContext'.
+ * @param cls the `struct GNUNET_BLOCK_Group`
  * @param key peer identity to add to the bloom filter
  * @param value value the peer information (unused)
  * @return #GNUNET_YES (we should continue to iterate)
@@ -627,22 +615,18 @@ add_known_to_bloom (void *cls,
                     const struct GNUNET_PeerIdentity *key,
                     void *value)
 {
-  struct BloomConstructorContext *ctx = cls;
+  struct GNUNET_BLOCK_Group *bg = cls;
   struct GNUNET_HashCode key_hash;
-  struct GNUNET_HashCode mh;
 
   GNUNET_CRYPTO_hash (key,
                       sizeof (struct GNUNET_PeerIdentity),
                       &key_hash);
-  GNUNET_BLOCK_mingle_hash (&key_hash,
-                            ctx->bf_mutator,
-                            &mh);
+  GNUNET_BLOCK_group_set_seen (bg,
+                               &key_hash,
+                               1);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Adding known peer (%s) to bloomfilter for FIND PEER with mutation %u\n",
-              GNUNET_i2s (key),
-              ctx->bf_mutator);
-  GNUNET_CONTAINER_bloomfilter_add (ctx->bloom,
-                                    &mh);
+              "Adding known peer (%s) to bloomfilter for FIND PEER\n",
+              GNUNET_i2s (key));
   return GNUNET_YES;
 }
 
@@ -658,7 +642,7 @@ static void
 send_find_peer_message (void *cls)
 {
   struct GNUNET_TIME_Relative next_send_time;
-  struct BloomConstructorContext bcc;
+  struct GNUNET_BLOCK_Group *bg;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
 
   find_peer_task = NULL;
@@ -672,30 +656,38 @@ send_find_peer_message (void *cls)
     newly_found_peers = 0;
     return;
   }
-  bcc.bf_mutator =
-      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                UINT32_MAX);
-  bcc.bloom =
-      GNUNET_CONTAINER_bloomfilter_init (NULL, DHT_BLOOM_SIZE,
-                                         GNUNET_CONSTANTS_BLOOMFILTER_K);
+  bg = GNUNET_BLOCK_group_create (GDS_block_context,
+                                  GNUNET_BLOCK_TYPE_DHT_HELLO,
+                                  GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                                            UINT32_MAX),
+                                  NULL,
+                                  0,
+                                  "filter-size",
+                                  DHT_BLOOM_SIZE,
+                                  NULL);
   GNUNET_CONTAINER_multipeermap_iterate (all_connected_peers,
                                          &add_known_to_bloom,
-                                         &bcc);
+                                         bg);
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# FIND PEER messages initiated"),
                             1,
                             GNUNET_NO);
-  peer_bf =
-      GNUNET_CONTAINER_bloomfilter_init (NULL, DHT_BLOOM_SIZE,
+  peer_bf
+    = GNUNET_CONTAINER_bloomfilter_init (NULL,
+                                         DHT_BLOOM_SIZE,
                                          GNUNET_CONSTANTS_BLOOMFILTER_K);
   // FIXME: pass priority!?
   GDS_NEIGHBOURS_handle_get (GNUNET_BLOCK_TYPE_DHT_HELLO,
-                             GNUNET_DHT_RO_FIND_PEER,
-                             FIND_PEER_REPLICATION_LEVEL, 0,
-                             &my_identity_hash, NULL, 0, bcc.bloom,
-                             bcc.bf_mutator, peer_bf);
+                             GNUNET_DHT_RO_FIND_PEER | GNUNET_DHT_RO_RECORD_ROUTE,
+                             FIND_PEER_REPLICATION_LEVEL,
+                             0,
+                             &my_identity_hash,
+                             NULL,
+                             0,
+                             bg,
+                             peer_bf);
   GNUNET_CONTAINER_bloomfilter_free (peer_bf);
-  GNUNET_CONTAINER_bloomfilter_free (bcc.bloom);
+  GNUNET_BLOCK_group_destroy (bg);
   /* schedule next round */
   next_send_time.rel_value_us =
       DHT_MINIMUM_FIND_PEER_INTERVAL.rel_value_us +
@@ -1031,10 +1023,10 @@ select_peer (const struct GNUNET_HashCode *key,
       count = 0;
       while ((pos != NULL) && (count < bucket_size))
       {
-        if ((bloom == NULL) ||
-            (GNUNET_NO ==
-             GNUNET_CONTAINER_bloomfilter_test (bloom,
-                                                &pos->phash)))
+        if ( (NULL == bloom) ||
+             (GNUNET_NO ==
+              GNUNET_CONTAINER_bloomfilter_test (bloom,
+                                                 &pos->phash)))
         {
           dist = get_distance (key,
                                &pos->phash);
@@ -1068,8 +1060,14 @@ select_peer (const struct GNUNET_HashCode *key,
     }
     if (NULL == chosen)
       GNUNET_STATISTICS_update (GDS_stats,
-                                gettext_noop ("# Peer selection failed"), 1,
+                                gettext_noop ("# Peer selection failed"),
+                                1,
                                 GNUNET_NO);
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Selected peer `%s' in greedy routing for %s\n",
+                  GNUNET_i2s (chosen->id),
+                  GNUNET_h2s (key));
     return chosen;
   }
 
@@ -1079,12 +1077,12 @@ select_peer (const struct GNUNET_HashCode *key,
   for (bc = 0; bc <= closest_bucket; bc++)
   {
     pos = k_buckets[bc].head;
-    while ((pos != NULL) && (count < bucket_size))
+    while ( (NULL != pos) && (count < bucket_size) )
     {
-      if ((bloom != NULL) &&
-          (GNUNET_YES ==
-           GNUNET_CONTAINER_bloomfilter_test (bloom,
-                                              &pos->phash)))
+      if ( (NULL != bloom) &&
+           (GNUNET_YES ==
+            GNUNET_CONTAINER_bloomfilter_test (bloom,
+                                               &pos->phash)) )
       {
         GNUNET_STATISTICS_update (GDS_stats,
                                   gettext_noop
@@ -1124,7 +1122,13 @@ select_peer (const struct GNUNET_HashCode *key,
         continue;               /* Ignore bloomfiltered peers */
       }
       if (0 == selected--)
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                    "Selected peer `%s' in random routing for %s\n",
+                    GNUNET_i2s (pos->id),
+                    GNUNET_h2s (key));
         return pos;
+      }
     }
   }
   GNUNET_break (0);
@@ -1169,7 +1173,9 @@ get_target_peers (const struct GNUNET_HashCode *key,
                                struct PeerInfo *);
   for (off = 0; off < ret; off++)
   {
-    nxt = select_peer (key, bloom, hop_count);
+    nxt = select_peer (key,
+                       bloom,
+                       hop_count);
     if (NULL == nxt)
       break;
     rtargets[off] = nxt;
@@ -1250,7 +1256,8 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
               "Adding myself (%s) to PUT bloomfilter for %s\n",
               GNUNET_i2s (&my_identity),
               GNUNET_h2s (key));
-  GNUNET_CONTAINER_bloomfilter_add (bf, &my_identity_hash);
+  GNUNET_CONTAINER_bloomfilter_add (bf,
+                                    &my_identity_hash);
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# PUT requests routed"),
                             1,
@@ -1352,8 +1359,7 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
  * @param key key for the content
  * @param xquery extended query
  * @param xquery_size number of bytes in @a xquery
- * @param reply_bf bloomfilter to filter duplicates
- * @param reply_bf_mutator mutator for @a reply_bf
+ * @param bg group to use for filtering replies
  * @param peer_bf filter for peers not to select (again)
  * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
  */
@@ -1361,14 +1367,14 @@ int
 GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                            enum GNUNET_DHT_RouteOption options,
                            uint32_t desired_replication_level,
-                           uint32_t hop_count, const struct GNUNET_HashCode * key,
-                           const void *xquery, size_t xquery_size,
-                           const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                           uint32_t reply_bf_mutator,
+                           uint32_t hop_count,
+                           const struct GNUNET_HashCode *key,
+                           const void *xquery,
+                           size_t xquery_size,
+                           struct GNUNET_BLOCK_Group *bg,
                            struct GNUNET_CONTAINER_BloomFilter *peer_bf)
 {
   unsigned int target_count;
-  unsigned int i;
   struct PeerInfo **targets;
   struct PeerInfo *target;
   struct GNUNET_MQ_Envelope *env;
@@ -1376,7 +1382,9 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
   struct PeerGetMessage *pgm;
   char *xq;
   size_t reply_bf_size;
+  void *reply_bf;
   unsigned int skip_count;
+  uint32_t bf_nonce;
 
   GNUNET_assert (NULL != peer_bf);
   GNUNET_STATISTICS_update (GDS_stats,
@@ -1403,11 +1411,22 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                 GNUNET_i2s (&my_identity));
     return GNUNET_NO;
   }
-  reply_bf_size = GNUNET_CONTAINER_bloomfilter_get_size (reply_bf);
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_group_serialize (bg,
+                                    &bf_nonce,
+                                    &reply_bf,
+                                    &reply_bf_size))
+  {
+    reply_bf = NULL;
+    reply_bf_size = 0;
+    bf_nonce = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                         UINT32_MAX);
+  }
   msize = xquery_size + reply_bf_size;
-  if (msize + sizeof (struct PeerGetMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize + sizeof (struct PeerGetMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
+    GNUNET_free_non_null (reply_bf);
     GNUNET_free (targets);
     return GNUNET_NO;
   }
@@ -1417,7 +1436,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                             GNUNET_NO);
   /* forward request */
   skip_count = 0;
-  for (i = 0; i < target_count; i++)
+  for (unsigned int i = 0; i < target_count; i++)
   {
     target = targets[i];
     if (GNUNET_MQ_get_length (target->mq) >= MAXIMUM_PENDING_PER_PEER)
@@ -1442,7 +1461,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
     pgm->hop_count = htonl (hop_count + 1);
     pgm->desired_replication_level = htonl (desired_replication_level);
     pgm->xquery_size = htonl (xquery_size);
-    pgm->bf_mutator = reply_bf_mutator;
+    pgm->bf_mutator = bf_nonce;
     GNUNET_break (GNUNET_YES ==
                   GNUNET_CONTAINER_bloomfilter_test (peer_bf,
                                                      &target->phash));
@@ -1455,16 +1474,14 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
     GNUNET_memcpy (xq,
                    xquery,
                    xquery_size);
-    if (NULL != reply_bf)
-      GNUNET_assert (GNUNET_OK ==
-                     GNUNET_CONTAINER_bloomfilter_get_raw_data (reply_bf,
-                                                                &xq
-                                                                [xquery_size],
-                                                                reply_bf_size));
+    GNUNET_memcpy (&xq[xquery_size],
+                   reply_bf,
+                   reply_bf_size);
     GNUNET_MQ_send (target->mq,
                     env);
   }
   GNUNET_free (targets);
+  GNUNET_free_non_null (reply_bf);
   return (skip_count < target_count) ? GNUNET_OK : GNUNET_NO;
 }
 
@@ -1505,12 +1522,12 @@ GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
 
   msize = data_size + (get_path_length + put_path_length) *
       sizeof (struct GNUNET_PeerIdentity);
-  if ((msize + sizeof (struct PeerResultMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
+  if ((msize + sizeof (struct PeerResultMessage) >= GNUNET_MAX_MESSAGE_SIZE) ||
       (get_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
       (put_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
-      (data_size > GNUNET_SERVER_MAX_MESSAGE_SIZE))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+      (data_size > GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return;
@@ -1610,7 +1627,7 @@ check_dht_p2p_put (void *cls,
        sizeof (struct PeerPutMessage) +
        putlen * sizeof (struct GNUNET_PeerIdentity)) ||
       (putlen >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
@@ -1663,10 +1680,13 @@ handle_dht_p2p_put (void *cls,
   if (GNUNET_YES == log_route_details_stderr)
   {
     char *tmp;
+    char *pp;
 
+    pp = GNUNET_STRINGS_pp2s (put_path,
+                              putlen);
     tmp = GNUNET_strdup (GNUNET_i2s (&my_identity));
     LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
-                 "R5N PUT %s: %s->%s (%u, %u=>%u)\n",
+                 "R5N PUT %s: %s->%s (%u, %u=>%u, PP: %s)\n",
                  GNUNET_h2s (&put->key),
                  GNUNET_i2s (peer->id),
                  tmp,
@@ -1674,8 +1694,9 @@ handle_dht_p2p_put (void *cls,
                  GNUNET_CRYPTO_hash_matching_bits (&peer->phash,
                                                    &put->key),
                  GNUNET_CRYPTO_hash_matching_bits (&my_identity_hash,
-                                                   &put->key)
-                );
+                                                   &put->key),
+                 pp);
+    GNUNET_free (pp);
     GNUNET_free (tmp);
   }
   switch (GNUNET_BLOCK_get_key
@@ -1712,11 +1733,12 @@ handle_dht_p2p_put (void *cls,
   {
     switch (GNUNET_BLOCK_evaluate (GDS_block_context,
                                    ntohl (put->type),
+                                   NULL, /* query group */
                                    GNUNET_BLOCK_EO_NONE,
                                    NULL,    /* query */
-                                   NULL, 0, /* bloom filer */
                                    NULL, 0, /* xquery */
-                                   payload, payload_size))
+                                   payload,
+                                   payload_size))
     {
     case GNUNET_BLOCK_EVALUATION_OK_MORE:
     case GNUNET_BLOCK_EVALUATION_OK_LAST:
@@ -1746,6 +1768,20 @@ handle_dht_p2p_put (void *cls,
     /* extend 'put path' by sender */
     if (0 != (options & GNUNET_DHT_RO_RECORD_ROUTE))
     {
+#if SANITY_CHECKS
+      for (unsigned int i=0;i<=putlen;i++)
+      {
+        for (unsigned int j=0;j<i;j++)
+        {
+          GNUNET_break (0 != memcmp (&pp[i],
+                                     &pp[j],
+                                     sizeof (struct GNUNET_PeerIdentity)));
+        }
+        GNUNET_break (0 != memcmp (&pp[i],
+                                   peer->id,
+                                   sizeof (struct GNUNET_PeerIdentity)));
+      }
+#endif
       GNUNET_memcpy (pp,
                      put_path,
                      putlen * sizeof (struct GNUNET_PeerIdentity));
@@ -1811,39 +1847,38 @@ handle_dht_p2p_put (void *cls,
  *
  * @param sender sender of the FIND PEER request
  * @param key peers close to this key are desired
- * @param bf peers matching this bf are excluded
- * @param bf_mutator mutator for bf
+ * @param bg group for filtering peers
  */
 static void
 handle_find_peer (const struct GNUNET_PeerIdentity *sender,
                   const struct GNUNET_HashCode *key,
-                  struct GNUNET_CONTAINER_BloomFilter *bf,
-                  uint32_t bf_mutator)
+                  struct GNUNET_BLOCK_Group *bg)
 {
   int bucket_idx;
   struct PeerBucket *bucket;
   struct PeerInfo *peer;
   unsigned int choice;
-  struct GNUNET_HashCode mhash;
   const struct GNUNET_HELLO_Message *hello;
+  size_t hello_size;
 
   /* first, check about our own HELLO */
   if (NULL != GDS_my_hello)
   {
-    GNUNET_BLOCK_mingle_hash (&my_identity_hash,
-                              bf_mutator,
-                              &mhash);
-    if ((NULL == bf) ||
-        (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (bf, &mhash)))
+    hello_size = GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) GDS_my_hello);
+    GNUNET_break (hello_size >= sizeof (struct GNUNET_MessageHeader));
+    if (GNUNET_BLOCK_EVALUATION_OK_MORE ==
+        GNUNET_BLOCK_evaluate (GDS_block_context,
+                               GNUNET_BLOCK_TYPE_DHT_HELLO,
+                               bg,
+                               GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO,
+                               &my_identity_hash,
+                               NULL, 0,
+                               GDS_my_hello,
+                               hello_size))
     {
-      size_t hello_size;
-
-      hello_size = GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) GDS_my_hello);
-      GNUNET_break (hello_size >= sizeof (struct GNUNET_MessageHeader));
       GDS_NEIGHBOURS_handle_reply (sender,
                                    GNUNET_BLOCK_TYPE_DHT_HELLO,
-                                   GNUNET_TIME_relative_to_absolute
-                                   (hello_expiration),
+                                   GNUNET_TIME_relative_to_absolute (hello_expiration),
                                    key,
                                    0,
                                    NULL,
@@ -1894,18 +1929,21 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
   do
   {
     peer = peer->next;
-    if (choice-- == 0)
+    if (0 == choice--)
       return;                   /* no non-masked peer available */
     if (NULL == peer)
       peer = bucket->head;
-    GNUNET_BLOCK_mingle_hash (&peer->phash,
-                              bf_mutator,
-                              &mhash);
     hello = GDS_HELLO_get (peer->id);
-  } while ( (hello == NULL) ||
-            (GNUNET_YES ==
-             GNUNET_CONTAINER_bloomfilter_test (bf,
-                                                &mhash)) );
+  } while ( (NULL == hello) ||
+            (GNUNET_BLOCK_EVALUATION_OK_MORE !=
+             GNUNET_BLOCK_evaluate (GDS_block_context,
+                                    GNUNET_BLOCK_TYPE_DHT_HELLO,
+                                    bg,
+                                    GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO,
+                                    &peer->phash,
+                                    NULL, 0,
+                                    hello,
+                                    (hello_size = GNUNET_HELLO_size (hello)))) );
   GDS_NEIGHBOURS_handle_reply (sender,
                                GNUNET_BLOCK_TYPE_DHT_HELLO,
                                GNUNET_TIME_relative_to_absolute
@@ -1916,14 +1954,14 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
                                0,
                                NULL,
                                hello,
-                               GNUNET_HELLO_size (hello));
+                               hello_size);
 }
 
 
 /**
  * Handle a result from local datacache for a GET operation.
  *
- * @param cls the `struct ClientHandle` of the client doing the query
+ * @param cls the `struct PeerInfo` for which this is a reply
  * @param type type of the block
  * @param expiration_time when does the content expire
  * @param key key for the content
@@ -1946,15 +1984,23 @@ handle_local_result (void *cls,
                      const void *data,
                      size_t data_size)
 {
-  // FIXME: we can probably do better here by
-  // passing the peer that did the query in the closure...
-  GDS_ROUTING_process (NULL,
-                       type,
-                       expiration_time,
-                       key,
-                       put_path_length, put_path,
-                       0, NULL,
-                       data, data_size);
+  struct PeerInfo *peer = cls;
+  char *pp;
+
+  pp = GNUNET_STRINGS_pp2s (put_path,
+                            put_path_length);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Found local result for %s (PP: %s)\n",
+              GNUNET_h2s (key),
+              pp);
+  GNUNET_free (pp);
+  GDS_NEIGHBOURS_handle_reply (peer->id,
+                               type,
+                               expiration_time,
+                               key,
+                               put_path_length, put_path,
+                               get_path_length, get_path,
+                               data, data_size);
 }
 
 
@@ -2000,16 +2046,11 @@ handle_dht_p2p_get (void *cls,
   enum GNUNET_BLOCK_Type type;
   enum GNUNET_DHT_RouteOption options;
   enum GNUNET_BLOCK_EvaluationResult eval;
-  struct GNUNET_CONTAINER_BloomFilter *reply_bf;
+  struct GNUNET_BLOCK_Group *bg;
   struct GNUNET_CONTAINER_BloomFilter *peer_bf;
   const char *xquery;
   int forwarded;
 
-  if (NULL == peer)
-  {
-    GNUNET_break (0);
-    return;
-  }
   /* parse and validate message */
   msize = ntohs (get->header.size);
   xquery_size = ntohl (get->xquery_size);
@@ -2017,7 +2058,6 @@ handle_dht_p2p_get (void *cls,
   type = ntohl (get->type);
   options = ntohl (get->options);
   xquery = (const char *) &get[1];
-  reply_bf = NULL;
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# P2P GET requests received"),
                             1,
@@ -2045,19 +2085,12 @@ handle_dht_p2p_get (void *cls,
                  xquery);
     GNUNET_free (tmp);
   }
-
-  if (reply_bf_size > 0)
-    reply_bf =
-        GNUNET_CONTAINER_bloomfilter_init (&xquery[xquery_size],
-                                           reply_bf_size,
-                                           GNUNET_CONSTANTS_BLOOMFILTER_K);
-  eval =
-      GNUNET_BLOCK_evaluate (GDS_block_context,
+  eval
+    = GNUNET_BLOCK_evaluate (GDS_block_context,
                              type,
+                             NULL,
                              GNUNET_BLOCK_EO_NONE,
                              &get->key,
-                             &reply_bf,
-                             get->bf_mutator,
                              xquery,
                              xquery_size,
                              NULL,
@@ -2066,8 +2099,6 @@ handle_dht_p2p_get (void *cls,
   {
     /* request invalid or block type not supported */
     GNUNET_break_op (eval == GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED);
-    if (NULL != reply_bf)
-      GNUNET_CONTAINER_bloomfilter_free (reply_bf);
     return;
   }
   peer_bf = GNUNET_CONTAINER_bloomfilter_init (get->bloomfilter,
@@ -2076,15 +2107,14 @@ handle_dht_p2p_get (void *cls,
   GNUNET_break_op (GNUNET_YES ==
                    GNUNET_CONTAINER_bloomfilter_test (peer_bf,
                                                       &peer->phash));
-  /* remember request for routing replies */
-  GDS_ROUTING_add (peer->id,
-                   type,
-                   options,
-                   &get->key,
-                   xquery,
-                   xquery_size,
-                   reply_bf,
-                   get->bf_mutator);
+  bg = GNUNET_BLOCK_group_create (GDS_block_context,
+                                  type,
+                                  get->bf_mutator,
+                                  &xquery[xquery_size],
+                                  reply_bf_size,
+                                  "filter-size",
+                                  reply_bf_size,
+                                  NULL);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "GET for %s at %s after %u hops\n",
               GNUNET_h2s (&get->key),
@@ -2103,8 +2133,7 @@ handle_dht_p2p_get (void *cls,
                                 GNUNET_NO);
       handle_find_peer (peer->id,
                         &get->key,
-                        reply_bf,
-                        get->bf_mutator);
+                        bg);
     }
     else
     {
@@ -2112,10 +2141,9 @@ handle_dht_p2p_get (void *cls,
                                        type,
                                        xquery,
                                        xquery_size,
-                                       &reply_bf,
-                                       get->bf_mutator,
+                                       bg,
                                        &handle_local_result,
-                                       NULL);
+                                       peer);
     }
   }
   else
@@ -2126,6 +2154,15 @@ handle_dht_p2p_get (void *cls,
                               GNUNET_NO);
   }
 
+  /* remember request for routing replies */
+  GDS_ROUTING_add (peer->id,
+                        type,
+                        bg, /* bg now owned by routing, but valid at least until end of this function! */
+                        options,
+                        &get->key,
+                        xquery,
+                        xquery_size);
+
   /* P2P forwarding */
   forwarded = GNUNET_NO;
   if (eval != GNUNET_BLOCK_EVALUATION_OK_LAST)
@@ -2136,8 +2173,7 @@ handle_dht_p2p_get (void *cls,
                                            &get->key,
                                            xquery,
                                            xquery_size,
-                                           reply_bf,
-                                           get->bf_mutator,
+                                           bg,
                                            peer_bf);
   GDS_CLIENTS_process_get (options
                            | (GNUNET_OK == forwarded)
@@ -2149,10 +2185,7 @@ handle_dht_p2p_get (void *cls,
                            NULL,
                            &get->key);
 
-
-  /* clean up */
-  if (NULL != reply_bf)
-    GNUNET_CONTAINER_bloomfilter_free (reply_bf);
+  /* clean up; note that 'bg' is owned by routing now! */
   GNUNET_CONTAINER_bloomfilter_free (peer_bf);
 }
 
@@ -2180,9 +2213,9 @@ check_dht_p2p_result (void *cls,
                                             put_path_length) *
        sizeof (struct GNUNET_PeerIdentity)) ||
       (get_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
       (put_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
@@ -2192,6 +2225,81 @@ check_dht_p2p_result (void *cls,
 
 
 /**
+ * Process a reply, after the @a get_path has been updated.
+ *
+ * @param expiration_time when does the reply expire
+ * @param key key matching the query
+ * @param get_path_length number of entries in @a get_path
+ * @param get_path path the reply has taken
+ * @param put_path_length number of entries in @a put_path
+ * @param put_path path the PUT has taken
+ * @param type type of the block
+ * @param data_size number of bytes in @a data
+ * @param data payload of the reply
+ */
+static void
+process_reply_with_path (struct GNUNET_TIME_Absolute expiration_time,
+                         const struct GNUNET_HashCode *key,
+                         unsigned int get_path_length,
+                         const struct GNUNET_PeerIdentity *get_path,
+                         unsigned int put_path_length,
+                         const struct GNUNET_PeerIdentity *put_path,
+                         enum GNUNET_BLOCK_Type type,
+                         size_t data_size,
+                         const void *data)
+{
+  /* forward to local clients */
+  GDS_CLIENTS_handle_reply (expiration_time,
+                            key,
+                            get_path_length,
+                            get_path,
+                            put_path_length,
+                            put_path,
+                            type,
+                            data_size,
+                            data);
+  GDS_CLIENTS_process_get_resp (type,
+                                get_path,
+                                get_path_length,
+                                put_path,
+                                put_path_length,
+                                expiration_time,
+                                key,
+                                data,
+                                data_size);
+  if (GNUNET_YES == cache_results)
+  {
+    struct GNUNET_PeerIdentity xput_path[get_path_length + 1 + put_path_length];
+
+    GNUNET_memcpy (xput_path,
+                   put_path,
+                   put_path_length * sizeof (struct GNUNET_PeerIdentity));
+    GNUNET_memcpy (&xput_path[put_path_length],
+                   get_path,
+                   get_path_length * sizeof (struct GNUNET_PeerIdentity));
+
+    GDS_DATACACHE_handle_put (expiration_time,
+                              key,
+                              get_path_length + put_path_length,
+                              xput_path,
+                              type,
+                              data_size,
+                              data);
+  }
+  /* forward to other peers */
+  GDS_ROUTING_process (type,
+                       expiration_time,
+                       key,
+                       put_path_length,
+                       put_path,
+                       get_path_length,
+                       get_path,
+                       data,
+                       data_size);
+}
+
+
+/**
  * Core handler for p2p result messages.
  *
  * @param cls closure
@@ -2233,14 +2341,23 @@ handle_dht_p2p_result (void *cls,
   if (GNUNET_YES == log_route_details_stderr)
   {
     char *tmp;
+    char *pp;
+    char *gp;
 
+    gp = GNUNET_STRINGS_pp2s (get_path,
+                              get_path_length);
+    pp = GNUNET_STRINGS_pp2s (put_path,
+                              put_path_length);
     tmp = GNUNET_strdup (GNUNET_i2s (&my_identity));
     LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
-                 "R5N RESULT %s: %s->%s (%u)\n",
+                 "R5N RESULT %s: %s->%s (GP: %s, PP: %s)\n",
                  GNUNET_h2s (&prm->key),
                  GNUNET_i2s (peer->id),
                  tmp,
-                 get_path_length + 1);
+                 gp,
+                 pp);
+    GNUNET_free (gp);
+    GNUNET_free (pp);
     GNUNET_free (tmp);
   }
   /* if we got a HELLO, consider it for our own routing table */
@@ -2276,7 +2393,27 @@ handle_dht_p2p_result (void *cls,
                    h);
   }
 
-  /* append 'peer' to 'get_path' */
+
+  /* First, check if 'peer' is already on the path, and if
+     so, truncate it instead of expanding. */
+  for (unsigned int i=0;i<=get_path_length;i++)
+    if (0 == memcmp (&get_path[i],
+                     peer->id,
+                     sizeof (struct GNUNET_PeerIdentity)))
+    {
+      process_reply_with_path (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+                               &prm->key,
+                               i,
+                               get_path,
+                               put_path_length,
+                               put_path,
+                               type,
+                               data_size,
+                               data);
+      return;
+    }
+
+  /* Need to append 'peer' to 'get_path' (normal case) */
   {
     struct GNUNET_PeerIdentity xget_path[get_path_length + 1];
 
@@ -2284,56 +2421,16 @@ handle_dht_p2p_result (void *cls,
                    get_path,
                    get_path_length * sizeof (struct GNUNET_PeerIdentity));
     xget_path[get_path_length] = *peer->id;
-    get_path_length++;
-
-    /* forward to local clients */
-    GDS_CLIENTS_handle_reply (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                              &prm->key,
-                              get_path_length,
-                              xget_path,
-                              put_path_length,
-                              put_path,
-                              type,
-                              data_size,
-                              data);
-    GDS_CLIENTS_process_get_resp (type,
-                                  xget_path,
-                                  get_path_length,
-                                  put_path, put_path_length,
-                                  GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                                  &prm->key,
-                                  data,
-                                  data_size);
-    if (GNUNET_YES == cache_results)
-    {
-      struct GNUNET_PeerIdentity xput_path[get_path_length + 1 + put_path_length];
 
-      GNUNET_memcpy (xput_path,
-                     put_path,
-                     put_path_length * sizeof (struct GNUNET_PeerIdentity));
-      GNUNET_memcpy (&xput_path[put_path_length],
-                     xget_path,
-                     get_path_length * sizeof (struct GNUNET_PeerIdentity));
-
-      GDS_DATACACHE_handle_put (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                                &prm->key,
-                                get_path_length + put_path_length,
-                                xput_path,
-                                type,
-                                data_size,
-                                data);
-    }
-    /* forward to other peers */
-    GDS_ROUTING_process (NULL,
-                         type,
-                         GNUNET_TIME_absolute_ntoh (prm->expiration_time),
-                         &prm->key,
-                         put_path_length,
-                         put_path,
-                         get_path_length,
-                         xget_path,
-                         data,
-                         data_size);
+    process_reply_with_path (GNUNET_TIME_absolute_ntoh (prm->expiration_time),
+                             &prm->key,
+                             get_path_length + 1,
+                             xget_path,
+                             put_path_length,
+                             put_path,
+                             type,
+                             data_size,
+                             data);
   }
 }
 
diff --git a/src/dht/gnunet-service-dht_neighbours.h b/src/dht/gnunet-service-dht_neighbours.h
index d89e5c54f..34b76ee8a 100644
--- a/src/dht/gnunet-service-dht_neighbours.h
+++ b/src/dht/gnunet-service-dht_neighbours.h
@@ -77,8 +77,7 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
  * @param key key for the content
  * @param xquery extended query
  * @param xquery_size number of bytes in @a xquery
- * @param reply_bf bloomfilter to filter duplicates
- * @param reply_bf_mutator mutator for @a reply_bf
+ * @param bg block group to filter replies
  * @param peer_bf filter for peers not to select (again, updated)
  * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
  */
@@ -88,9 +87,9 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                            uint32_t desired_replication_level,
                            uint32_t hop_count,
                            const struct GNUNET_HashCode *key,
-                           const void *xquery, size_t xquery_size,
-                           const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                           uint32_t reply_bf_mutator,
+                           const void *xquery,
+                           size_t xquery_size,
+                           struct GNUNET_BLOCK_Group *bg,
                            struct GNUNET_CONTAINER_BloomFilter *peer_bf);
 
 
@@ -114,12 +113,13 @@ void
 GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
                              enum GNUNET_BLOCK_Type type,
                              struct GNUNET_TIME_Absolute expiration_time,
-                             const struct GNUNET_HashCode * key,
+                             const struct GNUNET_HashCode *key,
                              unsigned int put_path_length,
                              const struct GNUNET_PeerIdentity *put_path,
                              unsigned int get_path_length,
                              const struct GNUNET_PeerIdentity *get_path,
-                             const void *data, size_t data_size);
+                             const void *data,
+                             size_t data_size);
 
 
 /**
diff --git a/src/dht/gnunet-service-dht_routing.c b/src/dht/gnunet-service-dht_routing.c
index 978c46d73..098b6e895 100644
--- a/src/dht/gnunet-service-dht_routing.c
+++ b/src/dht/gnunet-service-dht_routing.c
@@ -58,9 +58,9 @@ struct RecentRequest
   struct GNUNET_CONTAINER_HeapNode *heap_node;
 
   /**
-   * Bloomfilter for replies to drop.
+   * Block group for filtering replies.
    */
-  struct GNUNET_CONTAINER_BloomFilter *reply_bf;
+  struct GNUNET_BLOCK_Group *bg;
 
   /**
    * Type of the requested block.
@@ -79,11 +79,6 @@ struct RecentRequest
   size_t xquery_size;
 
   /**
-   * Mutator value for the reply_bf, see gnunet_block_lib.h
-   */
-  uint32_t reply_bf_mutator;
-
-  /**
    * Request options.
    */
   enum GNUNET_DHT_RouteOption options;
@@ -128,17 +123,17 @@ struct ProcessContext
   struct GNUNET_TIME_Absolute expiration_time;
 
   /**
-   * Number of entries in 'put_path'.
+   * Number of entries in @e put_path.
    */
   unsigned int put_path_length;
 
   /**
-   * Number of entries in 'get_path'.
+   * Number of entries in @e get_path.
    */
   unsigned int get_path_length;
 
   /**
-   * Number of bytes in 'data'.
+   * Number of bytes in @e data.
    */
   size_t data_size;
 
@@ -207,14 +202,18 @@ process (void *cls,
   eval
     = GNUNET_BLOCK_evaluate (GDS_block_context,
                              pc->type,
+                             rr->bg,
                              GNUNET_BLOCK_EO_NONE,
                              eval_key,
-                             &rr->reply_bf,
-                             rr->reply_bf_mutator,
                              rr->xquery,
                              rr->xquery_size,
                              pc->data,
                              pc->data_size);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Result for %s of type %d was evaluated as %d\n",
+              GNUNET_h2s (key),
+              pc->type,
+              eval);
   switch (eval)
   {
   case GNUNET_BLOCK_EVALUATION_OK_MORE:
@@ -223,8 +222,13 @@ process (void *cls,
                               gettext_noop
                               ("# Good REPLIES matched against routing table"),
                               1, GNUNET_NO);
-    GDS_NEIGHBOURS_handle_reply (&rr->peer, pc->type, pc->expiration_time, key,
-                                 ppl, pc->put_path, gpl, pc->get_path, pc->data,
+    GDS_NEIGHBOURS_handle_reply (&rr->peer,
+                                 pc->type,
+                                 pc->expiration_time,
+                                 key,
+                                 ppl, pc->put_path,
+                                 gpl, pc->get_path,
+                                 pc->data,
                                  pc->data_size);
     break;
   case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE:
@@ -275,16 +279,15 @@ process (void *cls,
  * @param type type of the block
  * @param expiration_time when does the content expire
  * @param key key for the content
- * @param put_path_length number of entries in put_path
+ * @param put_path_length number of entries in @a put_path
  * @param put_path peers the original PUT traversed (if tracked)
- * @param get_path_length number of entries in get_path
+ * @param get_path_length number of entries in @a get_path
  * @param get_path peers this reply has traversed so far (if tracked)
  * @param data payload of the reply
  * @param data_size number of bytes in data
  */
 void
-GDS_ROUTING_process (void *cls,
-                     enum GNUNET_BLOCK_Type type,
+GDS_ROUTING_process (enum GNUNET_BLOCK_Type type,
                      struct GNUNET_TIME_Absolute expiration_time,
                      const struct GNUNET_HashCode *key,
                      unsigned int put_path_length,
@@ -338,7 +341,7 @@ expire_oldest_entry ()
   recent_req = GNUNET_CONTAINER_heap_peek (recent_heap);
   GNUNET_assert (recent_req != NULL);
   GNUNET_CONTAINER_heap_remove_node (recent_req->heap_node);
-  GNUNET_CONTAINER_bloomfilter_free (recent_req->reply_bf);
+  GNUNET_BLOCK_group_destroy (recent_req->bg);
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap_remove (recent_map,
                                                        &recent_req->key,
@@ -374,18 +377,10 @@ try_combine_recent (void *cls,
                      rr->xquery,
                      in->xquery_size)) )
     return GNUNET_OK;
-  if (in->reply_bf_mutator != rr->reply_bf_mutator)
-  {
-    rr->reply_bf_mutator = in->reply_bf_mutator;
-    GNUNET_CONTAINER_bloomfilter_free (rr->reply_bf);
-    rr->reply_bf = in->reply_bf;
-  }
-  else
-  {
-    GNUNET_CONTAINER_bloomfilter_or2 (rr->reply_bf,
-                                      in->reply_bf);
-    GNUNET_CONTAINER_bloomfilter_free (in->reply_bf);
-  }
+  GNUNET_break (GNUNET_SYSERR !=
+                GNUNET_BLOCK_group_merge (in->bg,
+                                          rr->bg));
+  rr->bg = in->bg;
   GNUNET_free (in);
   return GNUNET_SYSERR;
 }
@@ -406,12 +401,11 @@ try_combine_recent (void *cls,
 void
 GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
                  enum GNUNET_BLOCK_Type type,
+                 struct GNUNET_BLOCK_Group *bg,
                  enum GNUNET_DHT_RouteOption options,
                  const struct GNUNET_HashCode *key,
                  const void *xquery,
-                 size_t xquery_size,
-                 const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                 uint32_t reply_bf_mutator)
+                 size_t xquery_size)
 {
   struct RecentRequest *recent_req;
 
@@ -419,17 +413,19 @@ GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
     expire_oldest_entry ();
   GNUNET_STATISTICS_update (GDS_stats,
                             gettext_noop ("# Entries added to routing table"),
-                            1, GNUNET_NO);
+                            1,
+                            GNUNET_NO);
   recent_req = GNUNET_malloc (sizeof (struct RecentRequest) + xquery_size);
   recent_req->peer = *sender;
   recent_req->key = *key;
-  recent_req->reply_bf = GNUNET_CONTAINER_bloomfilter_copy (reply_bf);
+  recent_req->bg = bg;
   recent_req->type = type;
   recent_req->options = options;
   recent_req->xquery = &recent_req[1];
-  GNUNET_memcpy (&recent_req[1], xquery, xquery_size);
+  GNUNET_memcpy (&recent_req[1],
+                 xquery,
+                 xquery_size);
   recent_req->xquery_size = xquery_size;
-  recent_req->reply_bf_mutator = reply_bf_mutator;
   if (GNUNET_SYSERR ==
       GNUNET_CONTAINER_multihashmap_get_multiple (recent_map,
                                                   key,
@@ -442,13 +438,14 @@ GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
                               1, GNUNET_NO);
     return;
   }
-  recent_req->heap_node =
-      GNUNET_CONTAINER_heap_insert (recent_heap, recent_req,
+  recent_req->heap_node
+    = GNUNET_CONTAINER_heap_insert (recent_heap,
+                                    recent_req,
                                     GNUNET_TIME_absolute_get ().abs_value_us);
-  GNUNET_CONTAINER_multihashmap_put (recent_map, key, recent_req,
+  GNUNET_CONTAINER_multihashmap_put (recent_map,
+                                     key,
+                                     recent_req,
                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-
 }
 
 
diff --git a/src/dht/gnunet-service-dht_routing.h b/src/dht/gnunet-service-dht_routing.h
index 7c57361dc..ad7958363 100644
--- a/src/dht/gnunet-service-dht_routing.h
+++ b/src/dht/gnunet-service-dht_routing.h
@@ -35,10 +35,9 @@
  * Handle a reply (route to origin).  Only forwards the reply back to
  * other peers waiting for it.  Does not do local caching or
  * forwarding to local clients.  Essentially calls
- * GDS_NEIGHBOURS_handle_reply for all peers that sent us a matching
+ * #GDS_NEIGHBOURS_handle_reply() for all peers that sent us a matching
  * request recently.
  *
- * @param cls closure
  * @param type type of the block
  * @param expiration_time when does the content expire
  * @param key key for the content
@@ -50,8 +49,7 @@
  * @param data_size number of bytes in @a data
  */
 void
-GDS_ROUTING_process (void *cls,
-                     enum GNUNET_BLOCK_Type type,
+GDS_ROUTING_process (enum GNUNET_BLOCK_Type type,
                      struct GNUNET_TIME_Absolute expiration_time,
                      const struct GNUNET_HashCode *key,
                      unsigned int put_path_length,
@@ -67,21 +65,20 @@ GDS_ROUTING_process (void *cls,
  *
  * @param sender peer that originated the request
  * @param type type of the block
+ * @param bg block group to evaluate replies, henceforth owned by routing
  * @param options options for processing
  * @param key key for the content
  * @param xquery extended query
  * @param xquery_size number of bytes in @a xquery
- * @param reply_bf bloomfilter to filter duplicates
- * @param reply_bf_mutator mutator for @a reply_bf
-*/
+ */
 void
 GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
                  enum GNUNET_BLOCK_Type type,
+                 struct GNUNET_BLOCK_Group *bg,
                  enum GNUNET_DHT_RouteOption options,
-                 const struct GNUNET_HashCode * key, const void *xquery,
-                 size_t xquery_size,
-                 const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                 uint32_t reply_bf_mutator);
+                 const struct GNUNET_HashCode * key,
+                 const void *xquery,
+                 size_t xquery_size);
 
 
 /**
diff --git a/src/dht/gnunet-service-wdht.c b/src/dht/gnunet-service-wdht.c
deleted file mode 100644
index e1ca1c968..000000000
--- a/src/dht/gnunet-service-wdht.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-wdht.c
- * @brief GNUnet DHT service
- * @author Christian Grothoff
- * @author Nathan Evans
- */
-#include "platform.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-wdht.h"
-#include "gnunet-service-dht_datacache.h"
-#include "gnunet-service-dht_neighbours.h"
-#include "gnunet-service-dht_nse.h"
-
-
-/* Code shared between different DHT implementations */
-#include "gnunet-service-dht_clients.c"
-
-
-/**
- * Task run during shutdown.
- *
- * @param cls unused
- */
-static void
-shutdown_task (void *cls)
-{
-  GDS_NEIGHBOURS_done ();
-  GDS_DATACACHE_done ();
-  GDS_NSE_done ();
-  if (NULL != GDS_block_context)
-  {
-    GNUNET_BLOCK_context_destroy (GDS_block_context);
-    GDS_block_context = NULL;
-  }
-  if (NULL != GDS_stats)
-  {
-    GNUNET_STATISTICS_destroy (GDS_stats, GNUNET_YES);
-    GDS_stats = NULL;
-  }
-  GDS_CLIENTS_stop ();
-}
-
-
-/**
- * Process dht requests.
- *
- * @param cls closure
- * @param c configuration to use
- * @param service the initialized service
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *c,
-     struct GNUNET_SERVICE_Handle *service)
-{
-  GDS_cfg = c;
-  GDS_service = service;
-  GDS_block_context = GNUNET_BLOCK_context_create (GDS_cfg);
-  GDS_stats = GNUNET_STATISTICS_create ("dht",
-                                        GDS_cfg);
-  GDS_NSE_init ();
-  GDS_DATACACHE_init ();
-  GDS_CLIENTS_init ();
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  if (GNUNET_OK != GDS_NEIGHBOURS_init ())
-  {
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-}
-
-
-/* Finally, define the main method */
-GDS_DHT_SERVICE_INIT("wdht", &run);
-
-
-/* end of gnunet-service-wdht.c */
diff --git a/src/dht/gnunet-service-wdht.h b/src/dht/gnunet-service-wdht.h
deleted file mode 100644
index 5a8e2e21d..000000000
--- a/src/dht/gnunet-service-wdht.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht.h
- * @brief GNUnet DHT globals
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_XDHT_H
-#define GNUNET_SERVICE_XDHT_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet_transport_service.h"
-
-#define DEBUG_DHT GNUNET_EXTRA_LOGGING
-
-/**
- * Configuration we use.
- */
-extern const struct GNUNET_CONFIGURATION_Handle *GDS_cfg;
-
-/**
- * Our handle to the BLOCK library.
- */
-extern struct GNUNET_BLOCK_Context *GDS_block_context;
-
-/**
- * Handle for the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *GDS_stats;
-
-#endif
diff --git a/src/dht/gnunet-service-wdht_clients.c b/src/dht/gnunet-service-wdht_clients.c
deleted file mode 100644
index 7ad0d2904..000000000
--- a/src/dht/gnunet-service-wdht_clients.c
+++ /dev/null
@@ -1,1428 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-wdht_clients.c
- * @brief GNUnet DHT service's client management code
- * @author Christian Grothoff
- * @author Nathan Evans
- */
-
-#include "platform.h"
-#include "gnunet_constants.h"
-#include "gnunet_protocols.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-wdht.h"
-#include "gnunet-service-wdht_clients.h"
-#include "gnunet-service-dht_datacache.h"
-#include "gnunet-service-wdht_neighbours.h"
-#include "dht.h"
-
-
-/**
- * Should routing details be logged to stderr (for debugging)?
- */
-#define LOG_TRAFFIC(kind,...) GNUNET_log_from (kind, "dht-traffic",__VA_ARGS__)
-
-#define LOG(kind,...) GNUNET_log_from (kind, "dht-clients",__VA_ARGS__)
-
-#define DEBUG(...)                                           \
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
-
-/**
- * Linked list of messages to send to clients.
- */
-struct PendingMessage
-{
-  /**
-   * Pointer to next item in the list
-   */
-  struct PendingMessage *next;
-
-  /**
-   * Pointer to previous item in the list
-   */
-  struct PendingMessage *prev;
-
-  /**
-   * Actual message to be sent, allocated at the end of the struct:
-   * // msg = (cast) &pm[1];
-   * // GNUNET_memcpy (&pm[1], data, len);
-   */
-  const struct GNUNET_MessageHeader *msg;
-
-};
-
-
-/**
- * Struct containing information about a client,
- * handle to connect to it, and any pending messages
- * that need to be sent to it.
- */
-struct ClientList
-{
-  /**
-   * Linked list of active clients
-   */
-  struct ClientList *next;
-
-  /**
-   * Linked list of active clients
-   */
-  struct ClientList *prev;
-
-  /**
-   * The handle to this client
-   */
-  struct GNUNET_SERVER_Client *client_handle;
-
-  /**
-   * Handle to the current transmission request, NULL
-   * if none pending.
-   */
-  struct GNUNET_SERVER_TransmitHandle *transmit_handle;
-
-  /**
-   * Linked list of pending messages for this client
-   */
-  struct PendingMessage *pending_head;
-
-  /**
-   * Tail of linked list of pending messages for this client
-   */
-  struct PendingMessage *pending_tail;
-
-};
-
-
-/**
- * Entry in the local forwarding map for a client's GET request.
- */
-struct ClientQueryRecord
-{
-
-  /**
-   * The key this request was about
-   */
-  struct GNUNET_HashCode key;
-
-  /**
-   * Client responsible for the request.
-   */
-  struct ClientList *client;
-
-  /**
-   * Extended query (see gnunet_block_lib.h), allocated at the end of this struct.
-   */
-  const void *xquery;
-
-  /**
-   * Replies we have already seen for this request.
-   */
-  struct GNUNET_HashCode *seen_replies;
-
-  /**
-   * Pointer to this nodes heap location in the retry-heap (for fast removal)
-   */
-  struct GNUNET_CONTAINER_HeapNode *hnode;
-
-  /**
-   * What's the delay between re-try operations that we currently use for this
-   * request?
-   */
-  struct GNUNET_TIME_Relative retry_frequency;
-
-  /**
-   * What's the next time we should re-try this request?
-   */
-  struct GNUNET_TIME_Absolute retry_time;
-
-  /**
-   * The unique identifier of this request
-   */
-  uint64_t unique_id;
-
-  /**
-   * Number of bytes in xquery.
-   */
-  size_t xquery_size;
-
-  /**
-   * Number of entries in @e seen_replies.
-   */
-  unsigned int seen_replies_count;
-
-  /**
-   * Desired replication level
-   */
-  uint32_t replication;
-
-  /**
-   * Any message options for this request
-   */
-  uint32_t msg_options;
-
-  /**
-   * The type for the data for the GET request.
-   */
-  enum GNUNET_BLOCK_Type type;
-
-};
-
-
-/**
- * Struct containing paremeters of monitoring requests.
- */
-struct ClientMonitorRecord
-{
-
-  /**
-   * Next element in DLL.
-   */
-  struct ClientMonitorRecord    *next;
-
-  /**
-   * Previous element in DLL.
-   */
-  struct ClientMonitorRecord    *prev;
-
-  /**
-   * Type of blocks that are of interest
-   */
-  enum GNUNET_BLOCK_Type        type;
-
-  /**
-   * Key of data of interest, NULL for all.
-   */
-  struct GNUNET_HashCode         *key;
-
-  /**
-   * Flag whether to notify about GET messages.
-   */
-  int16_t get;
-
-  /**
-   * Flag whether to notify about GET_REPONSE messages.
-   */
-  int16_t get_resp;
-
-  /**
-   * Flag whether to notify about PUT messages.
-   */
-  uint16_t put;
-
-  /**
-   * Client to notify of these requests.
-   */
-  struct ClientList             *client;
-};
-
-
-/**
- * List of active clients.
- */
-static struct ClientList *client_head;
-
-/**
- * List of active clients.
- */
-static struct ClientList *client_tail;
-
-/**
- * List of active monitoring requests.
- */
-static struct ClientMonitorRecord *monitor_head;
-
-/**
- * List of active monitoring requests.
- */
-static struct ClientMonitorRecord *monitor_tail;
-
-/**
- * Hashmap for fast key based lookup, maps keys to `struct ClientQueryRecord` entries.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *forward_map;
-
-/**
- * Heap with all of our client's request, sorted by retry time (earliest on top).
- */
-static struct GNUNET_CONTAINER_Heap *retry_heap;
-
-/**
- * Task that re-transmits requests (using retry_heap).
- */
-static struct GNUNET_SCHEDULER_Task * retry_task;
-
-
-/**
- * Task run to check for messages that need to be sent to a client.
- *
- * @param client a ClientList, containing the client and any messages to be sent to it
- */
-static void
-process_pending_messages (struct ClientList *client);
-
-
-/**
- * Callback called as a result of issuing a #GNUNET_SERVER_notify_transmit_ready()
- * request.  A ClientList is passed as closure, take the head of the list
- * and copy it into buf, which has the result of sending the message to the
- * client.
- *
- * @param cls closure to this call
- * @param size maximum number of bytes available to send
- * @param buf where to copy the actual message to
- *
- * @return the number of bytes actually copied, 0 indicates failure
- */
-static size_t
-send_reply_to_client (void *cls, size_t size, void *buf)
-{
-  struct ClientList *client = cls;
-  char *cbuf = buf;
-  struct PendingMessage *reply;
-  size_t off;
-  size_t msize;
-
-  client->transmit_handle = NULL;
-  if (buf == NULL)
-  {
-    /* client disconnected */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client %p disconnected, pending messages will be discarded\n",
-                client->client_handle);
-    return 0;
-  }
-  off = 0;
-  while ((NULL != (reply = client->pending_head)) &&
-         (size >= off + (msize = ntohs (reply->msg->size))))
-  {
-    GNUNET_CONTAINER_DLL_remove (client->pending_head, client->pending_tail,
-                                 reply);
-    GNUNET_memcpy (&cbuf[off], reply->msg, msize);
-    GNUNET_free (reply);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Transmitting %u bytes to client %p\n",
-                (unsigned int) msize,
-                client->client_handle);
-    off += msize;
-  }
-  process_pending_messages (client);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Transmitted %u/%u bytes to client %p\n",
-              (unsigned int) off,
-              (unsigned int) size,
-              client->client_handle);
-  return off;
-}
-
-
-/**
- * Task run to check for messages that need to be sent to a client.
- *
- * @param client a ClientList, containing the client and any messages to be sent to it
- */
-static void
-process_pending_messages (struct ClientList *client)
-{
-  if ((client->pending_head == NULL) || (client->transmit_handle != NULL))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Not asking for transmission to %p now: %s\n",
-                client->client_handle,
-                client->pending_head ==
-                NULL ? "no more messages" : "request already pending");
-    return;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Asking for transmission of %u bytes to client %p\n",
-              ntohs (client->pending_head->msg->size), client->client_handle);
-  client->transmit_handle =
-      GNUNET_SERVER_notify_transmit_ready (client->client_handle,
-                                           ntohs (client->pending_head->
-                                                  msg->size),
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           &send_reply_to_client, client);
-}
-
-
-/**
- * Add a PendingMessage to the clients list of messages to be sent
- *
- * @param client the active client to send the message to
- * @param pending_message the actual message to send
- */
-static void
-add_pending_message (struct ClientList *client,
-                     struct PendingMessage *pending_message)
-{
-  GNUNET_CONTAINER_DLL_insert_tail (client->pending_head, client->pending_tail,
-                                    pending_message);
-  process_pending_messages (client);
-}
-
-
-/**
- * Closure for #forward_reply()
- */
-struct ForwardReplyContext
-{
-
-  /**
-   * Actual message to send to matching clients.
-   */
-  struct PendingMessage *pm;
-
-  /**
-   * Embedded payload.
-   */
-  const void *data;
-
-  /**
-   * Type of the data.
-   */
-  enum GNUNET_BLOCK_Type type;
-
-  /**
-   * Number of bytes in data.
-   */
-  size_t data_size;
-
-  /**
-   * Do we need to copy @e pm because it was already used?
-   */
-  int do_copy;
-
-};
-
-
-/**
- * Find a client if it exists, add it otherwise.
- *
- * @param client the server handle to the client
- *
- * @return the client if found, a new client otherwise
- */
-static struct ClientList *
-find_active_client (struct GNUNET_SERVER_Client *client)
-{
-  struct ClientList *pos = client_head;
-  struct ClientList *ret;
-
-  while (pos != NULL)
-  {
-    if (pos->client_handle == client)
-      return pos;
-    pos = pos->next;
-  }
-  ret = GNUNET_new (struct ClientList);
-  ret->client_handle = client;
-  GNUNET_CONTAINER_DLL_insert (client_head, client_tail, ret);
-  return ret;
-}
-
-
-/**
- * Iterator over hash map entries that frees all entries
- * associated with the given client.
- *
- * @param cls client to search for in source routes
- * @param key current key code (ignored)
- * @param value value in the hash map, a ClientQueryRecord
- * @return #GNUNET_YES (we should continue to iterate)
- */
-static int
-remove_client_records (void *cls, const struct GNUNET_HashCode * key, void *value)
-{
-  struct ClientList *client = cls;
-  struct ClientQueryRecord *record = value;
-
-  if (record->client != client)
-    return GNUNET_YES;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Removing client %p's record for key %s\n", client,
-              GNUNET_h2s (key));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (forward_map, key,
-                                                       record));
-  if (NULL != record->hnode)
-    GNUNET_CONTAINER_heap_remove_node (record->hnode);
-  GNUNET_array_grow (record->seen_replies, record->seen_replies_count, 0);
-  GNUNET_free (record);
-  return GNUNET_YES;
-}
-
-
-/**
- * Iterator over hash map entries that send a given reply to
- * each of the matching clients.  With some tricky recycling
- * of the buffer.
- *
- * @param cls the `struct ForwardReplyContext`
- * @param key current key
- * @param value value in the hash map, a ClientQueryRecord
- * @return #GNUNET_YES (we should continue to iterate),
- *         if the result is mal-formed, #GNUNET_NO
- */
-static int
-forward_reply (void *cls,
-               const struct GNUNET_HashCode *key,
-               void *value)
-{
-  struct ForwardReplyContext *frc = cls;
-  struct ClientQueryRecord *record = value;
-  struct PendingMessage *pm;
-  struct GNUNET_DHT_ClientResultMessage *reply;
-  enum GNUNET_BLOCK_EvaluationResult eval;
-  int do_free;
-  struct GNUNET_HashCode ch;
-  unsigned int i;
-
-  LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG,
-               "XVINE CLIENT-RESULT %s\n",
-               GNUNET_h2s_full (key));
-#if 0
-  if ((record->type != GNUNET_BLOCK_TYPE_ANY) && (record->type != frc->type))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Record type missmatch, not passing request for key %s to local client\n",
-         GNUNET_h2s (key));
-
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop
-                              ("# Key match, type mismatches in REPLY to CLIENT"),
-                              1, GNUNET_NO);
-    return GNUNET_YES;          /* type mismatch */
-  }
-#endif
-  GNUNET_CRYPTO_hash (frc->data, frc->data_size, &ch);
-  for (i = 0; i < record->seen_replies_count; i++)
-    if (0 == memcmp (&record->seen_replies[i], &ch, sizeof (struct GNUNET_HashCode)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Duplicate reply, not passing request for key %s to local client\n",
-           GNUNET_h2s (key));
-      GNUNET_STATISTICS_update (GDS_stats,
-                                gettext_noop
-                                ("# Duplicate REPLIES to CLIENT request dropped"),
-                                1, GNUNET_NO);
-      return GNUNET_YES;        /* duplicate */
-    }
-  eval =
-      GNUNET_BLOCK_evaluate (GDS_block_context,
-                             record->type,
-                             GNUNET_BLOCK_EO_NONE,
-                             key, NULL, 0,
-                             record->xquery,
-                             record->xquery_size,
-                             frc->data,
-                             frc->data_size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Evaluation result is %d for key %s for local client's query\n",
-       (int) eval, GNUNET_h2s (key));
-  switch (eval)
-  {
-  case GNUNET_BLOCK_EVALUATION_OK_LAST:
-    do_free = GNUNET_YES;
-    break;
-  case GNUNET_BLOCK_EVALUATION_OK_MORE:
-    GNUNET_array_append (record->seen_replies, record->seen_replies_count, ch);
-    do_free = GNUNET_NO;
-    break;
-  case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE:
-    /* should be impossible to encounter here */
-    GNUNET_break (0);
-    return GNUNET_YES;
-  case GNUNET_BLOCK_EVALUATION_RESULT_INVALID:
-    GNUNET_break_op (0);
-    return GNUNET_NO;
-  case GNUNET_BLOCK_EVALUATION_REQUEST_VALID:
-    GNUNET_break (0);
-    return GNUNET_NO;
-  case GNUNET_BLOCK_EVALUATION_REQUEST_INVALID:
-    GNUNET_break (0);
-    return GNUNET_NO;
-  case GNUNET_BLOCK_EVALUATION_RESULT_IRRELEVANT:
-    return GNUNET_YES;
-  case GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED:
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Unsupported block type (%u) in request!\n"), record->type);
-    return GNUNET_NO;
-  default:
-    GNUNET_break (0);
-    return GNUNET_NO;
-  }
-  if (GNUNET_NO == frc->do_copy)
-  {
-    /* first time, we can use the original data */
-    pm = frc->pm;
-    frc->do_copy = GNUNET_YES;
-  }
-  else
-  {
-    /* two clients waiting for same reply, must copy for queueing */
-    pm = GNUNET_malloc (sizeof (struct PendingMessage) +
-                        ntohs (frc->pm->msg->size));
-    GNUNET_memcpy (pm, frc->pm,
-            sizeof (struct PendingMessage) + ntohs (frc->pm->msg->size));
-    pm->next = pm->prev = NULL;
-    pm->msg = (struct GNUNET_MessageHeader *) &pm[1];
-  }
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# RESULTS queued for clients"), 1,
-                            GNUNET_NO);
-  reply = (struct GNUNET_DHT_ClientResultMessage *) &pm[1];
-  reply->unique_id = record->unique_id;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Queueing reply to query %s for client %p\n",
-       GNUNET_h2s (key),
-       record->client->client_handle);
-  add_pending_message (record->client, pm);
-  if (GNUNET_YES == do_free)
-    remove_client_records (record->client, key, record);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handle a reply we've received from another peer.  If the reply
- * matches any of our pending queries, forward it to the respective
- * client(s).
- *
- * @param expiration when will the reply expire
- * @param key the query this reply is for
- * @param get_path_length number of peers in @a get_path
- * @param get_path path the reply took on get
- * @param put_path_length number of peers in @a put_path
- * @param put_path path the reply took on put
- * @param type type of the reply
- * @param data_size number of bytes in @a data
- * @param data application payload data
- */
-void
-GDS_CLIENTS_handle_reply (struct GNUNET_TIME_Absolute expiration,
-                          const struct GNUNET_HashCode *key,
-                          unsigned int get_path_length,
-                          const struct GNUNET_PeerIdentity *get_path,
-                          unsigned int put_path_length,
-                          const struct GNUNET_PeerIdentity *put_path,
-                          enum GNUNET_BLOCK_Type type, size_t data_size,
-                          const void *data)
-{
-  struct ForwardReplyContext frc;
-  struct PendingMessage *pm;
-  struct GNUNET_DHT_ClientResultMessage *reply;
-  struct GNUNET_PeerIdentity *paths;
-  size_t msize;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "reply for key %s\n",
-       GNUNET_h2s (key));
-
-  if (NULL == GNUNET_CONTAINER_multihashmap_get (forward_map, key))
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop
-                              ("# REPLIES ignored for CLIENTS (no match)"), 1,
-                              GNUNET_NO);
-    return;                     /* no matching request, fast exit! */
-  }
-  msize =
-      sizeof (struct GNUNET_DHT_ClientResultMessage) + data_size +
-      (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Could not pass reply to client, message too big!\n"));
-    return;
-  }
-  DEBUG ("reply FOR DATA_SIZE = %u\n",
-         (unsigned int) msize);
-  pm = GNUNET_malloc (msize + sizeof (struct PendingMessage));
-  reply = (struct GNUNET_DHT_ClientResultMessage *) &pm[1];
-  pm->msg = &reply->header;
-  reply->header.size = htons ((uint16_t) msize);
-  reply->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CLIENT_RESULT);
-  reply->type = htonl (type);
-  reply->get_path_length = htonl (get_path_length);
-  reply->put_path_length = htonl (put_path_length);
-  reply->unique_id = 0;         /* filled in later */
-  reply->expiration = GNUNET_TIME_absolute_hton (expiration);
-  reply->key = *key;
-  paths = (struct GNUNET_PeerIdentity *) &reply[1];
-  GNUNET_memcpy (paths,
-                 put_path,
-                 sizeof (struct GNUNET_PeerIdentity) * put_path_length);
-  GNUNET_memcpy (&paths[put_path_length],
-                 get_path,
-                 sizeof (struct GNUNET_PeerIdentity) * get_path_length);
-  GNUNET_memcpy (&paths[get_path_length + put_path_length],
-                 data,
-                 data_size);
-  frc.do_copy = GNUNET_NO;
-  frc.pm = pm;
-  frc.data = data;
-  frc.data_size = data_size;
-  frc.type = type;
-  GNUNET_CONTAINER_multihashmap_get_multiple (forward_map,
-                                              key,
-                                              &forward_reply,
-                                              &frc);
-  if (GNUNET_NO == frc.do_copy)
-  {
-    /* did not match any of the requests, free! */
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop
-                              ("# REPLIES ignored for CLIENTS (no match)"), 1,
-                              GNUNET_NO);
-    GNUNET_free (pm);
-  }
-}
-
-
-/**
- * Check if some client is monitoring GET messages and notify
- * them in that case.
- *
- * @param options Options, for instance RecordRoute, DemultiplexEverywhere.
- * @param type The type of data in the request.
- * @param hop_count Hop count so far.
- * @param path_length number of entries in path (or 0 if not recorded).
- * @param path peers on the GET path (or NULL if not recorded).
- * @param desired_replication_level Desired replication level.
- * @param key Key of the requested data.
- */
-void
-GDS_CLIENTS_process_get (uint32_t options,
-                         enum GNUNET_BLOCK_Type type,
-                         uint32_t hop_count,
-                         uint32_t desired_replication_level,
-                         unsigned int path_length,
-                         const struct GNUNET_PeerIdentity *path,
-                         const struct GNUNET_HashCode *key)
-{
-  struct ClientMonitorRecord *m;
-  struct ClientList **cl;
-  unsigned int cl_size;
-
-  cl = NULL;
-  cl_size = 0;
-  for (m = monitor_head; NULL != m; m = m->next)
-  {
-    if ((GNUNET_BLOCK_TYPE_ANY == m->type || m->type == type) &&
-        (NULL == m->key ||
-         memcmp (key, m->key, sizeof(struct GNUNET_HashCode)) == 0))
-    {
-      struct PendingMessage *pm;
-      struct GNUNET_DHT_MonitorGetMessage *mmsg;
-      struct GNUNET_PeerIdentity *msg_path;
-      size_t msize;
-      unsigned int i;
-
-      /* Don't send duplicates */
-      for (i = 0; i < cl_size; i++)
-        if (cl[i] == m->client)
-          break;
-      if (i < cl_size)
-        continue;
-      GNUNET_array_append (cl, cl_size, m->client);
-
-      msize = path_length * sizeof (struct GNUNET_PeerIdentity);
-      msize += sizeof (struct GNUNET_DHT_MonitorGetMessage);
-      msize += sizeof (struct PendingMessage);
-      pm = GNUNET_malloc (msize);
-      mmsg = (struct GNUNET_DHT_MonitorGetMessage *) &pm[1];
-      pm->msg = &mmsg->header;
-      mmsg->header.size = htons (msize - sizeof (struct PendingMessage));
-      mmsg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_MONITOR_GET);
-      mmsg->options = htonl(options);
-      mmsg->type = htonl(type);
-      mmsg->hop_count = htonl(hop_count);
-      mmsg->desired_replication_level = htonl(desired_replication_level);
-      mmsg->get_path_length = htonl(path_length);
-      GNUNET_memcpy (&mmsg->key, key, sizeof (struct GNUNET_HashCode));
-      msg_path = (struct GNUNET_PeerIdentity *) &mmsg[1];
-      if (path_length > 0)
-        GNUNET_memcpy (msg_path, path,
-                path_length * sizeof (struct GNUNET_PeerIdentity));
-      add_pending_message (m->client, pm);
-    }
-  }
-  GNUNET_free_non_null (cl);
-}
-
-
-/**
- * Check if some client is monitoring PUT messages and notify
- * them in that case.
- *
- * @param options options, for instance RecordRoute, DemultiplexEverywhere.
- * @param type type of data in the request.
- * @param hop_count hop count so far.
- * @param path_length number of entries in @a path (or 0 if not recorded).
- * @param path peers on the PUT path (or NULL if not recorded).
- * @param desired_replication_level desired replication level.
- * @param exp expiration time of the data.
- * @param key key under which @a data is to be stored.
- * @param data pointer to the data carried.
- * @param size number of bytes in @a data.
- */
-void
-GDS_CLIENTS_process_put (uint32_t options,
-                         enum GNUNET_BLOCK_Type type,
-                         uint32_t hop_count,
-                         uint32_t desired_replication_level,
-                         unsigned int path_length,
-                         const struct GNUNET_PeerIdentity *path,
-                         struct GNUNET_TIME_Absolute exp,
-                         const struct GNUNET_HashCode *key,
-                         const void *data,
-                         size_t size)
-{
-  struct ClientMonitorRecord *m;
-  struct ClientList **cl;
-  unsigned int cl_size;
-
-  cl = NULL;
-  cl_size = 0;
-  for (m = monitor_head; NULL != m; m = m->next)
-  {
-    if ((GNUNET_BLOCK_TYPE_ANY == m->type || m->type == type) &&
-        (NULL == m->key ||
-         memcmp (key, m->key, sizeof(struct GNUNET_HashCode)) == 0))
-    {
-      struct PendingMessage *pm;
-      struct GNUNET_DHT_MonitorPutMessage *mmsg;
-      struct GNUNET_PeerIdentity *msg_path;
-      size_t msize;
-      unsigned int i;
-
-      /* Don't send duplicates */
-      for (i = 0; i < cl_size; i++)
-        if (cl[i] == m->client)
-          break;
-      if (i < cl_size)
-        continue;
-      GNUNET_array_append (cl, cl_size, m->client);
-
-      msize = size;
-      msize += path_length * sizeof (struct GNUNET_PeerIdentity);
-      msize += sizeof (struct GNUNET_DHT_MonitorPutMessage);
-      msize += sizeof (struct PendingMessage);
-      pm = GNUNET_malloc (msize);
-      mmsg = (struct GNUNET_DHT_MonitorPutMessage *) &pm[1];
-      pm->msg = (struct GNUNET_MessageHeader *) mmsg;
-      mmsg->header.size = htons (msize - sizeof (struct PendingMessage));
-      mmsg->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_MONITOR_PUT);
-      mmsg->options = htonl(options);
-      mmsg->type = htonl(type);
-      mmsg->hop_count = htonl(hop_count);
-      mmsg->desired_replication_level = htonl(desired_replication_level);
-      mmsg->put_path_length = htonl(path_length);
-      msg_path = (struct GNUNET_PeerIdentity *) &mmsg[1];
-      if (path_length > 0)
-      {
-        GNUNET_memcpy (msg_path, path,
-                path_length * sizeof (struct GNUNET_PeerIdentity));
-      }
-      mmsg->expiration_time = GNUNET_TIME_absolute_hton(exp);
-      GNUNET_memcpy (&mmsg->key, key, sizeof (struct GNUNET_HashCode));
-      if (size > 0)
-        GNUNET_memcpy (&msg_path[path_length], data, size);
-      add_pending_message (m->client, pm);
-    }
-  }
-  GNUNET_free_non_null (cl);
-}
-
-
-/**
- * Route the given request via the DHT.
- */
-static void
-transmit_request (struct ClientQueryRecord *cqr)
-{
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop
-                            ("# GET requests from clients injected"), 1,
-                            GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Initiating GET for %s, replication %u, already have %u replies\n",
-       GNUNET_h2s (&cqr->key),
-       cqr->replication,
-       cqr->seen_replies_count);
-
-  GDS_NEIGHBOURS_handle_get (&cqr->key, cqr->type, cqr->msg_options,
-                              cqr->replication);
-
-  /* exponential back-off for retries.
-   * max GNUNET_TIME_STD_EXPONENTIAL_BACKOFF_THRESHOLD (15 min) */
-  cqr->retry_frequency = GNUNET_TIME_STD_BACKOFF (cqr->retry_frequency);
-  cqr->retry_time = GNUNET_TIME_relative_to_absolute (cqr->retry_frequency);
-}
-
-
-/**
- * Task that looks at the 'retry_heap' and transmits all of the requests
- * on the heap that are ready for transmission.  Then re-schedules
- * itself (unless the heap is empty).
- *
- * @param cls unused
- */
-static void
-transmit_next_request_task (void *cls)
-{
-  struct ClientQueryRecord *cqr;
-  struct GNUNET_TIME_Relative delay;
-
-  retry_task = NULL;
-  while (NULL != (cqr = GNUNET_CONTAINER_heap_remove_root (retry_heap)))
-  {
-    cqr->hnode = NULL;
-    delay = GNUNET_TIME_absolute_get_remaining (cqr->retry_time);
-    if (delay.rel_value_us > 0)
-    {
-      cqr->hnode =
-          GNUNET_CONTAINER_heap_insert (retry_heap, cqr,
-                                        cqr->retry_time.abs_value_us);
-      retry_task =
-          GNUNET_SCHEDULER_add_delayed (delay, &transmit_next_request_task,
-                                        NULL);
-      return;
-    }
-    transmit_request (cqr);
-    cqr->hnode =
-        GNUNET_CONTAINER_heap_insert (retry_heap, cqr,
-                                      cqr->retry_time.abs_value_us);
-  }
-}
-
-
-/**
- * Handler for PUT messages.
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- */
-static void
-handle_dht_local_put (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_DHT_ClientPutMessage *put_msg;
-  struct PendingMessage *pm;
-  struct GNUNET_DHT_ClientPutConfirmationMessage *conf;
-  uint16_t size;
-
-  size = ntohs (message->size);
-  if (size < sizeof (struct GNUNET_DHT_ClientPutMessage))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop
-                            ("# PUT requests received from clients"), 1,
-                            GNUNET_NO);
-  put_msg = (const struct GNUNET_DHT_ClientPutMessage *) message;
-  LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG, "X-VINE DHT CLIENT-PUT %s\n",
-               GNUNET_h2s_full (&put_msg->key));
-  /* give to local clients */
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Handling local PUT of %u-bytes for query %s\n",
-       size - sizeof (struct GNUNET_DHT_ClientPutMessage),
-       GNUNET_h2s (&put_msg->key));
-  DEBUG("PUT doing put i = %s\n",GNUNET_h2s(&(put_msg->key)));
-  GDS_CLIENTS_handle_reply (GNUNET_TIME_absolute_ntoh (put_msg->expiration),
-                            &put_msg->key, 0, NULL, 0, NULL,
-                            ntohl (put_msg->type),
-                            size - sizeof (struct GNUNET_DHT_ClientPutMessage),
-                            &put_msg[1]);
-
-  GDS_NEIGHBOURS_handle_put (&put_msg->key,
-                              ntohl (put_msg->type), ntohl (put_msg->options),
-                              ntohl (put_msg->desired_replication_level),
-                              GNUNET_TIME_absolute_ntoh (put_msg->expiration),
-                              &put_msg[1],
-                              size - sizeof (struct GNUNET_DHT_ClientPutMessage));
-  pm = GNUNET_malloc (sizeof (struct PendingMessage) +
-                      sizeof (struct GNUNET_DHT_ClientPutConfirmationMessage));
-  conf = (struct GNUNET_DHT_ClientPutConfirmationMessage *) &pm[1];
-  conf->header.size = htons (sizeof (struct GNUNET_DHT_ClientPutConfirmationMessage));
-  conf->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT_OK);
-  conf->reserved = htonl (0);
-  conf->unique_id = put_msg->unique_id;
-  pm->msg = &conf->header;
-  add_pending_message (find_active_client (client), pm);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for DHT GET messages from the client.
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- */
-static void
-handle_dht_local_get (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_DHT_ClientGetMessage *get;
-  struct ClientQueryRecord *cqr;
-  size_t xquery_size;
-  const char *xquery;
-  uint16_t size;
-
-  size = ntohs (message->size);
-  if (size < sizeof (struct GNUNET_DHT_ClientGetMessage))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  xquery_size = size - sizeof (struct GNUNET_DHT_ClientGetMessage);
-  get = (const struct GNUNET_DHT_ClientGetMessage *) message;
-  xquery = (const char *) &get[1];
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop
-                            ("# GET requests received from clients"), 1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received GET request for %s from local client %p, xq: %.*s\n",
-       GNUNET_h2s (&get->key), client, xquery_size, xquery);
-
-  LOG_TRAFFIC (GNUNET_ERROR_TYPE_DEBUG, "X-VINE CLIENT-GET %s\n",
-               GNUNET_h2s_full (&get->key));
-
-
-  cqr = GNUNET_malloc (sizeof (struct ClientQueryRecord) + xquery_size);
-  cqr->key = get->key;
-  cqr->client = find_active_client (client);
-  cqr->xquery = (void *) &cqr[1];
-  GNUNET_memcpy (&cqr[1], xquery, xquery_size);
-  cqr->hnode = GNUNET_CONTAINER_heap_insert (retry_heap, cqr, 0);
-  cqr->retry_frequency = GNUNET_TIME_UNIT_SECONDS;
-  cqr->retry_time = GNUNET_TIME_absolute_get ();
-  cqr->unique_id = get->unique_id;
-  cqr->xquery_size = xquery_size;
-  cqr->replication = ntohl (get->desired_replication_level);
-  cqr->msg_options = ntohl (get->options);
-  cqr->type = ntohl (get->type);
-
-  // FIXME use cqr->key, set multihashmap create to GNUNET_YES
-  GNUNET_CONTAINER_multihashmap_put (forward_map, &get->key, cqr,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-  struct GNUNET_PeerIdentity my_identity;
-  my_identity = GDS_NEIGHBOURS_get_my_id();
-  GDS_CLIENTS_process_get (ntohl (get->options),
-                           ntohl (get->type),
-                           0,
-                           ntohl (get->desired_replication_level),
-                           1,
-                           &my_identity,
-                           &get->key);
-  /* start remote requests */
-  if (NULL != retry_task)
-    GNUNET_SCHEDULER_cancel (retry_task);
-  retry_task = GNUNET_SCHEDULER_add_now (&transmit_next_request_task, NULL);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Closure for #find_by_unique_id().
- */
-struct FindByUniqueIdContext
-{
-  /**
-   * Where to store the result, if found.
-   */
-  struct ClientQueryRecord *cqr;
-
-  /**
-   * Which ID are we looking for?
-   */
-  uint64_t unique_id;
-};
-
-
-/**
- * Function called for each existing DHT record for the given
- * query.  Checks if it matches the UID given in the closure
- * and if so returns the entry as a result.
- *
- * @param cls the search context
- * @param key query for the lookup (not used)
- * @param value the `struct ClientQueryRecord`
- * @return #GNUNET_YES to continue iteration (result not yet found)
- */
-static int
-find_by_unique_id (void *cls,
-                   const struct GNUNET_HashCode *key,
-                   void *value)
-{
-  struct FindByUniqueIdContext *fui_ctx = cls;
-  struct ClientQueryRecord *cqr = value;
-
-  if (cqr->unique_id != fui_ctx->unique_id)
-    return GNUNET_YES;
-  fui_ctx->cqr = cqr;
-  return GNUNET_NO;
-}
-
-
-/**
- * Handler for "GET result seen" messages from the client.
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- */
-static void
-handle_dht_local_get_result_seen (void *cls,
-                                  struct GNUNET_SERVER_Client *client,
-                                  const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_DHT_ClientGetResultSeenMessage *seen;
-  uint16_t size;
-  unsigned int hash_count;
-  unsigned int old_count;
-  const struct GNUNET_HashCode *hc;
-  struct FindByUniqueIdContext fui_ctx;
-  struct ClientQueryRecord *cqr;
-
-  size = ntohs (message->size);
-  if (size < sizeof (struct GNUNET_DHT_ClientGetResultSeenMessage))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  seen = (const struct GNUNET_DHT_ClientGetResultSeenMessage *) message;
-  hash_count = (size - sizeof (struct GNUNET_DHT_ClientGetResultSeenMessage)) / sizeof (struct GNUNET_HashCode);
-  if (size != sizeof (struct GNUNET_DHT_ClientGetResultSeenMessage) + hash_count * sizeof (struct GNUNET_HashCode))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  hc = (const struct GNUNET_HashCode*) &seen[1];
-  fui_ctx.unique_id = seen->unique_id;
-  fui_ctx.cqr = NULL;
-  GNUNET_CONTAINER_multihashmap_get_multiple (forward_map,
-                                              &seen->key,
-                                              &find_by_unique_id,
-                                              &fui_ctx);
-  if (NULL == (cqr = fui_ctx.cqr))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  /* finally, update 'seen' list */
-  old_count = cqr->seen_replies_count;
-  GNUNET_array_grow (cqr->seen_replies,
-                     cqr->seen_replies_count,
-                     cqr->seen_replies_count + hash_count);
-  GNUNET_memcpy (&cqr->seen_replies[old_count],
-          hc,
-          sizeof (struct GNUNET_HashCode) * hash_count);
-}
-
-
-/**
- * Closure for #remove_by_unique_id().
- */
-struct RemoveByUniqueIdContext
-{
-  /**
-   * Client that issued the removal request.
-   */
-  struct ClientList *client;
-
-  /**
-   * Unique ID of the request.
-   */
-  uint64_t unique_id;
-};
-
-
-/**
- * Iterator over hash map entries that frees all entries
- * that match the given client and unique ID.
- *
- * @param cls unique ID and client to search for in source routes
- * @param key current key code
- * @param value value in the hash map, a ClientQueryRecord
- * @return #GNUNET_YES (we should continue to iterate)
- */
-static int
-remove_by_unique_id (void *cls,
-                     const struct GNUNET_HashCode *key,
-                     void *value)
-{
-  const struct RemoveByUniqueIdContext *ctx = cls;
-  struct ClientQueryRecord *record = value;
-
-  if (record->unique_id != ctx->unique_id)
-    return GNUNET_YES;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Removing client %p's record for key %s (by unique id)\n",
-              ctx->client->client_handle, GNUNET_h2s (key));
-  return remove_client_records (ctx->client, key, record);
-}
-
-
-/**
- * Handler for any generic DHT stop messages, calls the appropriate handler
- * depending on message type (if processed locally)
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- *
- */
-static void
-handle_dht_local_get_stop (void *cls,
-                           struct GNUNET_SERVER_Client *client,
-                           const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_DHT_ClientGetStopMessage *dht_stop_msg =
-      (const struct GNUNET_DHT_ClientGetStopMessage *) message;
-  struct RemoveByUniqueIdContext ctx;
-
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop
-                            ("# GET STOP requests received from clients"), 1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received GET STOP request for %s from local client %p\n",
-       GNUNET_h2s (&dht_stop_msg->key),
-       client);
-  ctx.client = find_active_client (client);
-  ctx.unique_id = dht_stop_msg->unique_id;
-  GNUNET_CONTAINER_multihashmap_get_multiple (forward_map, &dht_stop_msg->key,
-                                              &remove_by_unique_id, &ctx);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for monitor start messages
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- *
- */
-static void
-handle_dht_local_monitor (void *cls,
-                          struct GNUNET_SERVER_Client *client,
-                          const struct GNUNET_MessageHeader *message)
-{
-  struct ClientMonitorRecord *r;
-  const struct GNUNET_DHT_MonitorStartStopMessage *msg;
-
-  msg = (struct GNUNET_DHT_MonitorStartStopMessage *) message;
-  r = GNUNET_new (struct ClientMonitorRecord);
-
-  r->client = find_active_client(client);
-  r->type = ntohl(msg->type);
-  r->get = ntohs(msg->get);
-  r->get_resp = ntohs(msg->get_resp);
-  r->put = ntohs(msg->put);
-  if (0 == ntohs(msg->filter_key))
-      r->key = NULL;
-  else
-  {
-    r->key = GNUNET_new (struct GNUNET_HashCode);
-    GNUNET_memcpy (r->key, &msg->key, sizeof (struct GNUNET_HashCode));
-  }
-  GNUNET_CONTAINER_DLL_insert (monitor_head, monitor_tail, r);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for monitor stop messages
- *
- * @param cls closure for the service
- * @param client the client we received this message from
- * @param message the actual message received
- */
-static void
-handle_dht_local_monitor_stop (void *cls,
-                               struct GNUNET_SERVER_Client *client,
-                               const struct GNUNET_MessageHeader *message)
-{
-  struct ClientMonitorRecord *r;
-  const struct GNUNET_DHT_MonitorStartStopMessage *msg;
-  int keys_match;
-
-  msg = (struct GNUNET_DHT_MonitorStartStopMessage *) message;
-  r = monitor_head;
-
-  while (NULL != r)
-  {
-    if (NULL == r->key)
-        keys_match = (0 == ntohs(msg->filter_key));
-    else
-    {
-        keys_match = (0 != ntohs(msg->filter_key)
-                      && !memcmp(r->key, &msg->key, sizeof(struct GNUNET_HashCode)));
-    }
-    if (find_active_client(client) == r->client
-        && ntohl(msg->type) == r->type
-        && r->get == msg->get
-        && r->get_resp == msg->get_resp
-        && r->put == msg->put
-        && keys_match
-        )
-    {
-        GNUNET_CONTAINER_DLL_remove (monitor_head, monitor_tail, r);
-        GNUNET_free_non_null (r->key);
-        GNUNET_free (r);
-        GNUNET_SERVER_receive_done (client, GNUNET_OK);
-        return; /* Delete only ONE entry */
-    }
-    r = r->next;
-  }
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure (NULL for dht)
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct ClientList *pos;
-  struct PendingMessage *reply;
-  struct ClientMonitorRecord *monitor;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Local client %p disconnects\n",
-              client);
-  pos = find_active_client (client);
-  GNUNET_CONTAINER_DLL_remove (client_head, client_tail, pos);
-  if (pos->transmit_handle != NULL)
-    GNUNET_SERVER_notify_transmit_ready_cancel (pos->transmit_handle);
-  while (NULL != (reply = pos->pending_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (pos->pending_head, pos->pending_tail, reply);
-    GNUNET_free (reply);
-  }
-  monitor = monitor_head;
-  while (NULL != monitor)
-  {
-    if (monitor->client == pos)
-    {
-      struct ClientMonitorRecord *next;
-
-      GNUNET_free_non_null (monitor->key);
-      next = monitor->next;
-      GNUNET_CONTAINER_DLL_remove (monitor_head, monitor_tail, monitor);
-      GNUNET_free (monitor);
-      monitor = next;
-    }
-    else
-      monitor = monitor->next;
-  }
-  GNUNET_CONTAINER_multihashmap_iterate (forward_map, &remove_client_records,
-                                         pos);
-  GNUNET_free (pos);
-}
-
-
-/**
- * Initialize client subsystem.
- *
- * @param server the initialized server
- */
-void
-GDS_CLIENTS_init (struct GNUNET_SERVER_Handle *server)
-{
-  static struct GNUNET_SERVER_MessageHandler plugin_handlers[] = {
-    {&handle_dht_local_put, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_CLIENT_PUT, 0},
-    {&handle_dht_local_get, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET, 0},
-    {&handle_dht_local_get_stop, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET_STOP,
-     sizeof (struct GNUNET_DHT_ClientGetStopMessage)},
-    {&handle_dht_local_monitor, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_MONITOR_START,
-     sizeof (struct GNUNET_DHT_MonitorStartStopMessage)},
-    {&handle_dht_local_monitor_stop, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_MONITOR_STOP,
-     sizeof (struct GNUNET_DHT_MonitorStartStopMessage)},
-    {&handle_dht_local_get_result_seen, NULL,
-     GNUNET_MESSAGE_TYPE_DHT_CLIENT_GET_RESULTS_KNOWN, 0},
-    {NULL, NULL, 0, 0}
-  };
-  forward_map = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
-  retry_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  GNUNET_SERVER_add_handlers (server, plugin_handlers);
-  GNUNET_SERVER_disconnect_notify (server, &handle_client_disconnect, NULL);
-}
-
-
-/**
- * Shutdown client subsystem.
- */
-void
-GDS_CLIENTS_done ()
-{
-  GNUNET_assert (client_head == NULL);
-  GNUNET_assert (client_tail == NULL);
-  if (NULL != retry_task)
-  {
-    GNUNET_SCHEDULER_cancel (retry_task);
-    retry_task = NULL;
-  }
-  if (NULL != retry_heap)
-  {
-    GNUNET_assert (0 == GNUNET_CONTAINER_heap_get_size (retry_heap));
-    GNUNET_CONTAINER_heap_destroy (retry_heap);
-    retry_heap = NULL;
-  }
-  if (NULL != forward_map)
-  {
-    GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (forward_map));
-    GNUNET_CONTAINER_multihashmap_destroy (forward_map);
-    forward_map = NULL;
-  }
-}
-
-/* end of gnunet-service-wdht_clients.c */
diff --git a/src/dht/gnunet-service-wdht_neighbours.c b/src/dht/gnunet-service-wdht_neighbours.c
deleted file mode 100644
index 78a04d66d..000000000
--- a/src/dht/gnunet-service-wdht_neighbours.c
+++ /dev/null
@@ -1,1768 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file dht/gnunet-service-wdht_neighbours.c
- * @brief GNUnet DHT service's finger and friend table management code
- * @author Supriti Singh
- * @author Christian Grothoff
- * @author Arthur Dewarumez
- *
- * TODO:
- * - initiate finding of successors
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_constants.h"
-#include "gnunet_protocols.h"
-#include "gnunet_ats_service.h"
-#include "gnunet_core_service.h"
-#include "gnunet_datacache_lib.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-dht.h"
-#include "gnunet-service-dht_datacache.h"
-#include "gnunet-service-dht_neighbours.h"
-#include "gnunet-service-dht_nse.h"
-#include "dht.h"
-
-#define DEBUG(...)                                           \
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
-
-/**
- * Trail timeout. After what time do trails always die?
- */
-#define TRAIL_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 42)
-
-/**
- * Random walk delay. How often do we walk the overlay?
- */
-#define RANDOM_WALK_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 42)
-
-/**
- * The number of layered ID to use.
- */
-#define NUMBER_LAYERED_ID 8
-
-/**
- * The number of random walk to launch at the beginning of the initialization
- */
-/* FIXME: find a better value */
-#define NUMBER_RANDOM_WALK 20
-
-
-/******************* The db structure and related functions *******************/
-
-/**
- * Entry in #friends_peermap.
- */
-struct FriendInfo;
-
-/**
- *
- */
-struct FingerTable;
-
-/**
- * Information we keep per trail.
- */
-struct Trail
-{
-
-  /**
-   * Identifier of the trail with the predecessor.
-   */
-  struct GNUNET_HashCode pred_id;
-
-  /**
-   * Identifier of the trail with the successor.
-   */
-  struct GNUNET_HashCode succ_id;
-
-  /**
-   * When does this trail expire.
-   */
-  struct GNUNET_TIME_Absolute expiration_time;
-
-  /**
-   * MDLL entry in the list of all trails with the same predecessor.
-   */
-  struct Trail *prev_succ;
-
-  /**
-   * MDLL entry in the list of all trails with the same predecessor.
-   */
-  struct Trail *next_succ;
-
-  /**
-   * MDLL entry in the list of all trails with the same predecessor.
-   */
-  struct Trail *prev_pred;
-
-  /**
-   * MDLL entry in the list of all trails with the same predecessor.
-   */
-  struct Trail *next_pred;
-
-  /**
-   * Our predecessor in the trail, NULL if we are initiator (?).
-   */
-  struct FriendInfo *pred;
-
-  /**
-   * Our successor in the trail, NULL if we are the last peer.
-   */
-  struct FriendInfo *succ;
-
-  /**
-   * Location of this trail in the heap.
-   */
-  struct GNUNET_CONTAINER_HeapNode *hn;
-
-  /**
-   * If this peer started the to create a Finger (and thus @e pred is
-   * NULL), this is the finger table of the finger we are trying to
-   * intialize.
-   */
-  struct FingerTable *ft;
-
-  /**
-   * If this peer started the trail to create a Finger (and thus @e
-   * pred is NULL), this is the offset of the finger we are trying to
-   * intialize in the unsorted array.
-   */
-  unsigned int finger_off;
-
-};
-
-
-/**
- *  Entry in #friends_peermap.
- */
-struct FriendInfo
-{
-  /**
-   * Friend Identity
-   */
-  const struct GNUNET_PeerIdentity *id;
-
-  /**
-   *
-   */
-  struct Trail *pred_head;
-
-  /**
-   *
-   */
-  struct Trail *pred_tail;
-
-  /**
-   *
-   */
-  struct Trail *succ_head;
-
-  /**
-   *
-   */
-  struct Trail *succ_tail;
-
-  /**
-   * Core handle for sending messages to this friend.
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-};
-
-
-/**
- *
- */
-struct Finger
-{
-  /**
-   *
-   */
-  struct Trail *trail;
-
-  /**
-   *
-   */
-  struct FingerTable *ft;
-
-  /**
-   *
-   */
-  struct GNUNET_HashCode destination;
-
-  /**
-   * #GNUNET_YES if a response has been received. Otherwise #GNUNET_NO.
-   */
-  int valid;
-};
-
-
-struct FingerTable
-{
-  /**
-   * Array of our fingers, unsorted.
-   */
-  struct Finger **fingers;
-
-  /**
-   * Size of the finger array.
-   */
-  unsigned int finger_array_size;
-
-  /**
-   * Number of valid entries in @e fingers
-   */
-  unsigned int number_valid_fingers;
-
-  /**
-   * Which offset in @e fingers will we redo next.
-   */
-  unsigned int walk_offset;
-
-  /**
-   * Is the finger array sorted?
-   */
-  int is_sorted;
-
-};
-
-
-/***********************  end of the db structure part  ***********************/
-
-
-GNUNET_NETWORK_STRUCT_BEGIN
-
-/**
- * Setup a finger using the underlay topology ("social network").
- */
-struct RandomWalkMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Number of hops this message has taken so far, we stop at
-   * log(NSE), in NBO.
-   */
-  uint16_t hops_taken GNUNET_PACKED;
-
-  /**
-   * Layer for the request, in NBO.
-   */
-  uint16_t layer GNUNET_PACKED;
-
-  /**
-   * Unique (random) identifier this peer will use to
-   * identify the trail (in future messages).
-   */
-  struct GNUNET_HashCode trail_id;
-
-};
-
-/**
- * Response to a `struct RandomWalkMessage`.
- */
-struct RandomWalkResponseMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK_RESPONSE
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Zero, for alignment.
-   */
-  uint32_t reserved GNUNET_PACKED;
-
-  /**
-   * Unique (random) identifier from the
-   * `struct RandomWalkMessage`.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Random location in the respective layer where the
-   * random path of the finger setup terminated.
-   */
-  struct GNUNET_HashCode location;
-
-};
-
-/**
- * Response to an event that causes a trail to die.
- */
-struct TrailDestroyMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_TRAIL_DESTROY
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Zero, for alignment.
-   */
-  uint32_t reserved GNUNET_PACKED;
-
-  /**
-   * Unique (random) identifier this peer will use to
-   * identify the finger (in future messages).
-   */
-  struct GNUNET_HashCode trail_id;
-
-};
-
-
-/**
- * Send a message along a trail.
- */
-struct FindSuccessorMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_SUCCESSOR_FIND
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Zero, for alignment.
-   */
-  uint32_t reserved GNUNET_PACKED;
-
-  /**
-   * Key for which we would like close values returned.
-   * identify the finger (in future messages).
-   */
-  struct GNUNET_HashCode key;
-
-};
-
-
-/**
- * Send a message along a trail.
- */
-struct TrailRouteMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_TRAIL_ROUTE
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * #GNUNET_YES if the path should be recorded, #GNUNET_NO if not; in NBO.
-   */
-  uint16_t record_path GNUNET_PACKED;
-
-  /**
-   * Length of the recorded trail, 0 if @e record_path is #GNUNET_NO; in NBO.
-   */
-  uint16_t path_length GNUNET_PACKED;
-
-  /**
-   * Unique (random) identifier this peer will use to
-   * identify the finger (in future messages).
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Path the message has taken so far (excluding sender).
-   */
-  /* struct GNUNET_PeerIdentity path[path_length]; */
-
-  /* followed by payload (another `struct GNUNET_MessageHeader`) to
-     send along the trail */
-};
-
-
-/**
- * P2P PUT message
- */
-struct PeerPutMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_PUT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Processing options
-   */
-  uint32_t options GNUNET_PACKED;
-
-  /**
-   * Content type.
-   */
-  uint32_t block_type GNUNET_PACKED;
-
-  /**
-   * Hop count
-   */
-  uint32_t hop_count GNUNET_PACKED;
-
-  /**
-   * Replication level for this message
-   * In the current implementation, this value is not used.
-   */
-  uint32_t desired_replication_level GNUNET_PACKED;
-
-  /**
-   * Length of the PUT path that follows (if tracked).
-   */
-  uint32_t put_path_length GNUNET_PACKED;
-
-  /**
-   * When does the content expire?
-   */
-  struct GNUNET_TIME_AbsoluteNBO expiration_time;
-
-  /**
-   * The key to store the value under.
-   */
-  struct GNUNET_HashCode key GNUNET_PACKED;
-
-  /* put path (if tracked) */
-
-  /* Payload */
-
-};
-
-/**
- * P2P GET message
- */
-struct PeerGetMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_GET
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Processing options
-   */
-  uint32_t options GNUNET_PACKED;
-
-  /**
-   * Desired content type.
-   */
-  uint32_t block_type GNUNET_PACKED;
-
-  /**
-   * Hop count
-   */
-  uint32_t hop_count GNUNET_PACKED;
-
-  /**
-   * Desired replication level for this request.
-   * In the current implementation, this value is not used.
-   */
-  uint32_t desired_replication_level GNUNET_PACKED;
-
-  /**
-   * Total number of peers in get path.
-   */
-  unsigned int get_path_length;
-
-  /**
-   * The key we are looking for.
-   */
-  struct GNUNET_HashCode key;
-
-  /* Get path. */
-  /* struct GNUNET_PeerIdentity[]*/
-};
-
-
-/**
- * P2P Result message
- */
-struct PeerGetResultMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_WDHT_GET_RESULT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * The type for the data in NBO.
-   */
-  uint32_t type GNUNET_PACKED;
-
-  /**
-   * Number of peers recorded in the outgoing path from source to the
-   * stored location of this message.
-   */
-  uint32_t put_path_length GNUNET_PACKED;
-
-  /**
-   * When does the content expire?
-   */
-  struct GNUNET_TIME_AbsoluteNBO expiration_time;
-
-  /**
-   * The key of the corresponding GET request.
-   */
-  struct GNUNET_HashCode key;
-
-  /* put path (if tracked) */
-
-  /* Payload */
-
-};
-
-GNUNET_NETWORK_STRUCT_END
-
-
-/**
- * Contains all the layered IDs of this peer.
- */
-struct GNUNET_PeerIdentity layered_id[NUMBER_LAYERED_ID];
-
-/**
- * Task to timeout trails that have expired.
- */
-static struct GNUNET_SCHEDULER_Task *trail_timeout_task;
-
-/**
- * Task to perform random walks.
- */
-static struct GNUNET_SCHEDULER_Task *random_walk_task;
-
-/**
- * Identity of this peer.
- */
-static struct GNUNET_PeerIdentity my_identity;
-
-/**
- * Peer map of all the friends of a peer
- */
-static struct GNUNET_CONTAINER_MultiPeerMap *friends_peermap;
-
-/**
- * Fingers per layer.
- */
-static struct FingerTable fingers[NUMBER_LAYERED_ID];
-
-/**
- * Tail map, mapping tail identifiers to `struct Trail`s
- */
-static struct GNUNET_CONTAINER_MultiHashMap *trail_map;
-
-/**
- * Tail heap, organizing trails by expiration time.
- */
-static struct GNUNET_CONTAINER_Heap *trail_heap;
-
-/**
- * Handle to CORE.
- */
-static struct GNUNET_CORE_Handle *core_api;
-
-
-/**
- * Handle the put request from the client.
- *
- * @param block_type Type of the block
- * @param options routing options
- * @param desired_replication_level desired replication level
- * @param expiration_time when does the content expire
- * @param hop_count how many hops has this message traversed so far
- * @param bf Bloom filter of peers this PUT has already traversed
- * @param key key for the content
- * @param put_path_length number of entries in put_path
- * @param put_path peers this request has traversed so far (if tracked)
- * @param data payload to store
- * @param data_size number of bytes in data
- * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
- */
-int
-GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type block_type,
-                           enum GNUNET_DHT_RouteOption options,
-                           uint32_t desired_replication_level,
-                           struct GNUNET_TIME_Absolute expiration_time,
-                           uint32_t hop_count,
-                           struct GNUNET_CONTAINER_BloomFilter *bf,
-                           const struct GNUNET_HashCode *key,
-                           unsigned int put_path_length,
-                           struct GNUNET_PeerIdentity *put_path,
-                           const void *data,
-                           size_t data_size)
-{
-  GDS_DATACACHE_handle_put (expiration_time,
-                            key,
-                            0, NULL,
-                            block_type,
-                            data_size,
-                            data);
-  GDS_CLIENTS_process_put (options,
-                           block_type,
-                           hop_count,
-                           desired_replication_level,
-                           put_path_length, put_path,
-                           expiration_time,
-                           key,
-                           data,
-                           data_size);
-  return GNUNET_OK; /* FIXME... */
-}
-
-
-/**
- * Perform a GET operation.  Forwards the given request to other
- * peers.  Does not lookup the key locally.  May do nothing if this is
- * the only peer in the network (or if we are the closest peer in the
- * network).
- *
- * @param type type of the block
- * @param options routing options
- * @param desired_replication_level desired replication count
- * @param hop_count how many hops did this request traverse so far?
- * @param key key for the content
- * @param xquery extended query
- * @param xquery_size number of bytes in @a xquery
- * @param reply_bf bloomfilter to filter duplicates
- * @param reply_bf_mutator mutator for @a reply_bf
- * @param peer_bf filter for peers not to select (again, updated)
- * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
- */
-int
-GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
-                           enum GNUNET_DHT_RouteOption options,
-                           uint32_t desired_replication_level,
-                           uint32_t hop_count,
-                           const struct GNUNET_HashCode *key,
-                           const void *xquery, size_t xquery_size,
-                           const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                           uint32_t reply_bf_mutator,
-                           struct GNUNET_CONTAINER_BloomFilter *peer_bf)
-{
-  // find closest finger(s) on all layers
-  // use TrailRoute with PeerGetMessage embedded to contact peer
-  // NOTE: actually more complicated, see paper!
-  GNUNET_break (0); // not implemented!
-  return GNUNET_SYSERR;
-}
-
-
-/**
- * Delete a trail, it died (timeout, link failure, etc.).
- *
- * @param trail trail to delete from all data structures
- * @param inform_pred should we notify the predecessor?
- * @param inform_succ should we inform the successor?
- */
-static void
-delete_trail (struct Trail *trail,
-              int inform_pred,
-              int inform_succ)
-{
-  struct FriendInfo *friend;
-  struct GNUNET_MQ_Envelope *env;
-  struct TrailDestroyMessage *tdm;
-  struct Finger *finger;
-
-  friend = trail->pred;
-  if (NULL != friend)
-  {
-    if (GNUNET_YES == inform_pred)
-    {
-      env = GNUNET_MQ_msg (tdm,
-                           GNUNET_MESSAGE_TYPE_WDHT_TRAIL_DESTROY);
-      tdm->trail_id = trail->pred_id;
-      GNUNET_MQ_send (friend->mq,
-                      env);
-    }
-    GNUNET_CONTAINER_MDLL_remove (pred,
-                                  friend->pred_head,
-                                  friend->pred_tail,
-                                  trail);
-  }
-  friend = trail->succ;
-  if (NULL != friend)
-  {
-    if (GNUNET_YES == inform_succ)
-    {
-      env = GNUNET_MQ_msg (tdm,
-                           GNUNET_MESSAGE_TYPE_WDHT_TRAIL_DESTROY);
-      tdm->trail_id = trail->pred_id;
-      GNUNET_MQ_send (friend->mq,
-                      env);
-    }
-    GNUNET_CONTAINER_MDLL_remove (succ,
-                                  friend->pred_head,
-                                  friend->pred_tail,
-                                  trail);
-  }
-  GNUNET_break (trail ==
-                GNUNET_CONTAINER_heap_remove_node (trail->hn));
-  finger = trail->ft->fingers[trail->finger_off];
-  if (NULL != finger)
-  {
-    trail->ft->fingers[trail->finger_off] = NULL;
-    trail->ft->number_valid_fingers--;
-    GNUNET_free (finger);
-  }
-  GNUNET_free (trail);
-}
-
-
-/**
- * Forward the given payload message along the trail.
- *
- * @param next_target which direction along the trail should we forward
- * @param trail_id which trail should we forward along
- * @param have_path do we track the forwarding path?
- * @param predecessor which peer do we tack on to the path?
- * @param path path the message has taken so far along the trail
- * @param path_length number of entries in @a path
- * @param payload payload of the message
- */
-static void
-forward_message_on_trail (struct FriendInfo *next_target,
-                          const struct GNUNET_HashCode *trail_id,
-                          int have_path,
-                          const struct GNUNET_PeerIdentity *predecessor,
-                          const struct GNUNET_PeerIdentity *path,
-                          uint16_t path_length,
-                          const struct GNUNET_MessageHeader *payload)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct TrailRouteMessage *trm;
-  struct GNUNET_PeerIdentity *new_path;
-  unsigned int plen;
-  uint16_t payload_len;
-
-  payload_len = ntohs (payload->size);
-  if (have_path)
-  {
-    plen = path_length + 1;
-    if (plen >= (GNUNET_SERVER_MAX_MESSAGE_SIZE
-                 - payload_len
-                 - sizeof (struct TrailRouteMessage))
-        / sizeof (struct GNUNET_PeerIdentity))
-    {
-      /* Should really not have paths this long... */
-      GNUNET_break_op (0);
-      plen = 0;
-      have_path = 0;
-    }
-  }
-  else
-  {
-    GNUNET_break_op (0 == path_length);
-    path_length = 0;
-    plen = 0;
-  }
-  env = GNUNET_MQ_msg_extra (trm,
-                             payload_len +
-                             plen * sizeof (struct GNUNET_PeerIdentity),
-                             GNUNET_MESSAGE_TYPE_WDHT_TRAIL_ROUTE);
-  trm->record_path = htons (have_path);
-  trm->path_length = htons (plen);
-  trm->trail_id = *trail_id;
-  new_path = (struct GNUNET_PeerIdentity *) &trm[1];
-  if (have_path)
-  {
-    GNUNET_memcpy (new_path,
-            path,
-            path_length * sizeof (struct GNUNET_PeerIdentity));
-    new_path[path_length] = *predecessor;
-  }
-  GNUNET_memcpy (&new_path[plen],
-          payload,
-          payload_len);
-  GNUNET_MQ_send (next_target->mq,
-                  env);
-}
-
-
-/**
- * Send the get result to requesting client.
- *
- * @param cls trail identifying where to send the result to, NULL for us
- * @param options routing options (from GET request)
- * @param key Key of the requested data.
- * @param type Block type
- * @param put_path_length Number of peers in @a put_path
- * @param put_path Path taken to put the data at its stored location.
- * @param expiration When will this result expire?
- * @param data Payload to store
- * @param data_size Size of the @a data
- */
-void
-GDS_NEIGHBOURS_send_get_result (void *cls,
-                                enum GNUNET_DHT_RouteOption options,
-                                const struct GNUNET_HashCode *key,
-                                enum GNUNET_BLOCK_Type type,
-                                unsigned int put_path_length,
-                                const struct GNUNET_PeerIdentity *put_path,
-                                struct GNUNET_TIME_Absolute expiration,
-                                const void *data,
-                                size_t data_size)
-{
-  const struct GNUNET_HashCode *trail_id = cls;
-  struct GNUNET_MessageHeader *payload;
-  struct Trail *trail;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (trail_map,
-                                             trail_id);
-  if (NULL == trail)
-  {
-    /* TODO: inform statistics */
-    return;
-  }
-  if (NULL == trail->pred)
-  {
-    /* result is for *us* (local client) */
-    GDS_CLIENTS_handle_reply (expiration,
-                              key,
-                              0, NULL,
-                              put_path_length, put_path,
-                              type,
-                              data_size,
-                              data);
-    return;
-  }
-
-  payload = GNUNET_malloc(sizeof(struct GNUNET_MessageHeader) + data_size);
-  payload->size = data_size;
-  payload->type = GNUNET_MESSAGE_TYPE_WDHT_GET_RESULT;
-
-  forward_message_on_trail (trail->pred,
-                            trail_id,
-                            0 != (options & GNUNET_DHT_RO_RECORD_ROUTE),
-                            &my_identity,
-                            NULL, 0,
-                            payload);
-  GNUNET_free (payload);
-}
-
-
-/**
- * Method called whenever a peer disconnects.
- *
- * @param cls closure
- * @param peer peer identity this notification is about
- * @param internal_cls our `struct FriendInfo` for @a peer
- */
-static void
-handle_core_disconnect (void *cls,
-                        const struct GNUNET_PeerIdentity *peer,
-                        void *internal_cls)
-{
-  struct FriendInfo *remove_friend = internal_cls;
-  struct Trail *t;
-
-  /* If disconnected to own identity, then return. */
-  if (NULL == remove_friend)
-    return;
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_remove (friends_peermap,
-                                                       peer,
-                                                       remove_friend));
-  while (NULL != (t = remove_friend->succ_head))
-    delete_trail (t,
-                  GNUNET_YES,
-                  GNUNET_NO);
-  while (NULL != (t = remove_friend->pred_head))
-    delete_trail (t,
-                  GNUNET_NO,
-                  GNUNET_YES);
-  GNUNET_free (remove_friend);
-  if (0 == GNUNET_CONTAINER_multipeermap_size (friends_peermap))
-  {
-    GNUNET_SCHEDULER_cancel (random_walk_task);
-    random_walk_task = NULL;
-  }
-}
-
-
-/**
- * Function called with a random friend to be returned.
- *
- * @param cls a `struct FriendInfo **` with where to store the result
- * @param peer the peer identity of the friend (ignored)
- * @param value the `struct FriendInfo *` that was selected at random
- * @return #GNUNET_OK (all good)
- */
-static int
-pick_random_helper (void *cls,
-                    const struct GNUNET_PeerIdentity *peer,
-                    void *value)
-{
-  struct FriendInfo **fi = cls;
-  struct FriendInfo *v = value;
-
-  *fi = v;
-  return GNUNET_OK;
-}
-
-
-/**
- * Pick random friend from friends for random walk.
- *
- * @return NULL if we have no friends
- */
-static struct FriendInfo *
-pick_random_friend ()
-{
-  struct FriendInfo *ret;
-
-  ret = NULL;
-  if (0 ==
-      GNUNET_CONTAINER_multipeermap_get_random (friends_peermap,
-                                                &pick_random_helper,
-                                                &ret))
-    return NULL;
-  return ret;
-}
-
-
-/**
- * One of our trails might have timed out, check and
- * possibly initiate cleanup.
- *
- * @param cls NULL
- */
-static void
-trail_timeout_callback (void *cls)
-{
-  struct Trail *trail;
-  struct GNUNET_TIME_Relative left;
-
-  trail_timeout_task = NULL;
-  while (NULL != (trail = GNUNET_CONTAINER_heap_peek (trail_heap)))
-  {
-    left = GNUNET_TIME_absolute_get_remaining (trail->expiration_time);
-    if (0 != left.rel_value_us)
-      break;
-    delete_trail (trail,
-                  GNUNET_YES,
-                  GNUNET_YES);
-  }
-  if (NULL != trail)
-    trail_timeout_task = GNUNET_SCHEDULER_add_delayed (left,
-                                                       &trail_timeout_callback,
-                                                       NULL);
-}
-
-
-/**
- * Compute how big our finger arrays should be (at least).
- *
- * @return size of the finger array, never 0
- */
-static unsigned int
-get_desired_finger_array_size ()
-{
-  /* FIXME: This is just a stub... */
-  return 64;
-}
-
-
-/**
- * Initiate a random walk.
- *
- * @param cls NULL
- */
-static void
-do_random_walk (void *cls)
-{
-  static unsigned int walk_layer;
-  struct FriendInfo *friend;
-  struct GNUNET_MQ_Envelope *env;
-  struct RandomWalkMessage *rwm;
-  struct FingerTable *ft;
-  struct Finger *finger;
-  struct Trail *trail;
-  unsigned int nsize;
-
-  random_walk_task = NULL;
-  friend = pick_random_friend ();
-
-  trail = GNUNET_new (struct Trail);
-  /* We create the random walk so, no predecessor */
-  trail->succ = friend;
-  GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_NONCE,
-                                    &trail->succ_id);
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (trail_map,
-                                         &trail->succ_id,
-                                         trail,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break (0);
-    GNUNET_free (trail);
-    return;
-  }
-  GNUNET_CONTAINER_MDLL_insert (succ,
-                                friend->succ_head,
-                                friend->succ_tail,
-                                trail);
-  trail->expiration_time = GNUNET_TIME_relative_to_absolute (TRAIL_TIMEOUT);
-  trail->hn = GNUNET_CONTAINER_heap_insert (trail_heap,
-                                            trail,
-                                            trail->expiration_time.abs_value_us);
-  if (NULL == trail_timeout_task)
-    trail_timeout_task = GNUNET_SCHEDULER_add_delayed (TRAIL_TIMEOUT,
-                                                       &trail_timeout_callback,
-                                                       NULL);
-  env = GNUNET_MQ_msg (rwm,
-                       GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK);
-  rwm->hops_taken = htonl (0);
-  rwm->trail_id = trail->succ_id;
-  GNUNET_MQ_send (friend->mq,
-                  env);
-  /* clean up 'old' entry (implicitly via trail cleanup) */
-  ft = &fingers[walk_layer];
-
-  if ( (NULL != ft->fingers) &&
-       (NULL != (finger = ft->fingers[ft->walk_offset])) )
-    delete_trail (finger->trail,
-                  GNUNET_NO,
-                  GNUNET_YES);
-  if (ft->finger_array_size < (nsize = get_desired_finger_array_size()) )
-    GNUNET_array_grow (ft->fingers,
-                       ft->finger_array_size,
-                       nsize);
-  GNUNET_assert (NULL == ft->fingers[ft->walk_offset]);
-  trail->ft = ft;
-  trail->finger_off = ft->walk_offset;
-  finger = GNUNET_new (struct Finger);
-  finger->trail = trail;
-  finger->ft = ft;
-  ft->fingers[ft->walk_offset] = finger;
-  ft->is_sorted = GNUNET_NO;
-  ft->number_valid_fingers++;
-  ft->walk_offset = (ft->walk_offset + 1) % ft->finger_array_size;
-
-  walk_layer = (walk_layer + 1) % NUMBER_LAYERED_ID;
-  random_walk_task = GNUNET_SCHEDULER_add_delayed (RANDOM_WALK_DELAY,
-                                                   &do_random_walk,
-                                                   NULL);
-}
-
-
-/**
- * Method called whenever a peer connects.
- *
- * @param cls closure
- * @param peer_identity peer identity this notification is about
- * @param mq message queue for transmission to @a peer_identity
- * @return the `struct FriendInfo` for the @a peer_identity, NULL for us
- */
-static void *
-handle_core_connect (void *cls,
-                     const struct GNUNET_PeerIdentity *peer_identity,
-                     struct GNUNET_MQ_Handle *mq)
-{
-  struct FriendInfo *friend;
-
-  /* Check for connect to self message */
-  if (0 == memcmp (&my_identity,
-                   peer_identity,
-                   sizeof (struct GNUNET_PeerIdentity)))
-    return NULL;
-
-  friend = GNUNET_new (struct FriendInfo);
-  friend->id = peer_identity;
-  friend->mq = mq;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multipeermap_put (friends_peermap,
-                                                    peer_identity,
-                                                    friend,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  if (NULL == random_walk_task)
-  {
-    /* random walk needs to be started -- we have a first connection */
-    random_walk_task = GNUNET_SCHEDULER_add_now (&do_random_walk,
-                                                 NULL);
-  }
-  return friend;
-}
-
-
-/**
- * To be called on core init/fail.
- *
- * @param cls service closure
- * @param identity the public identity of this peer
- */
-static void
-core_init (void *cls,
-           const struct GNUNET_PeerIdentity *identity)
-{
-  my_identity = *identity;
-}
-
-
-/**
- * Handle a `struct RandomWalkMessage` from a
- * #GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK message.
- *
- * @param cls the `struct FriendInfo` for the sender
- * @param m the setup message
- */
-static void
-handle_dht_p2p_random_walk (void *cls,
-                            const struct RandomWalkMessage *m)
-{
-  struct FriendInfo *pred = cls;
-  struct Trail *t;
-  uint16_t layer;
-
-  layer = ntohs (m->layer);
-  if (layer > NUMBER_LAYERED_ID)
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  t = GNUNET_new (struct Trail);
-  t->pred_id = m->trail_id;
-  t->pred = pred;
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (trail_map,
-                                         &t->pred_id,
-                                         t,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break_op (0);
-    GNUNET_free (t);
-    return;
-  }
-  GNUNET_CONTAINER_MDLL_insert (pred,
-                                pred->pred_head,
-                                pred->pred_tail,
-                                t);
-  t->expiration_time = GNUNET_TIME_relative_to_absolute (TRAIL_TIMEOUT);
-  t->hn = GNUNET_CONTAINER_heap_insert (trail_heap,
-                                        t,
-                                        t->expiration_time.abs_value_us);
-  if (NULL == trail_timeout_task)
-    trail_timeout_task = GNUNET_SCHEDULER_add_delayed (TRAIL_TIMEOUT,
-                                                       &trail_timeout_callback,
-                                                       NULL);
-
-  if (ntohl (m->hops_taken) > GDS_NSE_get ())
-  {
-    /* We are the last hop, generate response */
-    struct GNUNET_MQ_Envelope *env;
-    struct RandomWalkResponseMessage *rwrm;
-
-    env = GNUNET_MQ_msg (rwrm,
-                         GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK_RESPONSE);
-    rwrm->reserved = htonl (0);
-    rwrm->trail_id = m->trail_id;
-    if (0 == layer)
-      (void) GDS_DATACACHE_get_random_key (&rwrm->location);
-    else
-    {
-      struct FingerTable *ft;
-
-      ft = &fingers[layer-1];
-      if (0 == ft->number_valid_fingers)
-      {
-        GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_NONCE,
-                                          &rwrm->location);
-      }
-      else
-      {
-        struct Finger *f;
-        unsigned int off;
-        unsigned int i;
-
-        off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                        ft->number_valid_fingers);
-        for (i=0; (NULL == (f = ft->fingers[i])) || (off > 0); i++)
-          if (NULL != f) off--;
-        rwrm->location = f->destination;
-      }
-    }
-    GNUNET_MQ_send (pred->mq,
-                    env);
-  }
-  else
-  {
-    struct GNUNET_MQ_Envelope *env;
-    struct RandomWalkMessage *rwm;
-    struct FriendInfo *succ;
-
-    /* extend the trail by another random hop */
-    succ = pick_random_friend ();
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_NONCE,
-                                      &t->succ_id);
-    t->succ = succ;
-    if (GNUNET_OK !=
-        GNUNET_CONTAINER_multihashmap_put (trail_map,
-                                           &t->succ_id,
-                                           t,
-                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-    {
-      GNUNET_break (0);
-      GNUNET_CONTAINER_MDLL_remove (pred,
-                                    pred->pred_head,
-                                    pred->pred_tail,
-                                    t);
-      GNUNET_free (t);
-      return;
-    }
-    GNUNET_CONTAINER_MDLL_insert (succ,
-                                  succ->succ_head,
-                                  succ->succ_tail,
-                                  t);
-    env = GNUNET_MQ_msg (rwm,
-                         GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK);
-    rwm->hops_taken = htons (1 + ntohs (m->hops_taken));
-    rwm->layer = m->layer;
-    rwm->trail_id = t->succ_id;
-    GNUNET_MQ_send (succ->mq,
-                    env);
-  }
-}
-
-
-/**
- * Handle a `struct RandomWalkResponseMessage`.
- *
- * @param cls closure
- * @param rwrm the setup response message
- */
-static void
-handle_dht_p2p_random_walk_response (void *cls,
-                                     const struct RandomWalkResponseMessage *rwrm)
-{
-  struct Trail *trail;
-  struct FriendInfo *pred;
-  struct FingerTable *ft;
-  struct Finger *finger;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (trail_map,
-                                             &rwrm->trail_id);
-  if (NULL == trail)
-  {
-    /* TODO: log/statistics: we didn't find the trail (can happen) */
-    return;
-  }
-  if (NULL != (pred = trail->pred))
-  {
-    /* We are not the first hop, keep forwarding */
-    struct GNUNET_MQ_Envelope *env;
-    struct RandomWalkResponseMessage *rwrm2;
-
-    env = GNUNET_MQ_msg (rwrm2,
-                         GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK_RESPONSE);
-    rwrm2->reserved = htonl (0);
-    rwrm2->location = rwrm->location;
-    rwrm2->trail_id = trail->pred_id;
-    GNUNET_MQ_send (pred->mq,
-                    env);
-    return;
-  }
-  /* We are the first hop, complete finger */
-  if (NULL == (ft = trail->ft))
-  {
-    /* Eh, why did we create the trail if we have no FT? */
-    GNUNET_break (0);
-    delete_trail (trail,
-                  GNUNET_NO,
-                  GNUNET_YES);
-    return;
-  }
-  if (NULL == (finger = ft->fingers[trail->finger_off]))
-  {
-    /* Eh, finger got deleted, but why not the trail as well? */
-    GNUNET_break (0);
-    delete_trail (trail,
-                  GNUNET_NO,
-                  GNUNET_YES);
-    return;
-  }
-
-
-  // 1) lookup trail => find Finger entry => fill in 'destination' and mark valid, move to end of sorted array,
-  //mark unsorted, update links from 'trails'
-  /*
-   * Steps :
-   *  1 check if we are the correct layer
-   *  1.a if true : add the returned value (finger) in the db structure
-   *  1.b if true : do nothing
-   */
-  /* FIXME: add the value in db structure 1.a */
-
-}
-
-
-/**
- * Handle a `struct TrailDestroyMessage`.
- *
- * @param cls closure
- * @param tdm the trail destroy message
- */
-static void
-handle_dht_p2p_trail_destroy (void *cls,
-                              const struct TrailDestroyMessage *tdm)
-{
-  struct FriendInfo *sender = cls;
-  struct Trail *trail;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (trail_map,
-                                             &tdm->trail_id);
-  delete_trail (trail,
-                ( (NULL != trail->succ) &&
-                  (0 == memcmp (sender->id,
-                                &trail->succ->id,
-                                sizeof (struct GNUNET_PeerIdentity))) ),
-                ( (NULL != trail->pred) &&
-                  (0 == memcmp (sender->id,
-                                &trail->pred->id,
-                                sizeof (struct GNUNET_PeerIdentity))) ));
-}
-
-
-/**
- * Handle a `struct FindSuccessorMessage` from a #GNUNET_MESSAGE_TYPE_WDHT_SUCCESSOR_FIND
- * message.
- *
- * @param cls closure (NULL)
- * @param trail_id path to the originator
- * @param trail_path path the message took on the trail, if available
- * @param trail_path_length number of entries on the @a trail_path
- * @param message the finger setup message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-handle_dht_p2p_successor_find (void *cls,
-                               const struct GNUNET_HashCode *trail_id,
-                               const struct GNUNET_PeerIdentity *trail_path,
-                               unsigned int trail_path_length,
-                               const struct GNUNET_MessageHeader *message)
-{
-  const struct FindSuccessorMessage *fsm;
-
-  /* We do not expect to track trails for the forward-direction
-     of successor finding... */
-  GNUNET_break_op (0 == trail_path_length);
-  fsm = (const struct FindSuccessorMessage *) message;
-  GDS_DATACACHE_get_successors (&fsm->key,
-                                &GDS_NEIGHBOURS_send_get_result,
-                                (void *) trail_id);
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a `struct PeerGetMessage`.
- *
- * @param cls closure (NULL)
- * @param trail_id path to the originator
- * @param trail_path path the message took on the trail, if available
- * @param trail_path_length number of entries on the @a trail_path
- * @param message the peer get message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-handle_dht_p2p_peer_get (void *cls,
-                         const struct GNUNET_HashCode *trail_id,
-                         const struct GNUNET_PeerIdentity *trail_path,
-                         unsigned int trail_path_length,
-                         const struct GNUNET_MessageHeader *message)
-{
-#if 0
-  const struct PeerGetMessage *pgm;
-
-  // FIXME: note: never called like this, message embedded with trail route!
-  pgm = (const struct PeerGetMessage *) message;
-#endif
-  // -> lookup in datacache (figure out way to remember trail!)
-     /*
-    * steps :
-    *   1 extract the result
-    *   2 save the peer
-    *   3 send it using the good trail
-    *
-    * What do i do when i don't have the key/value?
-    */
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a `struct PeerGetResultMessage`.
- *
- * @param cls closure (NULL)
- * @param trail_id path to the originator
- * @param trail_path path the message took on the trail, if available
- * @param trail_path_length number of entries on the @a trail_path
- * @param message the peer get result message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-handle_dht_p2p_peer_get_result (void *cls,
-                                const struct GNUNET_HashCode *trail_id,
-                                const struct GNUNET_PeerIdentity *trail_path,
-                                unsigned int trail_path_length,
-                                const struct GNUNET_MessageHeader *message)
-{
-#if 0
-  const struct PeerGetResultMessage *pgrm;
-
-  pgrm = (const struct PeerGetResultMessage *) message;
-#endif
-  // pretty much: parse, & pass to client (there is some call for that...)
-
-#if 0
-  GDS_CLIENTS_process_get (options,
-                           type,
-                           0, 0,
-                           path_length, path,
-                           key);
-  (void) GDS_DATACACHE_handle_get (trail_id,
-                                   key,
-                                   type,
-                                   xquery,
-                                   xquery_size,
-                                   &reply_bf,
-                                   reply_bf_mutator);
-#endif
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a `struct PeerPutMessage`.
- *
- * @param cls closure (NULL)
- * @param trail_id path to the originator
- * @param trail_path path the message took on the trail, if available
- * @param trail_path_length number of entries on the @a trail_path
- * @param message the peer put message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-handle_dht_p2p_peer_put (void *cls,
-                         const struct GNUNET_HashCode *trail_id,
-                         const struct GNUNET_PeerIdentity *trail_path,
-                         unsigned int trail_path_length,
-                         const struct GNUNET_MessageHeader *message)
-{
-#if 0
-  const struct PeerGetResultMessage *pgrm;
-
-  pgrm = (const struct PeerGetResultMessage *) message;
-#endif
-  // parse & store in datacache, this is in response to us asking for successors.
-  /*
-   * steps :
-   * 1 check the size of the message
-   * 2 use the API to add the value in the "database". Check on the xdht file, how to do it.
-   * 3 Did i a have to return a notification or did i have to return GNUNET_[OK|SYSERR]?
-   */
-#if 0
-  GDS_DATACACHE_handle_put (expiration_time,
-                            key,
-                            combined_path_length, combined_path,
-                            block_type,
-                            data_size,
-                            data);
-  GDS_CLIENTS_process_put (options,
-                           block_type,
-                           0, 0,
-                           combined_path_length, combined_path,
-                           expiration_time,
-                           key,
-                           data,
-                           data_size);
-#endif
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for a message we received along some trail.
- *
- * @param cls closure
- * @param trail_id trail identifier
- * @param trail_path path the message took on the trail, if available
- * @param trail_path_length number of entries on the @a trail_path
- * @param message the message we got
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-typedef int
-(*TrailHandlerCallback)(void *cls,
-                        const struct GNUNET_HashCode *trail_id,
-                        const struct GNUNET_PeerIdentity *trail_path,
-                        unsigned int trail_path_length,
-                        const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Definition of a handler for a message received along some trail.
- */
-struct TrailHandler
-{
-  /**
-   * NULL for end-of-list.
-   */
-  TrailHandlerCallback callback;
-
-  /**
-   * Closure for @e callback.
-   */
-  void *cls;
-
-  /**
-   * Message type this handler addresses.
-   */
-  uint16_t message_type;
-
-  /**
-   * Use 0 for variable-size.
-   */
-  uint16_t message_size;
-};
-
-
-/**
- * Check that a `struct TrailRouteMessage` is well-formed.
- *
- * @param cls closure
- * @param trm the finger destroy message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-check_dht_p2p_trail_route (void *cls,
-                           const struct TrailRouteMessage *trm)
-{
-  const struct GNUNET_PeerIdentity *path;
-  uint16_t path_length;
-  const struct GNUNET_MessageHeader *payload;
-  size_t msize;
-
-  msize = ntohs (trm->header.size);
-  path_length = ntohs (trm->path_length);
-  if (msize < sizeof (struct TrailRouteMessage) +
-      path_length * sizeof (struct GNUNET_PeerIdentity) +
-      sizeof (struct GNUNET_MessageHeader) )
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  path = (const struct GNUNET_PeerIdentity *) &trm[1];
-  payload = (const struct GNUNET_MessageHeader *) &path[path_length];
-  if (msize != (ntohs (payload->size) +
-                sizeof (struct TrailRouteMessage) +
-                path_length * sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  /* FIXME: verify payload is OK!? */
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a `struct TrailRouteMessage`.
- *
- * @param cls closure
- * @param trm the finger destroy message
- */
-static void
-handle_dht_p2p_trail_route (void *cls,
-                            const struct TrailRouteMessage *trm)
-{
-  static const struct TrailHandler handlers[] = {
-    { &handle_dht_p2p_successor_find, NULL,
-      GNUNET_MESSAGE_TYPE_WDHT_SUCCESSOR_FIND,
-      sizeof (struct FindSuccessorMessage) },
-    { &handle_dht_p2p_peer_get, NULL,
-      GNUNET_MESSAGE_TYPE_WDHT_GET,
-      0 },
-    { &handle_dht_p2p_peer_get_result, NULL,
-      GNUNET_MESSAGE_TYPE_WDHT_GET_RESULT,
-      0 },
-    { &handle_dht_p2p_peer_put, NULL,
-      GNUNET_MESSAGE_TYPE_WDHT_PUT,
-      0 },
-    { NULL, NULL, 0, 0 }
-  };
-  struct FriendInfo *sender = cls;
-  unsigned int i;
-  const struct GNUNET_PeerIdentity *path;
-  uint16_t path_length;
-  const struct GNUNET_MessageHeader *payload;
-  const struct TrailHandler *th;
-  struct Trail *trail;
-
-  path_length = ntohs (trm->path_length);
-  path = (const struct GNUNET_PeerIdentity *) &trm[1];
-  payload = (const struct GNUNET_MessageHeader *) &path[path_length];
-  /* Is this message for us? */
-  trail = GNUNET_CONTAINER_multihashmap_get (trail_map,
-                                             &trm->trail_id);
-  if ( (NULL != trail->pred) &&
-       (0 == memcmp (sender->id,
-                     &trail->pred->id,
-                     sizeof (struct GNUNET_PeerIdentity))) )
-  {
-    /* forward to 'successor' */
-    if (NULL != trail->succ)
-    {
-      forward_message_on_trail (trail->succ,
-                                &trail->succ_id,
-                                ntohs (trm->record_path),
-                                sender->id,
-                                path,
-                                path_length,
-                                payload);
-      return;
-    }
-  }
-  else
-  {
-    /* forward to 'predecessor' */
-    GNUNET_break_op ( (NULL != trail->succ) &&
-                      (0 == memcmp (sender->id,
-                                    &trail->succ->id,
-                                    sizeof (struct GNUNET_PeerIdentity))) );
-    if (NULL != trail->pred)
-    {
-      forward_message_on_trail (trail->pred,
-                                &trail->pred_id,
-                                ntohs (trm->record_path),
-                                sender->id,
-                                path,
-                                path_length,
-                                payload);
-      return;
-    }
-  }
-
-  /* Message is for us, dispatch to handler */
-  th = NULL;
-  for (i=0; NULL != handlers[i].callback; i++)
-  {
-    th = &handlers[i];
-    if (ntohs (payload->type) == th->message_type)
-    {
-      if ( (0 == th->message_size) ||
-           (ntohs (payload->size) == th->message_size) )
-        th->callback (th->cls,
-                      &trm->trail_id,
-                      path,
-                      path_length,
-                      payload);
-      else
-        GNUNET_break_op (0);
-      break;
-    }
-  }
-  GNUNET_break_op (NULL != th);
-}
-
-
-/**
- * Initialize neighbours subsystem.
- *
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-int
-GDS_NEIGHBOURS_init (void)
-{
-  struct GNUNET_MQ_MessageHandler core_handlers[] = {
-    GNUNET_MQ_hd_fixed_size (dht_p2p_random_walk,
-                             GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK,
-                             struct RandomWalkMessage,
-                             NULL),
-    GNUNET_MQ_hd_fixed_size (dht_p2p_random_walk_response,
-                             GNUNET_MESSAGE_TYPE_WDHT_RANDOM_WALK_RESPONSE,
-                             struct RandomWalkResponseMessage,
-                             NULL),
-    GNUNET_MQ_hd_fixed_size (dht_p2p_trail_destroy,
-                             GNUNET_MESSAGE_TYPE_WDHT_TRAIL_DESTROY,
-                             struct TrailDestroyMessage,
-                             NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_trail_route,
-                           GNUNET_MESSAGE_TYPE_WDHT_TRAIL_ROUTE,
-                           struct TrailRouteMessage,
-                           NULL),
-    GNUNET_MQ_handler_end ()
-  };
-
-  core_api = GNUNET_CORE_connect (GDS_cfg, NULL,
-                                  &core_init,
-                                  &handle_core_connect,
-                                  &handle_core_disconnect,
-                                  core_handlers);
-  if (NULL == core_api)
-    return GNUNET_SYSERR;
-  friends_peermap = GNUNET_CONTAINER_multipeermap_create (256,
-                                                          GNUNET_NO);
-  trail_map = GNUNET_CONTAINER_multihashmap_create (1024,
-                                                    GNUNET_YES);
-  trail_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  return GNUNET_OK;
-}
-
-
-/**
- * Shutdown neighbours subsystem.
- */
-void
-GDS_NEIGHBOURS_done (void)
-{
-  if (NULL == core_api)
-    return;
-  GNUNET_CORE_disconnect (core_api);
-  core_api = NULL;
-  GNUNET_assert (0 == GNUNET_CONTAINER_multipeermap_size (friends_peermap));
-  GNUNET_CONTAINER_multipeermap_destroy (friends_peermap);
-  friends_peermap = NULL;
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (trail_map));
-  GNUNET_CONTAINER_multihashmap_destroy (trail_map);
-  trail_map = NULL;
-  GNUNET_CONTAINER_heap_destroy (trail_heap);
-  trail_heap = NULL;
-  if (NULL != trail_timeout_task)
-  {
-    GNUNET_SCHEDULER_cancel (trail_timeout_task);
-    trail_timeout_task = NULL;
-  }
-}
-
-
-/**
- * Get my identity
- *
- * @return my identity
- */
-struct GNUNET_PeerIdentity *
-GDS_NEIGHBOURS_get_id (void)
-{
-  return &my_identity;
-}
-
-/* end of gnunet-service-wdht_neighbours.c */
diff --git a/src/dht/gnunet-service-xdht.c b/src/dht/gnunet-service-xdht.c
deleted file mode 100644
index 7d0cb8a3a..000000000
--- a/src/dht/gnunet-service-xdht.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file dht/gnunet-service-xdht.c
- * @brief GNUnet DHT service
- * @author Christian Grothoff
- * @author Nathan Evans
- */
-#include "platform.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-dht.h"
-#include "gnunet-service-dht_datacache.h"
-#include "gnunet-service-dht_neighbours.h"
-#include "gnunet-service-dht_nse.h"
-#include "gnunet-service-xdht_routing.h"
-
-
-/**
- * Should we store our topology predecessor and successor IDs into statistics?
- */
-extern unsigned int track_topology;
-
-
-/* Code shared between different DHT implementations */
-#include "gnunet-service-dht_clients.c"
-
-
-/**
- * Task run during shutdown.
- *
- * @param cls unused
- */
-static void
-shutdown_task (void *cls)
-{
-  GDS_NEIGHBOURS_done ();
-  GDS_DATACACHE_done ();
-  GDS_ROUTING_done ();
-  GDS_NSE_done ();
-  if (NULL != GDS_block_context)
-  {
-    GNUNET_BLOCK_context_destroy (GDS_block_context);
-    GDS_block_context = NULL;
-  }
-  if (NULL != GDS_stats)
-  {
-    GNUNET_STATISTICS_destroy (GDS_stats, GNUNET_YES);
-    GDS_stats = NULL;
-  }
-  GDS_CLIENTS_stop ();
-}
-
-
-/**
- * Process dht requests.
- *
- * @param cls closure
- * @param c configuration to use
- * @param service the initialized service
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *c,
-     struct GNUNET_SERVICE_Handle *service)
-{
-  unsigned long long _track_topology;
-
-  GDS_cfg = c;
-  GDS_service = service;
-  GDS_block_context = GNUNET_BLOCK_context_create (GDS_cfg);
-  GDS_stats = GNUNET_STATISTICS_create ("dht",
-                                        GDS_cfg);
-  GDS_ROUTING_init ();
-  GDS_NSE_init ();
-  GDS_DATACACHE_init ();
-  GDS_CLIENTS_init ();
-  if (GNUNET_OK ==
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "xdht",
-                                             "track_toplogy",
-                                             &_track_topology))
-  {
-    track_topology = (unsigned int) _track_topology;
-  }
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  if (GNUNET_OK != GDS_NEIGHBOURS_init ())
-  {
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-}
-
-
-/* Finally, define the main method */
-GDS_DHT_SERVICE_INIT("xdht", &run);
-
-
-/* end of gnunet-service-xdht.c */
diff --git a/src/dht/gnunet-service-xdht.h b/src/dht/gnunet-service-xdht.h
deleted file mode 100644
index 5a8e2e21d..000000000
--- a/src/dht/gnunet-service-xdht.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht.h
- * @brief GNUnet DHT globals
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_XDHT_H
-#define GNUNET_SERVICE_XDHT_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet_transport_service.h"
-
-#define DEBUG_DHT GNUNET_EXTRA_LOGGING
-
-/**
- * Configuration we use.
- */
-extern const struct GNUNET_CONFIGURATION_Handle *GDS_cfg;
-
-/**
- * Our handle to the BLOCK library.
- */
-extern struct GNUNET_BLOCK_Context *GDS_block_context;
-
-/**
- * Handle for the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *GDS_stats;
-
-#endif
diff --git a/src/dht/gnunet-service-xdht_hello.c b/src/dht/gnunet-service-xdht_hello.c
deleted file mode 100644
index ceaf6f853..000000000
--- a/src/dht/gnunet-service-xdht_hello.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_hello.c
- * @brief GNUnet DHT integration with peerinfo
- * @author Christian Grothoff
- *
- * TODO:
- * - consider adding mechanism to remove expired HELLOs
- */
-#include "platform.h"
-#include "gnunet-service-xdht.h"
-#include "gnunet-service-xdht_hello.h"
-#include "gnunet_peerinfo_service.h"
-
-
-/**
- * Handle for peerinfo notifications.
- */
-static struct GNUNET_PEERINFO_NotifyContext *pnc;
-
-/**
- * Hash map of peers to HELLOs.
- */
-static struct GNUNET_CONTAINER_MultiPeerMap *peer_to_hello;
-
-
-/**
- * Obtain a peer's HELLO if available
- *
- * @param peer peer to look for a HELLO from
- * @return HELLO for the given peer
- */
-const struct GNUNET_HELLO_Message *
-GDS_HELLO_get (const struct GNUNET_PeerIdentity *peer)
-{
-  if (NULL == peer_to_hello)
-    return NULL;
-  return GNUNET_CONTAINER_multipeermap_get (peer_to_hello, peer);
-}
-
-
-/**
- * Function called for each HELLO known to PEERINFO.
- *
- * @param cls closure
- * @param peer id of the peer, NULL for last call
- * @param hello hello message for the peer (can be NULL)
- * @param err_msg error message (not used)
- */
-static void
-process_hello (void *cls, const struct GNUNET_PeerIdentity *peer,
-               const struct GNUNET_HELLO_Message *hello, const char *err_msg)
-{
-  struct GNUNET_TIME_Absolute ex;
-  struct GNUNET_HELLO_Message *hm;
-
-  if (hello == NULL)
-    return;
-  ex = GNUNET_HELLO_get_last_expiration (hello);
-  if (0 == GNUNET_TIME_absolute_get_remaining (ex).rel_value_us)
-    return;
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# HELLOs obtained from peerinfo"), 1,
-                            GNUNET_NO);
-  hm = GNUNET_CONTAINER_multipeermap_get (peer_to_hello, peer);
-  GNUNET_free_non_null (hm);
-  hm = GNUNET_malloc (GNUNET_HELLO_size (hello));
-  GNUNET_memcpy (hm, hello, GNUNET_HELLO_size (hello));
-  GNUNET_assert (GNUNET_SYSERR !=
-                 GNUNET_CONTAINER_multipeermap_put (peer_to_hello,
-                                                    peer, hm,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE));
-}
-
-
-/**
- * Initialize HELLO subsystem.
- */
-void
-GDS_HELLO_init ()
-{
-  pnc = GNUNET_PEERINFO_notify (GDS_cfg, GNUNET_NO, &process_hello, NULL);
-  peer_to_hello = GNUNET_CONTAINER_multipeermap_create (256, GNUNET_NO);
-}
-
-
-/**
- * Free memory occopied by the HELLO.
- */
-static int
-free_hello (void *cls,
-            const struct GNUNET_PeerIdentity *key,
-            void *hello)
-{
-  GNUNET_free (hello);
-  return GNUNET_OK;
-}
-
-
-/**
- * Shutdown HELLO subsystem.
- */
-void
-GDS_HELLO_done ()
-{
-  if (NULL != pnc)
-  {
-    GNUNET_PEERINFO_notify_cancel (pnc);
-    pnc = NULL;
-  }
-  if (NULL != peer_to_hello)
-  {
-    GNUNET_CONTAINER_multipeermap_iterate (peer_to_hello, &free_hello, NULL);
-    GNUNET_CONTAINER_multipeermap_destroy (peer_to_hello);
-  }
-}
-
-/* end of gnunet-service-dht_hello.c */
diff --git a/src/dht/gnunet-service-xdht_hello.h b/src/dht/gnunet-service-xdht_hello.h
deleted file mode 100644
index 7e7a79bcf..000000000
--- a/src/dht/gnunet-service-xdht_hello.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_hello.h
- * @brief GNUnet DHT integration with peerinfo
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_XDHT_HELLO_H
-#define GNUNET_SERVICE_XDHT_HELLO_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet_hello_lib.h"
-
-/**
- * Obtain a peer's HELLO if available
- *
- * @param peer peer to look for a HELLO from
- * @return HELLO for the given peer
- */
-const struct GNUNET_HELLO_Message *
-GDS_HELLO_get (const struct GNUNET_PeerIdentity *peer);
-
-
-/**
- * Initialize HELLO subsystem.
- */
-void
-GDS_HELLO_init (void);
-
-
-/**
- * Shutdown HELLO subsystem.
- */
-void
-GDS_HELLO_done (void);
-
-#endif
diff --git a/src/dht/gnunet-service-xdht_neighbours.c b/src/dht/gnunet-service-xdht_neighbours.c
deleted file mode 100644
index d41eb1900..000000000
--- a/src/dht/gnunet-service-xdht_neighbours.c
+++ /dev/null
@@ -1,6265 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2014, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_neighbours.c
- * @brief GNUnet DHT service's finger and friend table management code
- * @author Supriti Singh
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_constants.h"
-#include "gnunet_protocols.h"
-#include "gnunet_ats_service.h"
-#include "gnunet_core_service.h"
-#include "gnunet_datacache_lib.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-dht.h"
-#include "gnunet-service-dht_datacache.h"
-#include "gnunet-service-dht_neighbours.h"
-#include "gnunet-service-xdht_routing.h"
-#include "dht.h"
-
-/**
- * TODO:
- * 1. In X-Vine paper, there is no policy defined for replicating the data to
- * recover in case of peer failure. We can do it in Chord way. In R5N, the key
- * is hashed and then data is stored according to the key value generated after
- * hashing.
- */
-
-#define DEBUG(...)                                           \
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
-
-/**
- * Maximum possible fingers (including predecessor) of a peer
- */
-#define MAX_FINGERS 65
-
-/**
- * Maximum allowed number of pending messages per friend peer.
- */
-#define MAXIMUM_PENDING_PER_FRIEND 64
-
-/**
- * How long to wait before sending another find finger trail request
- */
-#define DHT_FIND_FINGER_TRAIL_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2)
-
-/**
- * How long to wait before sending another verify successor message.
- */
-#define DHT_SEND_VERIFY_SUCCESSOR_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2)
-
-/**
- * How long to wait before sending another verify successor message.
- */
-#define DHT_SEND_VERIFY_SUCCESSOR_RETRY_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
-
-/**
- * How long to wait before retrying notify successor.
- */
-#define DHT_SEND_NOTIFY_SUCCESSOR_RETRY_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
-
-/**
- * How long at most to wait for transmission of a request to a friend ?
- */
-#define PENDING_MESSAGE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2)
-
-/**
- * Duration for which I may remain congested.
- * Note: Its a static value. In future, a peer may do some analysis and calculate
- * congestion_timeout based on 'some' parameters.
- */
-#define CONGESTION_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2)
-
-/**
- * In case we don't hear back from the current successor, then we can start
- * verify successor.
- */
-#define WAIT_NOTIFY_CONFIRMATION GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 200)
-
-/**
- * Maximum number of trails allowed to go through a friend.
- */
-#define TRAILS_THROUGH_FRIEND_THRESHOLD 64
-
-/**
- * Maximum number of trails stored per finger.
- */
-#define MAXIMUM_TRAILS_PER_FINGER 4
-
-/**
- * Finger map index for predecessor entry in finger table.
- */
-#define PREDECESSOR_FINGER_ID 64
-
-/**
- * FIXME: Its use only at 3 places check if you can remove it.
- * To check if a finger is predecessor or not.
- */
-enum GDS_NEIGHBOURS_finger_type
-{
-  GDS_FINGER_TYPE_PREDECESSOR = 1,
-  GDS_FINGER_TYPE_NON_PREDECESSOR = 0
-};
-
-GNUNET_NETWORK_STRUCT_BEGIN
-
-/**
- * P2P PUT message
- */
-struct PeerPutMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_PUT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Processing options
-   */
-  uint32_t options GNUNET_PACKED;
-
-  /**
-   * Content type.
-   */
-  uint32_t block_type GNUNET_PACKED;
-
-  /**
-   * Hop count
-   */
-  uint32_t hop_count GNUNET_PACKED;
-
-  /**
-   * Replication level for this message
-   * In the current implementation, this value is not used.
-   */
-  uint32_t desired_replication_level GNUNET_PACKED;
-
-  /**
-   * Length of the PUT path that follows (if tracked).
-   */
-  uint32_t put_path_length GNUNET_PACKED;
-
-  /**
-   * Best known destination (could be my friend or finger) which should
-   * get this message next.
-   */
-  struct GNUNET_PeerIdentity best_known_destination;
-
-  /**
-   * In case best_known_destination is a finger, then trail to reach
-   * to that finger. Else its default value is 0.
-   */
-  struct GNUNET_HashCode intermediate_trail_id;
-
-  /**
-   * When does the content expire?
-   */
-  struct GNUNET_TIME_AbsoluteNBO expiration_time;
-
-  /**
-   * The key to store the value under.
-   */
-  struct GNUNET_HashCode key GNUNET_PACKED;
-
-  /* put path (if tracked) */
-
-  /* Payload */
-
-};
-
-/**
- * P2P GET message
- */
-struct PeerGetMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_GET
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Processing options
-   */
-  uint32_t options GNUNET_PACKED;
-
-  /**
-   * Desired content type.
-   */
-  uint32_t block_type GNUNET_PACKED;
-
-  /**
-   * Hop count
-   */
-  uint32_t hop_count GNUNET_PACKED;
-
-  /**
-   * Desired replication level for this request.
-   * In the current implementation, this value is not used.
-   */
-  uint32_t desired_replication_level GNUNET_PACKED;
-
-  /**
-   * Total number of peers in get path.
-   */
-  unsigned int get_path_length;
-
-  /**
-   * Best known destination (could be my friend or finger) which should
-   * get this message next.
-   */
-  struct GNUNET_PeerIdentity best_known_destination;
-
-  /**
-   * In case best_known_destination is a finger, then trail to reach
-   * to that finger. Else its default value is 0.
-   */
-  struct GNUNET_HashCode intermediate_trail_id;
-
-  /**
-   * The key we are looking for.
-   */
-  struct GNUNET_HashCode key;
-
-  /* Get path. */
-  /* struct GNUNET_PeerIdentity[]*/
-};
-
-/**
- * P2P Result message
- */
-struct PeerGetResultMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_GET_RESULT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * The type for the data.
-   */
-  uint32_t type GNUNET_PACKED;
-
-  /**
-   * Number of peers recorded in the outgoing path from source to the
-   * stored location of this message.
-   */
-  uint32_t put_path_length GNUNET_PACKED;
-
-  /**
-   * Length of the GET path that follows (if tracked).
-   */
-  uint32_t get_path_length GNUNET_PACKED;
-
-  /**
-   * Peer which queried for get and should get the result.
-   */
-  struct GNUNET_PeerIdentity querying_peer;
-
-  /**
-   * When does the content expire?
-   */
-  struct GNUNET_TIME_AbsoluteNBO expiration_time;
-
-  /**
-   * The key of the corresponding GET request.
-   */
-  struct GNUNET_HashCode key;
-
-  /* put path (if tracked) */
-
-  /* get path (if tracked) */
-
-  /* Payload */
-
-};
-
-/**
- * P2P Trail setup message
- */
-struct PeerTrailSetupMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Is source_peer trying to setup the trail to a predecessor or any finger.
-   */
-  uint32_t is_predecessor;
-
-  /**
-   * Peer closest to this value will be our finger.
-   */
-  uint64_t final_destination_finger_value;
-
-  /**
-   * Source peer which wants to setup the trail to one of its finger.
-   */
-  struct GNUNET_PeerIdentity source_peer;
-
-  /**
-   * Best known destination (could be my friend or finger) which should
-   * get this message next.
-   *
-   * FIXME: this could be removed if we include trail_source / trail_dest
-   * in the routing table. This way we save 32 bytes of bandwidth by using
-   * extra 8 bytes of memory (2 * sizeof (GNUNET_PEER_ID))
-   */
-  struct GNUNET_PeerIdentity best_known_destination;
-
-  /**
-   * In case best_known_destination is a finger, then trail id of trail to
-   * reach to this finger.
-   */
-  struct GNUNET_HashCode intermediate_trail_id;
-
-  /**
-   * Trail id for trail which we are trying to setup.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /* List of peers which are part of trail setup so far.
-   * Trail does NOT include source_peer and peer which will be closest to
-   * ultimate_destination_finger_value.
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-/**
-  * P2P Trail Setup Result message
- */
-struct PeerTrailSetupResultMessage
-{
-
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_RESULT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Finger to which we have found the path.
-   */
-  struct GNUNET_PeerIdentity finger_identity;
-
-  /**
-   * Peer which started trail_setup to find trail to finger_identity
-   */
-  struct GNUNET_PeerIdentity querying_peer;
-
-  /**
-   * Is the trail setup to querying_peer's predecessor or finger?
-   */
-  uint32_t is_predecessor;
-
-  /**
-   * Value to which finger_identity is the closest peer.
-   */
-  uint64_t ultimate_destination_finger_value;
-
-  /**
-   * Identifier of the trail from querying peer to finger_identity, NOT
-   * including both endpoints.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /* List of peers which are part of the trail from querying peer to
-   * finger_identity, NOT including both endpoints.
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-/**
- * P2P Verify Successor Message.
- */
-struct PeerVerifySuccessorMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Peer which wants to verify its successor.
-   */
-  struct GNUNET_PeerIdentity source_peer;
-
-  /**
-   * Source Peer's current successor.
-   */
-  struct GNUNET_PeerIdentity successor;
-
-  /**
-   * Identifier of trail to reach from source_peer to successor.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /* List of the peers which are part of trail to reach  from source_peer
-   * to successor, NOT including them
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-/**
- * P2P Verify Successor Result Message
- */
-struct PeerVerifySuccessorResultMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR_RESULT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Peer which sent the request to verify its successor.
-   */
-  struct GNUNET_PeerIdentity querying_peer;
-
-  /**
-   * Successor to which PeerVerifySuccessorMessage was sent.
-   */
-  struct GNUNET_PeerIdentity current_successor;
-
-  /**
-   * Current Predecessor of source_successor. It can be same as querying peer
-   * or different. In case it is different then it can be querying_peer's
-   * probable successor.
-   */
-  struct GNUNET_PeerIdentity probable_successor;
-
-  /**
-   * Trail identifier of trail from querying_peer to current_successor.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Direction in which we are looking at the trail.
-   */
-  uint32_t trail_direction;
-
-  /* In case probable_successor != querying_peer, then trail to reach from
-   * querying_peer to probable_successor, NOT including end points.
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-/**
- * P2P Notify New Successor Message.
- */
-struct PeerNotifyNewSuccessorMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_NOTIFY_NEW_SUCCESSOR
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Peer which wants to notify its new successor.
-   */
-  struct GNUNET_PeerIdentity source_peer;
-
-  /**
-   * New successor of source_peer.
-   */
-  struct GNUNET_PeerIdentity new_successor;
-
-  /**
-   * Unique identifier of the trail from source_peer to new_successor,
-   * NOT including the endpoints.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /* List of peers in trail from source_peer to new_successor,
-   * NOT including the endpoints.
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-/**
- * P2P Notify Successor Confirmation message.
- */
-struct PeerNotifyConfirmationMessage
-{
-   /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_TEARDOWN
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique identifier of the trail.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Direction of trail.
-   */
-  uint32_t trail_direction;
-};
-
-
-/**
- * P2P Trail Tear Down message.
- */
-struct PeerTrailTearDownMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_TEARDOWN
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique identifier of the trail.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Direction of trail.
-   */
-  uint32_t trail_direction;
-};
-
-
-/**
- * P2P Trail Rejection Message.
- */
-struct PeerTrailRejectionMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_REJECTION
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Peer which wants to set up the trail.
-   */
-  struct GNUNET_PeerIdentity source_peer;
-
-  /**
-   * Peer which sent trail rejection message as it it congested.
-   */
-  struct GNUNET_PeerIdentity congested_peer;
-
-  /**
-   * Peer identity closest to this value will be finger of
-   * source_peer.
-   */
-  uint64_t ultimate_destination_finger_value;
-
-  /**
-   * Is source_peer trying to setup the trail to its predecessor or finger.
-   */
-  uint32_t is_predecessor;
-
-  /**
-   * Identifier for the trail that source peer is trying to setup.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Relative time for which congested_peer will remain congested.
-   */
-  struct GNUNET_TIME_Relative congestion_time;
-
-  /* Trail_list from source_peer to peer which sent the message for trail setup
-   * to congested peer. This trail does NOT include source_peer.
-   struct GNUNET_PeerIdnetity trail[]*/
-};
-
-/**
- * P2P Add Trail Message.
- */
-struct PeerAddTrailMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_XDHT_P2P_ADD_TRAIL
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Source of the routing trail.
-   */
-  struct GNUNET_PeerIdentity source_peer;
-
-  /**
-   * Destination of the routing trail.
-   */
-  struct GNUNET_PeerIdentity destination_peer;
-
-  /**
-   * Unique identifier of the trail from source_peer to destination_peer,
-   * NOT including the endpoints.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /* Trail from source peer to destination peer, NOT including them.
-   * struct GNUNET_PeerIdentity trail[]
-   */
-};
-
-
-GNUNET_NETWORK_STRUCT_END
-
-
-/**
- *  Entry in friend_peermap.
- */
-struct FriendInfo
-{
-  /**
-   * Friend Identity
-   */
-  const struct GNUNET_PeerIdentity *id;
-
-  /**
-   * Number of trails for which this friend is the first hop or if the friend
-   * is finger.
-   */
-  unsigned int trails_count;
-
-  /**
-   * In case not 0, then amount of time for which this friend is congested.
-   */
-  struct GNUNET_TIME_Absolute congestion_timestamp;
-
-  /**
-   * Handle for sending messages to this friend.
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-};
-
-/**
- * An individual element of the trail to reach to a finger.
- */
-struct Trail_Element
-{
-  /**
-    * Pointer to next item in the list
-    */
-  struct Trail_Element *next;
-
-  /**
-    * Pointer to prev item in the list
-    */
-  struct Trail_Element *prev;
-
-  /**
-   * An element in this trail.
-   */
-  struct GNUNET_PeerIdentity peer;
-};
-
-/**
- * Information about an individual trail.
- */
-struct Trail
-{
-  /**
-   * Head of trail.
-   */
-  struct Trail_Element *trail_head;
-
-  /**
-   * Tail of trail.
-   */
-  struct Trail_Element *trail_tail;
-
-  /**
-   * Unique identifier of this trail.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Length of trail pointed
-   */
-  unsigned int trail_length;
-
-  /**
-   * Is there a valid trail entry.
-   */
-  unsigned int is_present;
-};
-
-/**
- * An entry in finger_table
- */
-struct FingerInfo
-{
-  /**
-   * Finger identity.
-   */
-  struct GNUNET_PeerIdentity finger_identity;
-
-  /**
-   * In case not 0, this amount is time to wait for notify successor message.
-   * Used ONLY for successor. NOT for any other finger.
-   */
-  struct GNUNET_TIME_Absolute wait_notify_confirmation;
-
-  /**
-   * Is any finger stored at this finger index.
-   */
-  unsigned int is_present;
-
-  /**
-   * Index in finger peer map
-   */
-  uint32_t finger_table_index;
-
-  /**
-   * Number of trails setup so far for this finger.
-   * Should not cross MAXIMUM_TRAILS_PER_FINGER.
-   */
-  uint32_t trails_count;
-
-  /**
-   * Array of trails to reach to this finger.
-   */
-  struct Trail trail_list[MAXIMUM_TRAILS_PER_FINGER];
-};
-
-
-/**
- * Stores information about the peer which is closest to destination_finger_value.
- * 'closest' can be either successor or predecessor depending on is_predecessor
- * flag.
- */
-struct Closest_Peer
-{
-  /**
-   * Destination finger value.
-   */
-  uint64_t destination_finger_value;
-
-  /**
-   * Is finger_value a predecessor or any other finger.
-   */
-  unsigned int is_predecessor;
-
-  /**
-   * Trail id to reach to peer.
-   * In case peer is my identity or friend, it is set to 0.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * Next destination. In case of friend and my_identity , it is same as next_hop
-   * In case of finger it is finger identity.
-   */
-  struct GNUNET_PeerIdentity best_known_destination;
-
-  /**
-   * In case best_known_destination is a finger, then first friend in the trail
-   * to reach to it. In other case, same as best_known_destination.
-   */
-  struct GNUNET_PeerIdentity next_hop;
-
-  /**
-   * In case finger is the next hop, it contains a valid finger table index
-   * at which the finger is stored. Else, It contains 65, which is out of range
-   * of finger table index.
-   */
-  unsigned int finger_table_index;
-};
-
-/**
- * Context for send_verify_successor_task.
- */
-struct VerifySuccessorContext
-{
-  /**
-   * Number of times this has been scheduled.
-   */
-  unsigned int num_retries_scheduled;
-};
-
-/**
- * Task that sends FIND FINGER TRAIL requests. This task is started when we have
- * get our first friend.
- */
-static struct GNUNET_SCHEDULER_Task *find_finger_trail_task;
-
-/**
- * Task that sends verify successor message. This task is started when we get
- * our successor for the first time.
- */
-static struct GNUNET_SCHEDULER_Task *send_verify_successor_task;
-
-/**
- * Task that sends verify successor message. This task is started when we get
- * our successor for the first time.
- */
-static struct GNUNET_SCHEDULER_Task *send_verify_successor_retry_task;
-
-/**
- * Task that sends verify successor message. This task is started when we get
- * our successor for the first time.
- */
-static struct GNUNET_SCHEDULER_Task *send_notify_new_successor_retry_task;
-
-/**
- * Identity of this peer.
- */
-static struct GNUNET_PeerIdentity my_identity;
-
-/**
- * Peer map of all the friends of a peer
- */
-static struct GNUNET_CONTAINER_MultiPeerMap *friend_peermap;
-
-/**
- * Array of all the fingers.
- */
-static struct FingerInfo finger_table [MAX_FINGERS];
-
-/**
- * Handle to CORE.
- */
-static struct GNUNET_CORE_Handle *core_api;
-
-/**
- * The current finger index that we have want to find trail to. We start the
- * search with value = 0, i.e. successor  and then go to PREDCESSOR_FINGER_ID
- * and decrement it. For any index 63 <= index < 0, if finger is same as successor,
- * we reset this index to 0.
- */
-static unsigned int current_search_finger_index;
-
-/**
- * Time duration to schedule find finger trail task.
- */
-static struct GNUNET_TIME_Relative find_finger_trail_task_next_send_time;
-
-/**
- * Time duration to schedule verify successor task.
- */
-static struct GNUNET_TIME_Relative verify_successor_next_send_time;
-
-/**
- * Time duration to send verify successor again, if result was not received in time.
- */
-static struct GNUNET_TIME_Relative verify_successor_retry_time;
-
-/**
- * Time duration to retry send_notify_successor.
- */
-static struct GNUNET_TIME_Relative notify_successor_retry_time;
-
-/**
- * Are we waiting for confirmation from our new successor that it got the
- * message
- */
-//static unsigned int waiting_for_notify_confirmation;
-
-/* Below variables are used only for testing, and statistics collection. */
-/**
- * Should we store our topology predecessor and successor IDs into statistics?
- */
-unsigned int track_topology;
-
-/**
- * Count of fingers found. Ideally we should have O(logn) fingers for a
- * stable network.
- */
-static unsigned int total_fingers_found;
-
-/**
- * Number of times we found the same successor.
- */
-static unsigned int successor_times;
-
-/**
- * Number of rounds for which we should search for finger.
- */
-static unsigned int fingers_round_count;
-
-
-/**
- * Construct a trail setup message and forward it to @a target_friend
- *
- * @param source_peer Peer which wants to setup the trail
- * @param ultimate_destination_finger_value Peer identity closest to this value
- *                                          will be finger to @a source_peer
- * @param best_known_destination Best known destination (could be finger or friend)
- *                               which should get this message. In case it is
- *                               friend, then it is same as target_friend
- * @param target_friend Friend to which message is forwarded now.
- * @param trail_length Total number of peers in trail setup so far.
- * @param trail_peer_list Trail setup so far
- * @param is_predecessor Is @a source_peer looking for trail to a predecessor or not.
- * @param trail_id Unique identifier for the trail we are trying to setup.
- * @param intermediate_trail_id Trail id of intermediate trail to reach to
- *                              best_known_destination when its a finger. If not
- *                              used then set to 0.
- */
-static void
-GDS_NEIGHBOURS_send_trail_setup (const struct GNUNET_PeerIdentity *source_peer,
-                                 uint64_t ultimate_destination_finger_value,
-                                 const struct GNUNET_PeerIdentity *best_known_destination,
-                                 const struct FriendInfo *target_friend,
-                                 unsigned int trail_length,
-                                 const struct GNUNET_PeerIdentity *trail_peer_list,
-                                 unsigned int is_predecessor,
-                                 const struct GNUNET_HashCode *trail_id,
-                                 const struct GNUNET_HashCode *intermediate_trail_id)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct PeerTrailSetupMessage *tsm;
-  size_t msize;
-
-  msize = trail_length * sizeof (struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerTrailSetupMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (tsm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP);
-  tsm->final_destination_finger_value = GNUNET_htonll (ultimate_destination_finger_value);
-  tsm->source_peer = *source_peer;
-  tsm->best_known_destination = *best_known_destination;
-  tsm->is_predecessor = htonl (is_predecessor);
-  tsm->trail_id = *trail_id;
-  tsm->intermediate_trail_id = *intermediate_trail_id;
-  GNUNET_memcpy (&tsm[1],
-                 trail_peer_list,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Construct a trail setup result message and forward it to @a target_friend.
- *
- * @param querying_peer Peer which sent the trail setup request and should get
- *                      the result back.
- * @param Finger Peer to which the trail has been setup to.
- * @param target_friend Friend to which this message should be forwarded.
- * @param trail_length Numbers of peers in the trail.
- * @param trail_peer_list Peers which are part of the trail from
- *                        querying_peer to Finger, NOT including them.
- * @param is_predecessor Is @a Finger predecessor to @a querying_peer ?
- * @param ultimate_destination_finger_value Value to which @a finger is the closest
- *                                          peer.
- * @param trail_id Unique identifier of the trail.
- */
-static void
-GDS_NEIGHBOURS_send_trail_setup_result (const struct GNUNET_PeerIdentity *querying_peer,
-                                        const struct GNUNET_PeerIdentity *finger,
-                                        struct FriendInfo *target_friend,
-                                        unsigned int trail_length,
-                                        const struct GNUNET_PeerIdentity *trail_peer_list,
-                                        unsigned int is_predecessor,
-                                        uint64_t ultimate_destination_finger_value,
-                                        const struct GNUNET_HashCode *trail_id)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct PeerTrailSetupResultMessage *tsrm;
-  size_t msize;
-
-  msize = trail_length * sizeof (struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerTrailSetupResultMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (tsrm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_RESULT);
-  tsrm->querying_peer = *querying_peer;
-  tsrm->finger_identity = *finger;
-  tsrm->is_predecessor = htonl (is_predecessor);
-  tsrm->trail_id = *trail_id;
-  tsrm->ultimate_destination_finger_value
-    = GNUNET_htonll (ultimate_destination_finger_value);
-  GNUNET_memcpy (&tsrm[1],
-                 trail_peer_list,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Send notify successor confirmation message.
- *
- * @param trail_id Unique Identifier of the trail.
- * @param trail_direction Destination to Source.
- * @param target_friend Friend to get this message next.
- */
-static void
-GDS_NEIGHBOURS_send_notify_succcessor_confirmation (const struct GNUNET_HashCode *trail_id,
-                                                    unsigned int trail_direction,
-                                                    struct FriendInfo *target_friend)
-{
-  struct PeerNotifyConfirmationMessage *ncm;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg (ncm,
-                       GNUNET_MESSAGE_TYPE_XDHT_P2P_NOTIFY_SUCCESSOR_CONFIRMATION);
-  ncm->trail_id = *trail_id;
-  ncm->trail_direction = htonl (trail_direction);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Send trail rejection message to @a target_friend
- *
- * @param source_peer Peer which is trying to setup the trail.
- * @param ultimate_destination_finger_value Peer closest to this value will be
- *                                          @a source_peer's finger
- * @param congested_peer Peer which sent this message as it is congested.
- * @param is_predecessor Is source_peer looking for trail to a predecessor or not.
- * @param trail_peer_list Trails seen so far in trail setup before getting rejected
- *                        by congested_peer. This does NOT include @a source_peer
- *                        and congested_peer.
- * @param trail_length Total number of peers in trail_peer_list, NOT including
- *                     @a source_peer and @a congested_peer
- * @param trail_id Unique identifier of this trail.
- * @param congestion_timeout Duration given by congested peer as an estimate of
- *                           how long it may remain congested.
- */
-static void
-GDS_NEIGHBOURS_send_trail_rejection (const struct GNUNET_PeerIdentity *source_peer,
-                                     uint64_t ultimate_destination_finger_value,
-                                     const struct GNUNET_PeerIdentity *congested_peer,
-                                     unsigned int is_predecessor,
-                                     const struct GNUNET_PeerIdentity *trail_peer_list,
-                                     unsigned int trail_length,
-                                     const struct GNUNET_HashCode *trail_id,
-                                     struct FriendInfo *target_friend,
-                                     const struct GNUNET_TIME_Relative congestion_timeout)
-{
-  struct PeerTrailRejectionMessage *trm;
-  struct GNUNET_MQ_Envelope *env;
-  size_t msize;
-
-  msize = trail_length * sizeof (struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerTrailRejectionMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (trm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_REJECTION);
-  trm->source_peer = *source_peer;
-  trm->congested_peer = *congested_peer;
-  trm->congestion_time = congestion_timeout;
-  trm->is_predecessor = htonl (is_predecessor);
-  trm->trail_id = *trail_id;
-  trm->ultimate_destination_finger_value
-    = GNUNET_htonll (ultimate_destination_finger_value);
-  GNUNET_memcpy (&trm[1],
-                 trail_peer_list,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Construct a verify successor message and forward it to target_friend.
- * @param source_peer Peer which wants to verify its successor.
- * @param successor Peer which is @a source_peer's current successor.
- * @param trail_id Unique Identifier of trail from @a source_peer to @a successor,
- *                 NOT including them.
- * @param trail List of peers which are part of trail to reach from @a source_peer
- *              to @a successor, NOT including them.
- * @param trail_length Total number of peers in @a trail.
- * @param target_friend Next friend to get this message.
- */
-static void
-GDS_NEIGHBOURS_send_verify_successor_message (const struct GNUNET_PeerIdentity *source_peer,
-                                              const struct GNUNET_PeerIdentity *successor,
-                                              const struct GNUNET_HashCode *trail_id,
-                                              struct GNUNET_PeerIdentity *trail,
-                                              unsigned int trail_length,
-                                              struct FriendInfo *target_friend)
-{
-  struct PeerVerifySuccessorMessage *vsm;
-  struct GNUNET_MQ_Envelope *env;
-  size_t msize;
-
-  msize = trail_length * sizeof (struct GNUNET_PeerIdentity);
-  if (msize + sizeof (*vsm) >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (vsm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR);
-  vsm->source_peer = *source_peer;
-  vsm->successor = *successor;
-  vsm->trail_id = *trail_id;
-  GNUNET_memcpy (&vsm[1],
-                 trail,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * FIXME: In every function we pass target friend except for this one.
- * so, either change everything or this one. also, should we just store
- * the pointer to friend in routing table rather than gnunet_peeridentity.
- * if yes then we should keep friend info in.h  andmake lot of changes.
- * Construct a trail teardown message and forward it to target friend.
- *
- * @param trail_id Unique identifier of the trail.
- * @param trail_direction Direction of trail.
- * @param target_friend Friend to get this message.
- */
-void
-GDS_NEIGHBOURS_send_trail_teardown (const struct GNUNET_HashCode *trail_id,
-                                    unsigned int trail_direction,
-                                    const struct GNUNET_PeerIdentity *peer)
-{
-  struct PeerTrailTearDownMessage *ttdm;
-  struct GNUNET_MQ_Envelope *env;
-  struct FriendInfo *target_friend;
-
-  if (NULL == (target_friend =
-               GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                  peer)))
-  {
-    /* FIXME: In what case friend can be null. ?*/
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg (ttdm,
-                       GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_TEARDOWN);
-  ttdm->trail_id = *trail_id;
-  ttdm->trail_direction = htonl (trail_direction);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Construct a verify successor result message and send it to target_friend
- *
- * @param querying_peer Peer which sent the verify successor message.
- * @param source_successor Current_successor of @a querying_peer.
- * @param current_predecessor Current predecessor of @a successor. Could be same
- *                            or different from @a querying_peer.
- * @param trail_id Unique identifier of the trail from @a querying_peer to
- *                 @a successor, NOT including them.
- * @param trail List of peers which are part of trail from @a querying_peer to
- *                 @a successor, NOT including them.
- * @param trail_length Total number of peers in @a trail
- * @param trail_direction Direction in which we are sending the message. In this
- *                        case we are sending result from @a successor to @a querying_peer.
- * @param target_friend Next friend to get this message.
- */
-static void
-GDS_NEIGHBOURS_send_verify_successor_result (const struct GNUNET_PeerIdentity *querying_peer,
-                                             const struct GNUNET_PeerIdentity *current_successor,
-                                             const struct GNUNET_PeerIdentity *probable_successor,
-                                             const struct GNUNET_HashCode *trail_id,
-                                             const struct GNUNET_PeerIdentity *trail,
-                                             unsigned int trail_length,
-                                             enum GDS_ROUTING_trail_direction trail_direction,
-                                             struct FriendInfo *target_friend)
-{
-  struct PeerVerifySuccessorResultMessage *vsmr;
-  struct GNUNET_MQ_Envelope *env;
-  size_t msize;
-
-  msize = trail_length * sizeof(struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerVerifySuccessorResultMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (vsmr,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR_RESULT);
-  vsmr->querying_peer = *querying_peer;
-  vsmr->current_successor = *current_successor;
-  vsmr->probable_successor = *probable_successor;
-  vsmr->trail_direction = htonl (trail_direction);
-  vsmr->trail_id = *trail_id;
-  GNUNET_memcpy (&vsmr[1],
-                 trail,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Construct a notify new successor message and send it to target_friend
- * @param source_peer Peer which wants to notify to its new successor that it
- *                    could be its predecessor.
- * @param successor New successor of @a source_peer
- * @param successor_trail List of peers in Trail to reach from
- *                            @a source_peer to @a new_successor, NOT including
- *                            the endpoints.
- * @param successor_trail_length Total number of peers in @a new_successor_trail.
- * @param successor_trail_id Unique identifier of @a new_successor_trail.
- * @param target_friend Next friend to get this message.
- */
-static void
-GDS_NEIGHBOURS_send_notify_new_successor (const struct GNUNET_PeerIdentity *source_peer,
-                                          const struct GNUNET_PeerIdentity *successor,
-                                          const struct GNUNET_PeerIdentity *successor_trail,
-                                          unsigned int successor_trail_length,
-                                          const struct GNUNET_HashCode *succesor_trail_id,
-                                          struct FriendInfo *target_friend)
-{
-  struct PeerNotifyNewSuccessorMessage *nsm;
-  struct GNUNET_MQ_Envelope *env;
-  size_t msize;
-
-  msize = successor_trail_length * sizeof(struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerNotifyNewSuccessorMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (nsm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_NOTIFY_NEW_SUCCESSOR);
-  nsm->new_successor = *successor;
-  nsm->source_peer = *source_peer;
-  nsm->trail_id = *succesor_trail_id;
-  GNUNET_memcpy (&nsm[1],
-                 successor_trail,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Construct an add_trail message and send it to target_friend
- *
- * @param source_peer Source of the trail.
- * @param destination_peer Destination of the trail.
- * @param trail_id Unique identifier of the trail from
- *                 @a source_peer to @a destination_peer, NOT including the endpoints.
- * @param trail List of peers in Trail from @a source_peer to @a destination_peer,
- *              NOT including the endpoints.
- * @param trail_length Total number of peers in @a trail.
- * @param target_friend Next friend to get this message.
- */
-static void
-GDS_NEIGHBOURS_send_add_trail (const struct GNUNET_PeerIdentity *source_peer,
-                               const struct GNUNET_PeerIdentity *destination_peer,
-                               const struct GNUNET_HashCode *trail_id,
-                               const struct GNUNET_PeerIdentity *trail,
-                               unsigned int trail_length,
-                               struct FriendInfo *target_friend)
-{
-  struct PeerAddTrailMessage *adm;
-  struct GNUNET_MQ_Envelope *env;
-  size_t msize;
-
-  msize = trail_length * sizeof(struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerAddTrailMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_MQ_get_length (target_friend->mq) >= MAXIMUM_PENDING_PER_FRIEND)
-  {
-    GNUNET_STATISTICS_update (GDS_stats,
-                              gettext_noop ("# P2P messages dropped due to full queue"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (adm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_ADD_TRAIL);
-  adm->source_peer = *source_peer;
-  adm->destination_peer = *destination_peer;
-  adm->trail_id = *trail_id;
-  GNUNET_memcpy (&adm[1],
-                 trail,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Search my location in trail. In case I am present more than once in the
- * trail (can happen during trail setup), then return my lowest index.
- *
- * @param trail List of peers
- * @return my_index if found
- *         trail_length + 1 if an entry is present twice, It is an error.
- *         -1 if no entry found.
- */
-static int
-search_my_index (const struct GNUNET_PeerIdentity *trail,
-                 int trail_length)
-{
-  int i;
-  int index_seen = trail_length + 1;
-  int flag = 0;
-
-  for (i = 0; i < trail_length; i++)
-  {
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity, &trail[i]))
-    {
-      flag = 1;
-      if(index_seen == (trail_length + 1))
-        index_seen = i;
-      else
-      {
-        DEBUG("Entry is present twice in trail. Its not allowed\n");
-      }
-      break;
-    }
-  }
-
-  if (1 == flag)
-    return index_seen;
-  return -1;
-}
-
-
-/**
- * Check if the friend is congested or have reached maximum number of trails
- * it can be part of of.
- * @param friend Friend to be checked.
- * @return #GNUNET_NO if friend is not congested or have not crossed threshold.
- *         #GNUNET_YES if friend is either congested or have crossed threshold
- */
-static int
-is_friend_congested (struct FriendInfo *friend)
-{
-  if (( friend->trails_count < TRAILS_THROUGH_FRIEND_THRESHOLD) &&
-      ((0 == GNUNET_TIME_absolute_get_remaining
-             (friend->congestion_timestamp).rel_value_us)))
-    return GNUNET_NO;
-  return GNUNET_YES;
-}
-
-
-/**
- * Select closest finger to value.
- *
- * @param peer1 First peer
- * @param peer2 Second peer
- * @param value Value to be compare
- * @return Closest peer
- */
-static const struct GNUNET_PeerIdentity *
-select_closest_finger (const struct GNUNET_PeerIdentity *peer1,
-                       const struct GNUNET_PeerIdentity *peer2,
-                       uint64_t value)
-{
-  uint64_t peer1_value;
-  uint64_t peer2_value;
-
-  GNUNET_memcpy (&peer1_value, peer1, sizeof (uint64_t));
-  GNUNET_memcpy (&peer2_value, peer2, sizeof (uint64_t));
-  peer1_value = GNUNET_ntohll (peer1_value);
-  peer2_value = GNUNET_ntohll (peer2_value);
-
-  if (peer1_value == value)
-  {
-    return peer1;
-  }
-
-  if (peer2_value == value)
-  {
-    return peer2;
-  }
-
-  if (value < peer1_value && peer1_value < peer2_value)
-  {
-    return peer1;
-  }
-  else if (value < peer2_value && peer2_value < peer1_value)
-  {
-    return peer2;
-  }
-  else if (peer1_value < value && value < peer2_value)
-  {
-    return peer2;
-  }
-  else if (peer2_value < value && value < peer1_value)
-  {
-    return peer1;
-  }
-  else if (peer1_value < peer2_value && peer2_value < value)
-  {
-    return peer1;
-  }
-  else  // if (peer2_value < peer1_value && peer1_value < value)
-  {
-    return peer2;
-  }
-}
-
-
-/**
- * Select closest predecessor to value.
- *
- * @param peer1 First peer
- * @param peer2 Second peer
- * @param value Value to be compare
- * @return Peer which precedes value in the network.
- */
-static const struct GNUNET_PeerIdentity *
-select_closest_predecessor (const struct GNUNET_PeerIdentity *peer1,
-                            const struct GNUNET_PeerIdentity *peer2,
-                            uint64_t value)
-{
-  uint64_t peer1_value;
-  uint64_t peer2_value;
-
-  GNUNET_memcpy (&peer1_value, peer1, sizeof (uint64_t));
-  GNUNET_memcpy (&peer2_value, peer2, sizeof (uint64_t));
-  peer1_value = GNUNET_ntohll (peer1_value);
-  peer2_value = GNUNET_ntohll (peer2_value);
-
-   if (peer1_value == value)
-  {
-    return peer1;
-  }
-
-  if (peer2_value == value)
-  {
-    return peer2;
-  }
-
-  if (value < peer1_value && peer1_value < peer2_value)
-  {
-    return peer2;
-  }
-  else if (value < peer2_value && peer2_value < peer1_value)
-  {
-    return peer1;
-  }
-  else if (peer1_value < value && value < peer2_value)
-  {
-    return peer1;
-  }
-  else if (peer2_value < value && value < peer1_value)
-  {
-    return peer2;
-  }
-  else if (peer1_value < peer2_value && peer2_value < value)
-  {
-    return peer2;
-  }
-  else  // if (peer2_value < peer1_value && peer1_value < value)
-  {
-    return peer1;
-  }
-}
-
-#if 0
-/**
- *
- *
- */
-void
-test_print_trail (struct GNUNET_PeerIdentity *trail,
-                  unsigned int trail_length)
-{
-  struct GNUNET_PeerIdentity print_peer;
-  int i;
-
-  FPRINTF (stderr,_("\nSUPU %s, %s, %d,trail_length = %d"),
-  __FILE__, __func__,__LINE__,trail_length);
-  for (i =0 ; i< trail_length; i++)
-  {
-    print_peer = trail[i];
-    FPRINTF (stderr,_("\nSUPU %s, %s, %d,trail[%d]=%s"),
-            __FILE__, __func__,__LINE__,i,GNUNET_i2s(&print_peer));
-  }
-}
-#endif
-
-#if 0
-/**
- * This is a test function to print all the entries of friend table.
- */
-static void
-test_friend_peermap_print ()
-{
-  struct FriendInfo *friend;
-  struct GNUNET_CONTAINER_MultiPeerMapIterator *friend_iter;
-  struct GNUNET_PeerIdentity print_peer;
-  struct GNUNET_PeerIdentity key_ret;
-  int i;
-
-  print_peer = my_identity;
-  FPRINTF (stderr,_("\nSUPU************  FRIEND_PEERMAP of %s"),GNUNET_i2s(&print_peer));
-  friend_iter = GNUNET_CONTAINER_multipeermap_iterator_create (friend_peermap);
-
-  for (i = 0; i < GNUNET_CONTAINER_multipeermap_size (friend_peermap); i++)
-  {
-    if(GNUNET_YES == GNUNET_CONTAINER_multipeermap_iterator_next (friend_iter,
-                                                                  &key_ret,
-                                                                  (const void **)&friend))
-    {
-      GNUNET_memcpy (&print_peer, &key_ret, sizeof (struct GNUNET_PeerIdentity));
-      FPRINTF (stderr,_("\nSUPU %s, %s, %d, friend = %s, friend->trails_count = %d"),
-              __FILE__, __func__,__LINE__, GNUNET_i2s(&print_peer), friend->trails_count);
-    }
-  }
-}
-#endif
-
-#if 0
-/**
- * This is a test function, to print all the entries of finger table.
- */
-static void
-test_finger_table_print()
-{
-  struct FingerInfo *finger;
-  struct GNUNET_PeerIdentity print_peer;
-  //struct Trail *trail;
-  int i;
-  //int j;
-  //int k;
-  print_peer = my_identity;
-  FPRINTF (stderr,_("\nSUPU************  FINGER_TABLE of %s"),GNUNET_i2s(&print_peer));
-  for (i = 0; i < MAX_FINGERS; i++)
-  {
-    finger = &finger_table[i];
-
-    if (GNUNET_NO == finger->is_present)
-      continue;
-
-    print_peer = finger->finger_identity;
-    FPRINTF (stderr,_("\nSUPU %s, %s, %d, finger_table[%d] = %s, trails_count = %d"),
-            __FILE__, __func__,__LINE__,i,GNUNET_i2s (&print_peer), finger->trails_count);
-
-#if 0
-    for (j = 0; j < finger->trails_count; j++)
-    {
-      trail = &finger->trail_list[j];
-      FPRINTF (stderr,_("\nSUPU %s, %s, %d, trail_id[%d]=%s"),__FILE__, __func__,__LINE__,j, GNUNET_h2s(&trail->trail_id));
-      struct Trail_Element *element;
-      element = trail->trail_head;
-      for (k = 0; k < trail->trail_length; k++)
-      {
-        print_peer = element->peer;
-        FPRINTF (stderr,_("\nSUPU %s, %s, %d,trail[%d] = %s "),__FILE__, __func__,__LINE__,k, GNUNET_i2s(&print_peer));
-        element = element->next;
-      }
-    }
-    #endif
-  }
-}
-#endif
-
-/**
- * Select the closest peer among two peers (which should not be same)
- * with respect to value and finger_table_index
- * NOTE: peer1 != peer2
- * @param peer1 First peer
- * @param peer2 Second peer
- * @param value Value relative to which we find the closest
- * @param is_predecessor Is value a predecessor or any other finger.
- * @return Closest peer among two peers.
- */
-static const struct GNUNET_PeerIdentity *
-select_closest_peer (const struct GNUNET_PeerIdentity *peer1,
-                     const struct GNUNET_PeerIdentity *peer2,
-                     uint64_t value,
-                     unsigned int is_predecessor)
-{
-  /* This check is here to ensure that calling function never sends
-      same peer value in peer1 and peer2. Remove it later. */
-  GNUNET_assert(0 != GNUNET_CRYPTO_cmp_peer_identity (peer1, peer2));
-  if (1 == is_predecessor)
-    return select_closest_predecessor (peer1, peer2, value);
-
-  // TODO: Change name to something like select_closest_successor!!
-  return select_closest_finger (peer1, peer2, value);
-}
-
-
-/**
- * Iterate over the list of all the trails of a finger. In case the first
- * friend to reach the finger has reached trail threshold or is congested,
- * then don't select it. In case there multiple available good trails to reach
- * to Finger, choose the one with shortest trail length.
- * Note: We use length as parameter. But we can use any other suitable parameter
- * also.
- * @param finger Finger Finger whose trail we have to select.
- * @return Trail Selected Trail.
- */
-static struct Trail *
-select_finger_trail (struct FingerInfo *finger)
-{
-  struct FriendInfo *friend;
-  struct Trail *current_finger_trail;
-  struct Trail *best_trail = NULL;
-  unsigned int i;
-
-  GNUNET_assert (finger->trails_count > 0);
-  for (i = 0; i < finger->trails_count; i++)
-  {
-    current_finger_trail = &finger->trail_list[i];
-
-    /* No trail stored at this index. */
-    if (GNUNET_NO == current_finger_trail->is_present)
-      continue;
-
-    GNUNET_assert (NULL !=
-                  (friend =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      &current_finger_trail->trail_head->peer)));
-
-    /* First friend to reach trail is not free. */
-    if (GNUNET_YES == is_friend_congested (friend))
-      continue;
-
-    if (NULL == best_trail ||
-        best_trail->trail_length > current_finger_trail->trail_length)
-    {
-      best_trail = current_finger_trail;
-    }
-  }
-
-  return best_trail;
-}
-
-
-/**
- * Compare FINGER entry with current successor. If finger's first friend of all
- * its trail is not congested and  has not crossed trail threshold, then check
- * if finger peer identity is closer to final_destination_finger_value than
- * current_successor. If yes then update current_successor.
- * @param current_successor[in/out]
- * @return
- */
-static void
-compare_finger_and_current_closest_peer (struct Closest_Peer *current_closest_peer)
-{
-  struct FingerInfo *finger;
-  const struct GNUNET_PeerIdentity *closest_peer;
-  struct Trail *finger_trail;
-  int i;
-
-  /* Iterate over finger table. */
-  for (i = 0; i < MAX_FINGERS; i++)
-  {
-    finger = &finger_table[i];
-
-    if (GNUNET_NO == finger->is_present)
-      continue;
-
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&finger->finger_identity,
-                                              &current_closest_peer->best_known_destination))
-      continue;
-
-    /* If I am my own finger, then ignore this finger. */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&finger->finger_identity,
-                                              &my_identity))
-      continue;
-
-   /* If finger is a friend, we have already checked it in previous function. */
-    if (NULL != (GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                    &finger->finger_identity)))
-    {
-      continue;
-    }
-
-    closest_peer = select_closest_peer (&finger->finger_identity,
-                                        &current_closest_peer->best_known_destination,
-                                        current_closest_peer->destination_finger_value,
-                                        current_closest_peer->is_predecessor);
-
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity(&finger->finger_identity,
-                                             closest_peer))
-    {
-      /* Choose one of the trail to reach to finger. */
-      finger_trail = select_finger_trail (finger);
-
-      /* In case no trail found, ignore this finger. */
-      if (NULL == finger_trail)
-        continue;
-
-      current_closest_peer->best_known_destination = *closest_peer;
-      current_closest_peer->next_hop = finger_trail->trail_head->peer;
-      current_closest_peer->trail_id = finger_trail->trail_id;
-      current_closest_peer->finger_table_index = i;
-    }
-    continue;
-  }
-}
-
-
-/**
- * Compare friend entry with current successor.
- * If friend identity and current_successor is same, then do nothing.
- * If friend is not congested and has not crossed trail threshold, then check
- * if friend peer identity is closer to final_destination_finger_value than
- * current_successor. If yes then update current_successor.
- *
- * @param cls closure
- * @param key current public key
- * @param value struct Closest_Peer
- * @return #GNUNET_YES if we should continue to iterate,
- *         #GNUNET_NO if not.
- */
-static int
-compare_friend_and_current_closest_peer (void *cls,
-                                         const struct GNUNET_PeerIdentity *key,
-                                         void *value)
-{
-  struct FriendInfo *friend = value;
-  struct Closest_Peer *current_closest_peer = cls;
-  const struct GNUNET_PeerIdentity *closest_peer;
-
-  /* Friend is either congested or has crossed threshold. */
-  if (GNUNET_YES == is_friend_congested (friend))
-    return GNUNET_YES;
-
-  /* If current_closest_peer and friend identity are same, then do nothing.*/
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (friend->id,
-                                            &current_closest_peer->best_known_destination))
-  {
-    GNUNET_break (0);
-    return GNUNET_YES;
-  }
-
-  closest_peer = select_closest_peer (friend->id,
-                                      &current_closest_peer->best_known_destination,
-                                      current_closest_peer->destination_finger_value,
-                                      current_closest_peer->is_predecessor);
-
-  /* Is friend the closest successor? */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (friend->id,
-                                            closest_peer))
-  {
-    current_closest_peer->best_known_destination = *friend->id;
-    current_closest_peer->next_hop = *friend->id;
-  }
-  return GNUNET_YES;
-}
-
-
-/**
- * Initialize current_successor to my_identity.
- * @param my_identity My peer identity
- * @return Updated closest_peer
- */
-static struct Closest_Peer
-init_closest_peer (struct GNUNET_PeerIdentity my_identity,
-                   uint64_t destination_finger_value,
-                   unsigned int is_predecessor)
-{
-  struct Closest_Peer current_closest_peer;
-
-  memset (&current_closest_peer.trail_id,
-          0,
-          sizeof(struct GNUNET_HashCode));
-  current_closest_peer.destination_finger_value = destination_finger_value;
-  current_closest_peer.is_predecessor = is_predecessor;
-  current_closest_peer.next_hop = my_identity;
-  current_closest_peer.best_known_destination = my_identity;
-  current_closest_peer.finger_table_index = 65; //65 is a for non valid finger table index.
-  return current_closest_peer;
-}
-
-
-/**
- * Find locally best known peer, among your own identity, friend and finger list,
- * which is closest to given destination_finger_value.
- *
- * NOTE: In case a friend is also a finger, then it is always chosen as friend
- * not a finger.
- * @param destination_finger_value Peer closest to this value will be the next destination.
- * @param is_predecessor Are we looking for predecessor or finger?
- * @return Closest_Peer that contains all the relevant field to reach to
- *                      @a destination_finger_value
- */
-static struct Closest_Peer
-find_local_best_known_next_hop (uint64_t destination_finger_value,
-                                unsigned int is_predecessor)
-{
-  struct Closest_Peer current_closest_peer;
-
-   /* Initialize current_successor to my_identity. */
-  current_closest_peer = init_closest_peer (my_identity,
-                                            destination_finger_value,
-                                            is_predecessor);
-
-  /* Compare each friend entry with current_successor and update current_successor
-   * with friend if its closest. */
-  GNUNET_assert
-          (GNUNET_SYSERR !=
-           GNUNET_CONTAINER_multipeermap_iterate (friend_peermap,
-                                                  &compare_friend_and_current_closest_peer,
-                                                  &current_closest_peer));
-
-  /* Compare each finger entry with current_successor and update current_successor
-   * with finger if its closest. */
-  compare_finger_and_current_closest_peer (&current_closest_peer);
-  return current_closest_peer;
-}
-
-
-/**
- * Construct a Put message and send it to target_peer.
- * @param key Key for the content
- * @param block_type Type of the block
- * @param options Routing options
- * @param desired_replication_level Desired replication count
- * @param best_known_dest Peer to which this message should reach eventually,
- *                        as it is best known destination to me.
- * @param intermediate_trail_id Trail id in case
- * @param target_peer Peer to which this message will be forwarded.
- * @param hop_count Number of hops traversed so far.
- * @param put_path_length Total number of peers in @a put_path
- * @param put_path Number of peers traversed so far
- * @param expiration_time When does the content expire
- * @param data Content to store
- * @param data_size Size of content @a data in bytes
- */
-void
-GDS_NEIGHBOURS_send_put (const struct GNUNET_HashCode *key,
-                         enum GNUNET_BLOCK_Type block_type,
-                         enum GNUNET_DHT_RouteOption options,
-                         uint32_t desired_replication_level,
-                         struct GNUNET_PeerIdentity best_known_dest,
-                         struct GNUNET_HashCode intermediate_trail_id,
-                         struct GNUNET_PeerIdentity *target_peer,
-                         uint32_t hop_count,
-                         uint32_t put_path_length,
-                         struct GNUNET_PeerIdentity *put_path,
-                         struct GNUNET_TIME_Absolute expiration_time,
-                         const void *data, size_t data_size)
-{
-  struct PeerPutMessage *ppm;
-  struct GNUNET_MQ_Envelope *env;
-  struct FriendInfo *target_friend;
-  struct GNUNET_PeerIdentity *pp;
-  size_t msize;
-
-  msize = put_path_length * sizeof (struct GNUNET_PeerIdentity) + data_size;
-  if (msize + sizeof (struct PeerPutMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    put_path_length = 0;
-    msize = data_size;
-  }
-  if (msize + sizeof (struct PeerPutMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  GNUNET_assert (NULL !=
-                 (target_friend =
-                  GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     target_peer)));
-  env = GNUNET_MQ_msg_extra (ppm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_PUT);
-  ppm->options = htonl (options);
-  ppm->block_type = htonl (block_type);
-  ppm->hop_count = htonl (hop_count + 1);
-  ppm->desired_replication_level = htonl (desired_replication_level);
-  ppm->expiration_time = GNUNET_TIME_absolute_hton (expiration_time);
-  ppm->best_known_destination = best_known_dest;
-  ppm->intermediate_trail_id = intermediate_trail_id;
-  ppm->key = *key;
-  ppm->put_path_length = htonl (put_path_length);
-  pp = (struct GNUNET_PeerIdentity *) &ppm[1];
-  GNUNET_memcpy (pp,
-                 put_path,
-                 put_path_length * sizeof (struct GNUNET_PeerIdentity));
-  GNUNET_memcpy (&pp[put_path_length],
-                 data,
-                 data_size);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Handle the put request from the client.
- *
- * @param block_type Type of the block
- * @param options Routing options
- * @param desired_replication_level Desired replication count
- * @param expiration_time When does the content expire
- * @param hop_count how many hops has this message traversed so far
- * @param bf Bloom filter of peers this PUT has already traversed
- * @param key Key for the content
- * @param put_path_length number of entries in put_path
- * @param put_path peers this request has traversed so far (if tracked)
- * @param data Content to store
- * @param data_size Size of content @a data in bytes
- * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
- */
-int
-GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type block_type,
-                           enum GNUNET_DHT_RouteOption options,
-                           uint32_t desired_replication_level,
-                           struct GNUNET_TIME_Absolute expiration_time,
-                           uint32_t hop_count,
-                           struct GNUNET_CONTAINER_BloomFilter *bf,
-                           const struct GNUNET_HashCode *key,
-                           unsigned int put_path_length,
-                           struct GNUNET_PeerIdentity *put_path,
-                           const void *data,
-                           size_t data_size)
-{
-  struct GNUNET_PeerIdentity best_known_dest;
-  struct GNUNET_HashCode intermediate_trail_id;
-  struct GNUNET_PeerIdentity next_hop;
-  uint64_t key_value;
-  struct Closest_Peer successor;
-
-  GNUNET_memcpy (&key_value,
-                 key,
-                 sizeof (uint64_t));
-  key_value = GNUNET_ntohll (key_value);
-  successor = find_local_best_known_next_hop (key_value,
-                                              GDS_FINGER_TYPE_NON_PREDECESSOR);
-  best_known_dest = successor.best_known_destination;
-  next_hop = successor.next_hop;
-  intermediate_trail_id = successor.trail_id;
-
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&best_known_dest,
-                                            &my_identity))
-  {
-    DEBUG("\n PUT_REQUEST_SUCCESSFUL for key = %s",
-          GNUNET_h2s(key));
-    /* I am the destination. */
-    GDS_DATACACHE_handle_put (expiration_time,
-                              key,
-                              0,
-                              NULL,
-                              block_type,
-                              data_size,
-                              data);
-    GDS_CLIENTS_process_put (options,
-                             block_type,
-                             0,
-                             ntohl (desired_replication_level),
-                             1,
-                             &my_identity,
-                             expiration_time,
-                             key,
-                             data,
-                             data_size);
-    return GNUNET_NO;
-  }
-  /* In case we are sending the request to  a finger, then send across all of its
-   trail.*/
- GDS_NEIGHBOURS_send_put (key,
-                          block_type,
-                          options,
-                          desired_replication_level,
-                          best_known_dest,
-                          intermediate_trail_id,
-                          &next_hop,
-                          0,
-                          1,
-                          &my_identity,
-                          expiration_time,
-                          data,
-                          data_size);
- return GNUNET_OK;
-}
-
-
-/**
- * Construct a Get message and send it to target_peer.
- *
- * @param key Key for the content
- * @param block_type Type of the block
- * @param options Routing options
- * @param desired_replication_level Desired replication count
- * @param best_known_dest Peer which should get this message. Same as target peer
- *                        if best_known_dest is a friend else its a finger.
- * @param intermediate_trail_id  Trail id to reach to @a best_known_dest
- *                              in case it is a finger else set to 0.
- * @param target_peer Peer to which this message will be forwarded.
- * @param hop_count Number of hops traversed so far.
- * @param data Content to store
- * @param data_size Size of content @a data in bytes
- * @param get_path_length Total number of peers in @a get_path
- * @param get_path Number of peers traversed so far
- */
-void
-GDS_NEIGHBOURS_send_get (const struct GNUNET_HashCode *key,
-                         enum GNUNET_BLOCK_Type block_type,
-                         enum GNUNET_DHT_RouteOption options,
-                         uint32_t desired_replication_level,
-                         const struct GNUNET_PeerIdentity *best_known_dest,
-                         const struct GNUNET_HashCode *intermediate_trail_id,
-                         const struct GNUNET_PeerIdentity *target_peer,
-                         uint32_t hop_count,
-                         uint32_t get_path_length,
-                         const struct GNUNET_PeerIdentity *get_path)
-{
-  struct PeerGetMessage *pgm;
-  struct GNUNET_MQ_Envelope *env;
-  struct FriendInfo *target_friend;
-  size_t msize;
-
-  msize = get_path_length * sizeof (struct GNUNET_PeerIdentity);
-  if (msize + sizeof (struct PeerGetMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  GNUNET_assert (NULL !=
-                 (target_friend =
-                  GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     target_peer)));
-  env = GNUNET_MQ_msg_extra (pgm,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_GET);
-  pgm->get_path_length = htonl (get_path_length);
-  pgm->best_known_destination = *best_known_dest;
-  pgm->key = *key;
-  pgm->intermediate_trail_id = *intermediate_trail_id;
-  pgm->hop_count = htonl (hop_count + 1);
-  pgm->get_path_length = htonl (get_path_length);
-  GNUNET_memcpy (&pgm[1],
-                 get_path,
-                 msize);
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Send the get result to requesting client.
- *
- * @param key Key of the requested data.
- * @param block_type Block type
- * @param target_peer Next peer to forward the message to.
- * @param source_peer Peer which has the data for the key.
- * @param put_path_length Number of peers in @a put_path
- * @param put_path Path taken to put the data at its stored location.
- * @param get_path_length Number of peers in @a get_path
- * @param get_path Path taken to reach to the location of the key.
- * @param expiration When will this result expire?
- * @param data Payload to store
- * @param data_size Size of the @a data
- */
-void
-GDS_NEIGHBOURS_send_get_result (const struct GNUNET_HashCode *key,
-                                enum GNUNET_BLOCK_Type block_type,
-                                const struct GNUNET_PeerIdentity *target_peer,
-                                const struct GNUNET_PeerIdentity *source_peer,
-                                unsigned int put_path_length,
-                                const struct GNUNET_PeerIdentity *put_path,
-                                unsigned int get_path_length,
-                                const struct GNUNET_PeerIdentity *get_path,
-                                struct GNUNET_TIME_Absolute expiration,
-                                const void *data, size_t data_size)
-{
-  struct PeerGetResultMessage *get_result;
-  struct GNUNET_PeerIdentity *paths;
-  struct GNUNET_MQ_Envelope *env;
-  struct FriendInfo *target_friend;
-  int current_path_index;
-  size_t msize;
-
-  msize = (put_path_length + get_path_length) * sizeof (struct GNUNET_PeerIdentity) +
-    data_size;
-  if (msize + sizeof (struct PeerGetResultMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    put_path_length = 0;
-    msize = data_size;
-  }
-  if (msize + sizeof (struct PeerGetResultMessage)
-      >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE)
-  {
-    GNUNET_break(0);
-    return;
-  }
-  current_path_index = 0;
-  if (get_path_length > 0)
-  {
-    current_path_index = search_my_index (get_path,
-                                          get_path_length);
-    if (-1 == current_path_index)
-    {
-      GNUNET_break (0);
-      return;
-    }
-    if ((get_path_length + 1) == current_path_index)
-    {
-      DEBUG ("Peer found twice in get path. Not allowed \n");
-      GNUNET_break (0);
-      return;
-    }
-  }
-  if (0 == current_path_index)
-  {
-    DEBUG ("GET_RESULT TO CLIENT KEY = %s, Peer = %s",
-           GNUNET_h2s (key),
-           GNUNET_i2s (&my_identity));
-    GDS_CLIENTS_handle_reply (expiration,
-                              key,
-                              get_path_length,
-                              get_path,
-                              put_path_length,
-                              put_path,
-                              block_type,
-                              data_size,
-                              data);
-    return;
-  }
-  env = GNUNET_MQ_msg_extra (get_result,
-                             msize,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_GET_RESULT);
-  get_result->key = *key;
-  get_result->querying_peer = *source_peer;
-  get_result->expiration_time = GNUNET_TIME_absolute_hton (expiration);
-  get_result->get_path_length = htonl (get_path_length);
-  get_result->put_path_length = htonl (put_path_length);
-  paths = (struct GNUNET_PeerIdentity *)&get_result[1];
-  GNUNET_memcpy (paths,
-                 put_path,
-                 put_path_length * sizeof (struct GNUNET_PeerIdentity));
-  GNUNET_memcpy (&paths[put_path_length],
-                 get_path,
-                 get_path_length * sizeof (struct GNUNET_PeerIdentity));
-  GNUNET_memcpy (&paths[put_path_length + get_path_length],
-                 data,
-                 data_size);
-
-  GNUNET_assert (NULL !=
-                (target_friend =
-                 GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                    &get_path[current_path_index - 1])));
-  GNUNET_MQ_send (target_friend->mq,
-                  env);
-}
-
-
-/**
- * Handle a result for a GET operation.
- *
- * @param cls closure
- * @param type type of the block
- * @param expiration_time when does the content expire
- * @param key key for the content
- * @param put_path_length number of entries in @a put_path
- * @param put_path peers the original PUT traversed (if tracked)
- * @param get_path_length number of entries in @a get_path
- * @param get_path peers this reply has traversed so far (if tracked)
- * @param data payload of the reply
- * @param data_size number of bytes in @a data
- */
-static void
-get_cb (void *cls,
-        enum GNUNET_BLOCK_Type type,
-        struct GNUNET_TIME_Absolute expiration_time,
-        const struct GNUNET_HashCode *key,
-        unsigned int put_path_length,
-        const struct GNUNET_PeerIdentity *put_path,
-        unsigned int get_path_length,
-        const struct GNUNET_PeerIdentity *get_path,
-        const void *data,
-        size_t data_size)
-{
-  struct GNUNET_PeerIdentity *target_peer = cls;
-  // FIXME: inline?
-  GDS_NEIGHBOURS_send_get_result (key,
-                                  type,
-                                  target_peer,
-                                  &my_identity,
-                                  put_path_length,
-                                  put_path,
-                                  1,
-                                  &my_identity,
-                                  expiration_time,
-                                  data,
-                                  data_size);
-}
-
-
-/**
- * Perform a GET operation.  Forwards the given request to other
- * peers.  Does not lookup the key locally.  May do nothing if this is
- * the only peer in the network (or if we are the closest peer in the
- * network).
- *
- * @param block_type type of the block
- * @param options routing options
- * @param desired_replication_level desired replication count
- * @param hop_count how many hops did this request traverse so far?
- * @param key key for the content
- * @param xquery extended query
- * @param xquery_size number of bytes in @a xquery
- * @param reply_bf bloomfilter to filter duplicates
- * @param reply_bf_mutator mutator for @a reply_bf
- * @param peer_bf filter for peers not to select (again, updated)
- * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
- */
-int
-GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type block_type,
-                           enum GNUNET_DHT_RouteOption options,
-                           uint32_t desired_replication_level,
-                           uint32_t hop_count,
-                           const struct GNUNET_HashCode *key,
-                           const void *xquery, size_t xquery_size,
-                           const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
-                           uint32_t reply_bf_mutator,
-                           struct GNUNET_CONTAINER_BloomFilter *peer_bf)
-{
-  struct Closest_Peer successor;
-  struct GNUNET_PeerIdentity best_known_dest;
-  struct GNUNET_HashCode intermediate_trail_id;
-  uint64_t key_value;
-
-  GNUNET_memcpy (&key_value,
-                 key,
-                 sizeof (uint64_t));
-  key_value = GNUNET_ntohll (key_value);
-  successor = find_local_best_known_next_hop (key_value,
-                                              GDS_FINGER_TYPE_NON_PREDECESSOR);
-  best_known_dest = successor.best_known_destination;
-  intermediate_trail_id = successor.trail_id;
-
-  /* I am the destination. I have the data. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &best_known_dest))
-  {
-    GDS_DATACACHE_handle_get (key,
-                              block_type,
-                              NULL,
-                              0,
-                              NULL,
-                              0,
-                              &get_cb,
-                              NULL);
-    return GNUNET_NO;
-  }
-
-  GDS_NEIGHBOURS_send_get (key,
-                           block_type,
-                           options,
-                           desired_replication_level,
-                           &best_known_dest,
-                           &intermediate_trail_id,
-                           &successor.next_hop,
-                           0,
-                           1,
-                           &my_identity);
-  return GNUNET_OK;
-}
-
-
-/**
- * Randomly choose one of your friends (which is not congested and have not crossed
- * trail threshold) from the friend_peermap
- * @return Friend Randomly chosen friend.
- *         NULL in case friend peermap is empty, or all the friends are either
- *              congested or have crossed trail threshold.
- */
-static struct FriendInfo *
-select_random_friend ()
-{
-  unsigned int current_size;
-  uint32_t index;
-  unsigned int j = 0;
-  struct GNUNET_CONTAINER_MultiPeerMapIterator *iter;
-  struct GNUNET_PeerIdentity key_ret;
-  struct FriendInfo *friend;
-
-  current_size = GNUNET_CONTAINER_multipeermap_size (friend_peermap);
-
-  /* No friends.*/
-  if (0 == current_size)
-    return NULL;
-
-  index = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, current_size);
-  iter = GNUNET_CONTAINER_multipeermap_iterator_create (friend_peermap);
-
-  /* Iterate till you don't reach to index. */
-  for (j = 0; j < index ; j++)
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multipeermap_iterator_next (iter, NULL, NULL));
-
-  do
-  {
-    /* Reset the index in friend peermap to 0 as we reached to the end. */
-    if (j == current_size)
-    {
-      j = 0;
-      GNUNET_CONTAINER_multipeermap_iterator_destroy (iter);
-      iter = GNUNET_CONTAINER_multipeermap_iterator_create (friend_peermap);
-
-    }
-
-    /* Get the friend stored at the index, j*/
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multipeermap_iterator_next (iter,
-                                                                &key_ret,
-                                                                (const void **)&friend));
-
-    /* This friend is not congested and has not crossed trail threshold. */
-    if ((friend->trails_count < TRAILS_THROUGH_FRIEND_THRESHOLD) &&
-        (0 == GNUNET_TIME_absolute_get_remaining (friend->congestion_timestamp).rel_value_us))
-    {
-      break;
-    }
-    friend = NULL;
-    j++;
-  } while (j != index);
-
-  GNUNET_CONTAINER_multipeermap_iterator_destroy (iter);
-  return friend;
-}
-
-
-/**
- * Compute 64 bit value of finger_identity corresponding to a finger index using
- * Chord formula.
- * For all fingers, n.finger[i] = n + pow (2,i),
- * For predecessor, n.finger[PREDECESSOR_FINGER_ID] = n - 1, where
- * n = my_identity, i = finger_index, n.finger[i] = 64 bit finger value
- *
- * @param finger_index Index corresponding to which we calculate 64 bit value.
- * @return 64 bit value.
- */
-static uint64_t
-compute_finger_identity_value (unsigned int finger_index)
-{
-  uint64_t my_id64;
-
-  GNUNET_memcpy (&my_id64,
-                 &my_identity,
-                 sizeof (uint64_t));
-  my_id64 = GNUNET_ntohll (my_id64);
-
-  /* Are we looking for immediate predecessor? */
-  if (PREDECESSOR_FINGER_ID == finger_index)
-    return (my_id64 - 1);
-  uint64_t add = (uint64_t)1 << finger_index;
-  return (my_id64 + add);
-}
-
-
-/**
- * Choose a random friend. Calculate the next finger identity to search,from
- * current_search_finger_index. Start looking for the trail to reach to
- * finger identity through this random friend.
- *
- * @param cls closure for this task
- */
-static void
-send_find_finger_trail_message (void *cls)
-{
-  struct FriendInfo *target_friend;
-  struct GNUNET_HashCode trail_id;
-  struct GNUNET_HashCode intermediate_trail_id;
-  unsigned int is_predecessor = 0;
-  uint64_t finger_id_value;
-
-  /* Schedule another send_find_finger_trail_message task. After one round of
-   * finger search, this time is exponentially backoff. */
-  find_finger_trail_task_next_send_time.rel_value_us =
-      find_finger_trail_task_next_send_time.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_FIND_FINGER_TRAIL_INTERVAL.rel_value_us);
-  find_finger_trail_task =
-      GNUNET_SCHEDULER_add_delayed (find_finger_trail_task_next_send_time,
-                                    &send_find_finger_trail_message,
-                                    NULL);
-
-  /* No space in my routing table. (Source and destination peers also store entries
-   * in their routing table).  */
-  if (GNUNET_YES == GDS_ROUTING_threshold_reached())
-    return;
-
-  target_friend = select_random_friend ();
-  if (NULL == target_friend)
-    return;
-
-  finger_id_value = compute_finger_identity_value (current_search_finger_index);
-  if (PREDECESSOR_FINGER_ID == current_search_finger_index)
-    is_predecessor = 1;
-
-  /* Generate a unique trail id for trail we are trying to setup. */
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_STRONG,
-                              &trail_id,
-                              sizeof (trail_id));
-  memset (&intermediate_trail_id,
-          0,
-          sizeof (struct GNUNET_HashCode));
-  GDS_NEIGHBOURS_send_trail_setup (&my_identity,
-                                   finger_id_value,
-                                   target_friend->id,
-                                   target_friend,
-                                   0, NULL,
-                                   is_predecessor,
-                                   &trail_id,
-                                   &intermediate_trail_id);
-}
-
-
-/**
- * In case there are already maximum number of possible trails to reach to a
- * finger, then check if the new trail's length is lesser than any of the
- * existing trails.
- * If yes then replace that old trail by new trail.
- *
- * Note: Here we are taking length as a parameter to choose the best possible
- * trail, but there could be other parameters also like:
- * 1. duration of existence of a trail - older the better.
- * 2. if the new trail is completely disjoint than the
- *    other trails, then may be choosing it is better.
- *
- * @param finger Finger
- * @param new_finger_trail List of peers to reach from me to @a finger, NOT
- *                         including the endpoints.
- * @param new_finger_trail_length Total number of peers in @a new_finger_trail
- * @param new_finger_trail_id Unique identifier of @a new_finger_trail.
- */
-static void
-select_and_replace_trail (struct FingerInfo *finger,
-                          const struct GNUNET_PeerIdentity *new_trail,
-                          unsigned int new_trail_length,
-                          const struct GNUNET_HashCode *new_trail_id)
-{
-  struct Trail *current_trail;
-  unsigned int largest_trail_length;
-  unsigned int largest_trail_index;
-  struct Trail_Element *trail_element;
-  const struct GNUNET_PeerIdentity *next_hop;
-  unsigned int i;
-
-  largest_trail_length = new_trail_length;
-  largest_trail_index = MAXIMUM_TRAILS_PER_FINGER + 1;
-
-  GNUNET_assert (MAXIMUM_TRAILS_PER_FINGER == finger->trails_count);
-
-  for (i = 0; i < finger->trails_count; i++)
-  {
-    current_trail = &finger->trail_list[i];
-    GNUNET_assert (GNUNET_YES == current_trail->is_present);
-    if (current_trail->trail_length > largest_trail_length)
-    {
-      largest_trail_length = current_trail->trail_length;
-      largest_trail_index = i;
-    }
-  }
-
-  /* New trail is not better than existing ones. Send trail teardown. */
-  if (largest_trail_index == (MAXIMUM_TRAILS_PER_FINGER + 1))
-  {
-    next_hop = GDS_ROUTING_get_next_hop (new_trail_id,
-                                         GDS_ROUTING_SRC_TO_DEST);
-    GDS_ROUTING_remove_trail (new_trail_id);
-    GDS_NEIGHBOURS_send_trail_teardown (new_trail_id,
-                                        GDS_ROUTING_SRC_TO_DEST,
-                                        next_hop);
-    return;
-  }
-
-  /* Send trail teardown message across the replaced trail. */
-  struct Trail *replace_trail = &finger->trail_list[largest_trail_index];
-  next_hop = GDS_ROUTING_get_next_hop (&replace_trail->trail_id,
-                                       GDS_ROUTING_SRC_TO_DEST);
-  GNUNET_assert (GNUNET_YES == GDS_ROUTING_remove_trail (&replace_trail->trail_id));
-  GDS_NEIGHBOURS_send_trail_teardown (&replace_trail->trail_id,
-                                      GDS_ROUTING_SRC_TO_DEST,
-                                      next_hop);
-
-  /* Free the trail. */
-  while (NULL != (trail_element = replace_trail->trail_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (replace_trail->trail_head,
-                                 replace_trail->trail_tail,
-                                 trail_element);
-    GNUNET_free_non_null (trail_element);
-  }
-
-  /* Add new trial at that location. */
-  replace_trail->is_present = GNUNET_YES;
-  replace_trail->trail_length = new_trail_length;
-  replace_trail->trail_id = *new_trail_id;
-
-  for (i = 0; i < new_trail_length; i++)
-  {
-    struct Trail_Element *element = GNUNET_new (struct Trail_Element);
-    element->peer = new_trail[i];
-
-    GNUNET_CONTAINER_DLL_insert_tail (replace_trail->trail_head,
-                                      replace_trail->trail_tail,
-                                      element);
-  }
-  /* FIXME: URGENT Are we adding the trail back to the list. */
-}
-
-
-/**
- * Check if the new trail to reach to finger is unique or do we already have
- * such a trail present for finger.
- * @param existing_finger Finger identity
- * @param new_trail New trail to reach @a existing_finger
- * @param trail_length Total number of peers in new_trail.
- * @return #GNUNET_YES if the new trail is unique
- *         #GNUNET_NO if same trail is already present.
- */
-static int
-is_new_trail_unique (struct FingerInfo *existing_finger,
-                     const struct GNUNET_PeerIdentity *new_trail,
-                     unsigned int trail_length)
-{
-  struct Trail *current_trail;
-  struct Trail_Element *trail_element;
-  int i;
-  int j;
-
-  GNUNET_assert (existing_finger->trails_count > 0);
-
-  /* Iterate over list of trails. */
-  for (i = 0; i < existing_finger->trails_count; i++)
-  {
-    current_trail = &(existing_finger->trail_list[i]);
-    if(GNUNET_NO == current_trail->is_present)
-      continue;
-
-    /* New trail and existing trail length are not same. */
-    if (current_trail->trail_length != trail_length)
-    {
-      return GNUNET_YES;
-    }
-
-    trail_element = current_trail->trail_head;
-    for (j = 0; j < current_trail->trail_length; j++)
-    {
-      if (0 != GNUNET_CRYPTO_cmp_peer_identity (&new_trail[j],
-                                                &trail_element->peer))
-      {
-        return GNUNET_YES;
-      }
-      trail_element = trail_element->next;
-    }
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * FIXME; In case of multiple trails, we may have a case where a trail from in
- * between has been removed, then we should try to find a free slot , not simply
- * add a trail at then end of the list.
- * Add a new trail at a free slot in trail array of existing finger.
- * @param existing_finger Finger
- * @param new_finger_trail New trail from me to finger, NOT including endpoints
- * @param new_finger_trail_length Total number of peers in @a new_finger_trail
- * @param new_finger_trail_id Unique identifier of the trail.
- */
-static void
-add_new_trail (struct FingerInfo *existing_finger,
-               const struct GNUNET_PeerIdentity *new_trail,
-               unsigned int new_trail_length,
-               const struct GNUNET_HashCode *new_trail_id)
-{
-  struct FriendInfo *friend;
-  struct Trail *trail;
-  unsigned int i;
-  int free_slot = -1;
-
-  if (GNUNET_NO == is_new_trail_unique (existing_finger,
-                                        new_trail,
-                                        new_trail_length))
-    return;
-
-  for (i = 0; i < existing_finger->trails_count; i++)
-  {
-    if (GNUNET_NO == existing_finger->trail_list[i].is_present)
-    {
-      free_slot = i;
-      break;
-    }
-  }
-
-  if (-1 == free_slot)
-    free_slot = i;
-
-  trail = &existing_finger->trail_list[free_slot];
-  GNUNET_assert (GNUNET_NO == trail->is_present);
-  trail->trail_id = *new_trail_id;
-  trail->trail_length = new_trail_length;
-  existing_finger->trails_count++;
-  trail->is_present = GNUNET_YES;
-  if (0 == new_trail_length)
-  {
-    friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                &existing_finger->finger_identity);
-  }
-  else
-  {
-    friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                &new_trail[0]);
-  }
-  GNUNET_assert (NULL != friend);
-  friend->trails_count++;
-  for (i = 0; i < new_trail_length; i++)
-  {
-    struct Trail_Element *element;
-
-    element = GNUNET_new (struct Trail_Element);
-    element->peer = new_trail[i];
-    GNUNET_CONTAINER_DLL_insert_tail (trail->trail_head,
-                                      trail->trail_tail,
-                                      element);
-  }
-
-  existing_finger->trail_list[free_slot].trail_head = trail->trail_head;
-  existing_finger->trail_list[free_slot].trail_tail = trail->trail_tail;
-  existing_finger->trail_list[free_slot].trail_length = new_trail_length;
-  existing_finger->trail_list[free_slot].trail_id = *new_trail_id;
-  existing_finger->trail_list[free_slot].is_present = GNUNET_YES;
-}
-
-
-#if 0
-/**
- * FIXME; In case of multiple trails, we may have a case where a trail from in
- * between has been removed, then we should try to find a free slot , not simply
- * add a trail at then end of the list.
- * Add a new trail at a free slot in trail array of existing finger.
- * @param existing_finger Finger
- * @param new_finger_trail New trail from me to finger, NOT including endpoints
- * @param new_finger_trail_length Total number of peers in @a new_finger_trail
- * @param new_finger_trail_id Unique identifier of the trail.
- */
-static void
-add_new_trail (struct FingerInfo *existing_finger,
-               const struct GNUNET_PeerIdentity *new_trail,
-               unsigned int new_trail_length,
-               const struct GNUNET_HashCode *new_trail_id)
-{
-  struct Trail *trail;
-  struct FriendInfo *first_friend;
-  int i;
-  int index;
-
-  if (GNUNET_NO == is_new_trail_unique (existing_finger,
-                                        new_trail,
-                                        new_trail_length))
-    return;
-
-  index = existing_finger->trails_count;
-  trail = &existing_finger->trail_list[index];
-  GNUNET_assert (GNUNET_NO == trail->is_present);
-  trail->trail_id = *new_trail_id;
-  trail->trail_length = new_trail_length;
-  existing_finger->trails_count++;
-  trail->is_present = GNUNET_YES;
-
-  GNUNET_assert (NULL == (GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                             &existing_finger->finger_identity)));
-  /* If finger is a friend then we never call this function. */
-  GNUNET_assert (new_trail_length > 0);
-
-  first_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                    &new_trail[0]);
-  first_friend->trails_count++;
-
-  for (i = 0; i < new_trail_length; i++)
-  {
-    struct Trail_Element *element;
-
-    element = GNUNET_new (struct Trail_Element);
-    element->peer = new_trail[i];
-    GNUNET_CONTAINER_DLL_insert_tail (trail->trail_head,
-                                      trail->trail_tail,
-                                      element);
-  }
-  /* Do we need to add trail head and trail tail in the trail list itearator.*/
-  existing_finger->trail_list[index].trail_head = trail->trail_head;
-  existing_finger->trail_list[index].trail_tail = trail->trail_tail;
-  existing_finger->trail_list[index].trail_length = new_trail_length;
-  existing_finger->trail_list[index].trail_id = *new_trail_id;
-  existing_finger->trail_list[index].is_present = GNUNET_YES;
-}
-#endif
-
-/**
- * Get the next hop to send trail teardown message from routing table and
- * then delete the entry from routing table. Send trail teardown message for a
- * specific trail of a finger.
- * @param finger Finger whose trail is to be removed.
- * @param trail List of peers in trail from me to a finger, NOT including
- *              endpoints.
- */
-static void
-send_trail_teardown (struct FingerInfo *finger,
-                     struct Trail *trail)
-{
-  struct FriendInfo *friend;
-  const struct GNUNET_PeerIdentity *next_hop;
-
-  next_hop = GDS_ROUTING_get_next_hop (&trail->trail_id,
-                                       GDS_ROUTING_SRC_TO_DEST);
-  if (NULL == next_hop)
-  {
-//    DEBUG(" NO ENTRY FOUND IN %s ROUTING TABLE for trail id %s, line=%d,traillength = %d",
-//            GNUNET_i2s(&my_identity), GNUNET_h2s(&trail->trail_id), __LINE__,trail->trail_length);
-    return;
-  }
-  GNUNET_assert (0 != GNUNET_CRYPTO_cmp_peer_identity (&finger->finger_identity,
-                                                       &my_identity));
-
-  GNUNET_assert(GNUNET_YES == trail->is_present);
-  if (trail->trail_length > 0)
-  {
-    friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                &trail->trail_head->peer);
-  }
-  else
-  {
-    friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                &finger->finger_identity);
-  }
-
-  if(NULL == friend)
-  {
-    DEBUG ("\n LINE NO: = %d, Friend not found for trail id  %s of peer %s trail length = %d",
-           __LINE__,
-           GNUNET_h2s (&trail->trail_id),
-           GNUNET_i2s(&my_identity),
-           trail->trail_length);
-    return;
-  }
-  if ( (0 != GNUNET_CRYPTO_cmp_peer_identity (next_hop,
-                                              friend->id) ) &&
-       (0 == trail->trail_length))
-  {
-     DEBUG ("\n LINE NO: = %d, Friend not found for trail id  %s of peer %s trail length = %d",
-            __LINE__,
-            GNUNET_h2s (&trail->trail_id),
-            GNUNET_i2s (&my_identity),
-            trail->trail_length);
-    return;
-  }
-  GNUNET_assert (GNUNET_YES ==
-                 GDS_ROUTING_remove_trail (&trail->trail_id));
-  friend->trails_count--;
-  GDS_NEIGHBOURS_send_trail_teardown (&trail->trail_id,
-                                      GDS_ROUTING_SRC_TO_DEST,
-                                      friend->id);
-}
-
-
-/**
- * Send trail teardown message across all the trails to reach to finger.
- * @param finger Finger whose all the trail should be freed.
- */
-static void
-send_all_finger_trails_teardown (struct FingerInfo *finger)
-{
-  for (unsigned int i = 0; i < finger->trails_count; i++)
-  {
-    struct Trail *trail;
-
-    trail = &finger->trail_list[i];
-    if (GNUNET_YES == trail->is_present)
-    {
-      send_trail_teardown (finger, trail);
-      trail->is_present = GNUNET_NO;
-    }
-  }
-}
-
-
-/**
- * Free a specific trail
- * @param trail List of peers to be freed.
- */
-static void
-free_trail (struct Trail *trail)
-{
-  struct Trail_Element *trail_element;
-
-  while (NULL != (trail_element = trail->trail_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (trail->trail_head,
-                                 trail->trail_tail,
-                                 trail_element);
-    GNUNET_free_non_null (trail_element);
-  }
-  trail->trail_head = NULL;
-  trail->trail_tail = NULL;
-}
-
-
-/**
- * Free finger and its trail.
- *
- * @param finger Finger to be freed.
- * @param finger_table_index Index at which finger is stored.
- */
-static void
-free_finger (struct FingerInfo *finger,
-             unsigned int finger_table_index)
-{
-  struct Trail *trail;
-
-  for (unsigned int i = 0; i < finger->trails_count; i++)
-  {
-    trail = &finger->trail_list[i];
-    if (GNUNET_NO == trail->is_present)
-      continue;
-
-    if (trail->trail_length > 0)
-      free_trail (trail);
-    trail->is_present = GNUNET_NO;
-  }
-
-  finger->is_present = GNUNET_NO;
-  memset (&finger_table[finger_table_index],
-          0,
-          sizeof (finger_table[finger_table_index]));
-}
-
-
-/**
- * Add a new entry in finger table at finger_table_index.
- * In case I am my own finger, then we don't have a trail. In case of a friend,
- * we have a trail with unique id and '0' trail length.
- * In case a finger is a friend, then increment the trails count of the friend.
- *
- * @param finger_identity Peer Identity of new finger
- * @param finger_trail Trail to reach from me to finger (excluding both end points).
- * @param finger_trail_length Total number of peers in @a finger_trail.
- * @param trail_id Unique identifier of the trail.
- * @param finger_table_index Index in finger table.
- */
-static void
-add_new_finger (const struct GNUNET_PeerIdentity *finger_identity,
-                const struct GNUNET_PeerIdentity *finger_trail,
-                unsigned int finger_trail_length,
-                const struct GNUNET_HashCode *trail_id,
-                unsigned int finger_table_index)
-{
-  struct FingerInfo *new_entry;
-  struct FriendInfo *first_trail_hop;
-  struct Trail *trail;
-  unsigned int i;
-
-  new_entry = GNUNET_new (struct FingerInfo);
-  new_entry->finger_identity = *finger_identity;
-  new_entry->finger_table_index = finger_table_index;
-  new_entry->is_present = GNUNET_YES;
-
-  /* If the new entry is my own identity. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            finger_identity))
-  {
-    new_entry->trails_count = 0;
-    finger_table[finger_table_index] = *new_entry;
-    GNUNET_free (new_entry);
-    return;
-  }
-
-  /* Finger is a friend. */
-  if (0 == finger_trail_length)
-  {
-    new_entry->trail_list[0].trail_id = *trail_id;
-    new_entry->trails_count = 1;
-    new_entry->trail_list[0].is_present = GNUNET_YES;
-    new_entry->trail_list[0].trail_length = 0;
-    new_entry->trail_list[0].trail_head = NULL;
-    new_entry->trail_list[0].trail_tail = NULL;
-    finger_table[finger_table_index] = *new_entry;
-    GNUNET_assert (NULL !=
-                  (first_trail_hop =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      finger_identity)));
-
-    first_trail_hop->trails_count++;
-    GNUNET_free (new_entry);
-    return;
-  }
-
-  GNUNET_assert (NULL !=
-                 (first_trail_hop =
-                  GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &finger_trail[0])));
-  new_entry->trails_count = 1;
-  first_trail_hop->trails_count++;
-  /* Copy the finger trail into trail. */
-  trail = &new_entry->trail_list[0];
-  for(i = 0; i < finger_trail_length; i++)
-  {
-    struct Trail_Element *element = GNUNET_new (struct Trail_Element);
-
-    element->next = NULL;
-    element->prev = NULL;
-    element->peer = finger_trail[i];
-    GNUNET_CONTAINER_DLL_insert_tail (trail->trail_head,
-                                      trail->trail_tail,
-                                      element);
-  }
-
-  /* Add trail to trail list. */
-  trail->trail_length = finger_trail_length;
-  trail->trail_id = *trail_id;
-  trail->is_present = GNUNET_YES;
-  finger_table[finger_table_index] = *new_entry;
-  GNUNET_free (new_entry);
-}
-
-
-/**
- * Periodic task to verify current successor. There can be multiple trails to reach
- * to successor, choose the shortest one and send verify successor message
- * across that trail.
- *
- * @param cls closure for this task
- */
-static void
-send_verify_successor_message (void *cls)
-{
-  struct FriendInfo *target_friend;
-  struct Trail *trail;
-  struct Trail_Element *element;
-  unsigned int trail_length;
-  unsigned int i = 0;
-  struct FingerInfo *successor;
-
-  successor = &finger_table[0];
-
-  /* This task will be scheduled when the result for Verify Successor is received. */
-  send_verify_successor_task = NULL;
-
-  /* When verify successor is being called for first time *for current context*
-   * cls will be NULL. If send_verify_successor_retry_task is not NO_TASK, we
-   * must cancel the retry task scheduled for verify_successor of previous
-   * context.
-   */
-  if (NULL == cls)
-  {
-    /* FIXME: Here we are scheduling a new verify successor task, as we
-     got a new successor. But a send verify successor task may be in progress.
-     1. We need to be sure that this is indeed a new successor. As this function
-     is called even if we add a new trail to reach t old successor.
-     2. Assuming the new successor is different, then verify successor message
-     * to old successor may be following stages.
-     * --> Waiting for verify successor result. Don't wait anymore. there is
-     *     no trail to reach from old successor to me, hence, routing
-     *     lookup will fail.
-     * --> Waiting for notify confirmation. again don't wait for it. notify
-     *    confirmation will not succeded.
-     */
-    if (send_verify_successor_retry_task != NULL)
-    {
-      /* FIXME: Are we scheduling retry task as soon as we send verify message.
-       If yes then here before making this task, first check if the message
-       is for the same peer again. */
-      struct VerifySuccessorContext *old_ctx =
-          GNUNET_SCHEDULER_cancel(send_verify_successor_retry_task);
-      /* old_ctx must not be NULL, as the retry task had been scheduled */
-      GNUNET_assert(NULL != old_ctx);
-      GNUNET_free(old_ctx);
-      /* FIXME: Why don't we reset the task to NO_TASK here? */
-    }
-
-    struct VerifySuccessorContext *ctx;
-    ctx = GNUNET_new (struct VerifySuccessorContext);
-
-    ctx->num_retries_scheduled++;
-    send_verify_successor_retry_task =
-        GNUNET_SCHEDULER_add_delayed (verify_successor_retry_time,
-                                      &send_verify_successor_message,
-                                      ctx);
-  }
-  else
-  {
-    /* This is a retry attempt for verify_successor for a previous context */
-    struct VerifySuccessorContext *ctx;
-
-    ctx = cls;
-    ctx->num_retries_scheduled++;
-    send_verify_successor_retry_task =
-        GNUNET_SCHEDULER_add_delayed (verify_successor_retry_time,
-                                      &send_verify_successor_message,
-                                      ctx);
-  }
-
-  /* Among all the trails to reach to successor, select first one which is present.*/
-  for (i = 0; i < successor->trails_count; i++)
-  {
-    trail = &successor->trail_list[i];
-    if (GNUNET_YES == trail->is_present)
-      break;
-  }
-
-  /* No valid trail found to reach to successor. */
-  if (i == successor->trails_count)
-    return;
-
-  GNUNET_assert(0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                                      &successor->finger_identity));
-  /* Trail stored at this index. */
-  GNUNET_assert (GNUNET_YES == trail->is_present);
-  if (NULL == GDS_ROUTING_get_next_hop (&trail->trail_id,
-                                        GDS_ROUTING_SRC_TO_DEST))
-  {
-    DEBUG (" NO ENTRY FOUND IN %s ROUTING TABLE for trail id %s, line %u",
-           GNUNET_i2s (&my_identity),
-           GNUNET_h2s (&trail->trail_id),
-           __LINE__);
-    GNUNET_break(0);
-    return;
-  }
-  trail_length = trail->trail_length;
-  if (trail_length > 0)
-  {
-     /* Copy the trail into peer list. */
-    struct GNUNET_PeerIdentity peer_list[trail_length];
-
-    element = trail->trail_head;
-    for(i = 0; i < trail_length; i++)
-    {
-      peer_list[i] = element->peer;
-      element = element->next;
-    }
-    GNUNET_assert (NULL != (target_friend =
-                            GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                               &peer_list[0])));
-    GDS_NEIGHBOURS_send_verify_successor_message (&my_identity,
-                                                  &successor->finger_identity,
-                                                  &trail->trail_id,
-                                                  peer_list,
-                                                  trail_length,
-                                                  target_friend);
-  }
-  else
-  {
-    GNUNET_assert (NULL != (target_friend =
-                            GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                               &successor->finger_identity)));
-    GDS_NEIGHBOURS_send_verify_successor_message (&my_identity,
-                                                  &successor->finger_identity,
-                                                  &trail->trail_id,
-                                                  NULL,
-                                                  0,
-                                                  target_friend);
-  }
-}
-
-
-/**
- * FIXME: should this be a periodic task, incrementing the search finger index?
- * Update the current search finger index.
- * @a finger_identity
- * @a finger_table_index
- */
-static void
-update_current_search_finger_index (unsigned int finger_table_index)
-{
-  struct FingerInfo *successor;
-
-  /* FIXME correct this: only move current index periodically */
-  if (finger_table_index != current_search_finger_index)
-    return;
-
-  successor = &finger_table[0];
-  GNUNET_assert (GNUNET_YES == successor->is_present);
-
-  /* We were looking for immediate successor.  */
-  if (0 == current_search_finger_index)
-  {
-    current_search_finger_index = PREDECESSOR_FINGER_ID;
-    if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                              &successor->finger_identity))
-    {
-      if (NULL == send_verify_successor_task)
-      {
-        send_verify_successor_task
-          = GNUNET_SCHEDULER_add_now (&send_verify_successor_message,
-                                      NULL);
-      }
-    }
-    return;
-  }
-  current_search_finger_index--;
-}
-
-
-/**
- * Get the least significant bit set in val.
- *
- * @param val Value
- * @return Position of first bit set, 65 in case of error.
- */
-static unsigned int
-find_set_bit (uint64_t val)
-{
-  uint64_t i;
-  unsigned int pos;
-
-  i = 1;
-  pos = 0;
-
-  while (!(i & val))
-  {
-    i = i << 1;
-    pos++;
-    if (pos > 63)
-    {
-      GNUNET_break (0);
-      return 65;
-    }
-  }
-
-  if (val/i != 1)
-    return 65; /* Some other bit was set to 1 as well. */
-
-  return pos;
-}
-
-
-/**
- * Calculate finger_table_index from initial 64 bit finger identity value that
- * we send in trail setup message.
- * @param ultimate_destination_finger_value Value that we calculated from our
- *                                          identity and finger_table_index.
- * @param is_predecessor Is the entry for predecessor or not?
- * @return finger_table_index Value between 0 <= finger_table_index <= 64
- *         finger_table_index > PREDECESSOR_FINGER_ID, if error occurs.
- */
-static unsigned int
-get_finger_table_index (uint64_t ultimate_destination_finger_value,
-                        unsigned int is_predecessor)
-{
-  uint64_t my_id64;
-  uint64_t diff;
-  unsigned int finger_table_index;
-
-  GNUNET_memcpy (&my_id64, &my_identity, sizeof (uint64_t));
-  my_id64 = GNUNET_ntohll (my_id64);
-
-  /* Is this a predecessor finger? */
-  if (1 == is_predecessor)
-  {
-    diff =  my_id64 - ultimate_destination_finger_value;
-    if (1 == diff)
-      finger_table_index = PREDECESSOR_FINGER_ID;
-    else
-      finger_table_index = PREDECESSOR_FINGER_ID + 1; //error value
-
-  }
-  else
-  {
-    diff = ultimate_destination_finger_value - my_id64;
-    finger_table_index = find_set_bit (diff);
-  }
-  return finger_table_index;
-}
-
-
-/**
- * Remove finger and its associated data structures from finger table.
- * @param existing_finger Finger to be removed which is in finger table.
- * @param finger_table_index Index in finger table where @a existing_finger
- *                           is stored.
- */
-static void
-remove_existing_finger (struct FingerInfo *existing_finger,
-                        unsigned int finger_table_index)
-{
-  GNUNET_assert (GNUNET_YES == existing_finger->is_present);
-
-  /* If I am my own finger, then we have no trails. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&existing_finger->finger_identity,
-                                            &my_identity))
-  {
-    existing_finger->is_present = GNUNET_NO;
-    memset ((void *)&finger_table[finger_table_index], 0,
-            sizeof (finger_table[finger_table_index]));
-    return;
-  }
-
-  /* For all other fingers, send trail teardown across all the trails to reach
-   finger, and free the finger. */
-  send_all_finger_trails_teardown (existing_finger);
-  free_finger (existing_finger, finger_table_index);
-}
-
-
-/**
- * Check if there is already an entry in finger_table at finger_table_index.
- * We get the finger_table_index from 64bit finger value we got from the network.
- * -- If yes, then select the closest finger.
- *   -- If new and existing finger are same, then check if you can store more
- *      trails.
- *      -- If yes then add trail, else keep the best trails to reach to the
- *         finger.
- *   -- If the new finger is closest, remove the existing entry, send trail
- *      teardown message across all the trails to reach the existing entry.
- *      Add the new finger.
- *  -- If new and existing finger are different, and existing finger is closest
- *     then do nothing.
- * -- Update current_search_finger_index.
- * @param finger_identity Peer Identity of new finger
- * @param finger_trail Trail to reach the new finger
- * @param finger_trail_length Total number of peers in @a new_finger_trail.
- * @param is_predecessor Is this entry for predecessor in finger_table?
- * @param finger_value 64 bit value of finger identity that we got from network.
- * @param finger_trail_id Unique identifier of @finger_trail.
- */
-static void
-finger_table_add (const struct GNUNET_PeerIdentity *finger_identity,
-                  const struct GNUNET_PeerIdentity *finger_trail,
-                  unsigned int finger_trail_length,
-                  unsigned int is_predecessor,
-                  uint64_t finger_value,
-                  const struct GNUNET_HashCode *finger_trail_id)
-{
-  struct FingerInfo *existing_finger;
-  const struct GNUNET_PeerIdentity *closest_peer;
-  struct FingerInfo *successor;
-  unsigned int finger_table_index;
-
-  /* Get the finger_table_index corresponding to finger_value we got from network.*/
-  finger_table_index = get_finger_table_index (finger_value, is_predecessor);
-
-  /* Invalid finger_table_index. */
-  if ((finger_table_index > PREDECESSOR_FINGER_ID))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  /* Check if  new entry is same as successor. */
-  if ((0 != finger_table_index) &&
-      (PREDECESSOR_FINGER_ID != finger_table_index))
-  {
-    successor = &finger_table[0];
-    if (GNUNET_NO == successor->is_present)
-    {
-      GNUNET_break (0); //ASSERTION FAILS HERE. FIXME
-      return;
-    }
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (finger_identity,
-                                              &successor->finger_identity))
-    {
-      if (0 == fingers_round_count)
-      {
-         find_finger_trail_task_next_send_time =
-              GNUNET_TIME_STD_BACKOFF(find_finger_trail_task_next_send_time);
-      }
-      else
-        fingers_round_count--;
-      current_search_finger_index = 0;
-      GNUNET_STATISTICS_update (GDS_stats,
-                                gettext_noop
-                                ("# FINGERS_COUNT"), (int64_t) total_fingers_found,
-                                GNUNET_NO);
-      total_fingers_found  = 0;
-      return;
-    }
-
-    struct FingerInfo prev_finger;
-    prev_finger = finger_table[finger_table_index - 1];
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (finger_identity,
-                                              &prev_finger.finger_identity))
-    {
-       current_search_finger_index--;
-       return;
-    }
-  }
-
-  total_fingers_found++;
-  existing_finger = &finger_table[finger_table_index];
-
-  /* No entry present in finger_table for given finger map index. */
-  if (GNUNET_NO == existing_finger->is_present)
-  {
-    /* Shorten the trail if possible. */
-    add_new_finger (finger_identity,
-                    finger_trail,
-                    finger_trail_length,
-                    finger_trail_id,
-                    finger_table_index);
-    update_current_search_finger_index (finger_table_index);
-    return;
-  }
-
-  /* If existing entry and finger identity are not same. */
-  if (0 != GNUNET_CRYPTO_cmp_peer_identity (&existing_finger->finger_identity,
-                                            finger_identity))
-  {
-    closest_peer = select_closest_peer (&existing_finger->finger_identity,
-                                        finger_identity,
-                                        finger_value,
-                                        is_predecessor);
-
-    /* If the new finger is the closest peer. */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (finger_identity,
-                                              closest_peer))
-    {
-      remove_existing_finger (existing_finger,
-                              finger_table_index);
-      add_new_finger (finger_identity,
-                      finger_trail,
-                      finger_trail_length,
-                      finger_trail_id,
-                      finger_table_index);
-    }
-    else
-    {
-      /* Existing finger is the closest one. We need to send trail teardown
-         across the trail setup in routing table of all the peers. */
-      if (0 != GNUNET_CRYPTO_cmp_peer_identity (finger_identity,
-                                                &my_identity))
-      {
-        if (finger_trail_length > 0)
-          GDS_NEIGHBOURS_send_trail_teardown (finger_trail_id,
-                                              GDS_ROUTING_SRC_TO_DEST,
-                                              &finger_trail[0]);
-        else
-          GDS_NEIGHBOURS_send_trail_teardown (finger_trail_id,
-                                              GDS_ROUTING_SRC_TO_DEST,
-                                              finger_identity);
-      }
-    }
-  }
-  else
-  {
-    /* If both new and existing entry are same as my_identity, then do nothing. */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&existing_finger->finger_identity,
-                                              &my_identity))
-    {
-      return;
-    }
-
-    /* If there is space to store more trails. */
-    if (existing_finger->trails_count < MAXIMUM_TRAILS_PER_FINGER)
-        add_new_trail (existing_finger,
-                       finger_trail,
-                       finger_trail_length,
-                       finger_trail_id);
-    else
-        select_and_replace_trail (existing_finger,
-                                  finger_trail,
-                                  finger_trail_length,
-                                  finger_trail_id);
-  }
-  update_current_search_finger_index (finger_table_index);
-  return;
-}
-
-
-/**
- * Verify validity of P2P put messages.
- *
- * @param cls closure
- * @param put the message
- * @return #GNUNET_OK if the message is well-formed
- */
-static int
-check_dht_p2p_put (void *cls,
-                    const struct PeerPutMessage *put)
-{
-  size_t msize;
-  uint32_t putlen;
-
-  msize = ntohs (put->header.size);
-  putlen = ntohl (put->put_path_length);
-  if ((msize <
-       sizeof (struct PeerPutMessage) +
-       putlen * sizeof (struct GNUNET_PeerIdentity)) ||
-      (putlen >
-       GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handler for P2P put messages.
- *
- * @param cls closure
- * @param put the message
- */
-static void
-handle_dht_p2p_put (void *cls,
-                    const struct PeerPutMessage *put)
-{
-  struct GNUNET_PeerIdentity *put_path;
-  struct GNUNET_PeerIdentity current_best_known_dest;
-  struct GNUNET_PeerIdentity best_known_dest;
-  struct GNUNET_HashCode received_intermediate_trail_id;
-  struct GNUNET_HashCode intermediate_trail_id;
-  struct GNUNET_PeerIdentity next_hop;
-  const struct GNUNET_PeerIdentity *next_routing_hop;
-  enum GNUNET_DHT_RouteOption options;
-  struct GNUNET_HashCode test_key;
-  struct Closest_Peer successor;
-  void *payload;
-  size_t msize;
-  uint32_t putlen = ntohl (put->put_path_length);
-  struct GNUNET_PeerIdentity pp[putlen + 1];
-  uint32_t hop_count;
-  size_t payload_size;
-  uint64_t key_value;
-
-  msize = ntohs (put->header.size);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            (int64_t) msize,
-                            GNUNET_NO);
-
-  current_best_known_dest = put->best_known_destination;
-  put_path = (struct GNUNET_PeerIdentity *) &put[1];
-  payload = &put_path[putlen];
-  options = ntohl (put->options);
-  received_intermediate_trail_id = put->intermediate_trail_id;
-  hop_count = ntohl(put->hop_count);
-  payload_size = msize - (sizeof (struct PeerPutMessage) +
-                          putlen * sizeof (struct GNUNET_PeerIdentity));
-  hop_count++;
-  switch (GNUNET_BLOCK_get_key (GDS_block_context,
-                                ntohl (put->block_type),
-                                payload,
-                                payload_size,
-                                &test_key))
-  {
-    case GNUNET_YES:
-      if (0 != memcmp (&test_key,
-                       &put->key,
-                       sizeof (struct GNUNET_HashCode)))
-      {
-        char *put_s = GNUNET_strdup (GNUNET_h2s_full (&put->key));
-        GNUNET_break_op (0);
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "PUT with key `%s' for block with key %s\n",
-                    put_s,
-                    GNUNET_h2s_full (&test_key));
-        GNUNET_free (put_s);
-        return;
-      }
-    break;
-    case GNUNET_NO:
-      GNUNET_break_op (0);
-      return;
-    case GNUNET_SYSERR:
-      /* cannot verify, good luck */
-      break;
-  }
-
-  if (ntohl (put->block_type) == GNUNET_BLOCK_TYPE_REGEX) /* FIXME: do for all tpyes */
-  {
-    switch (GNUNET_BLOCK_evaluate (GDS_block_context,
-                                   ntohl (put->block_type),
-                                   GNUNET_BLOCK_EO_NONE,
-                                   NULL,    /* query */
-                                   NULL, 0, /* bloom filer */
-                                   NULL, 0, /* xquery */
-                                   payload,
-                                   payload_size))
-    {
-    case GNUNET_BLOCK_EVALUATION_OK_MORE:
-    case GNUNET_BLOCK_EVALUATION_OK_LAST:
-      break;
-
-    case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE:
-    case GNUNET_BLOCK_EVALUATION_RESULT_INVALID:
-    case GNUNET_BLOCK_EVALUATION_RESULT_IRRELEVANT:
-    case GNUNET_BLOCK_EVALUATION_REQUEST_VALID:
-    case GNUNET_BLOCK_EVALUATION_REQUEST_INVALID:
-    case GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED:
-    default:
-      GNUNET_break_op (0);
-      return;
-    }
-  }
-
-  /* Check if you are already a part of put path. */
-  unsigned int i;
-  for (i = 0; i < putlen; i++)
-  {
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                              &put_path[i]))
-    {
-      putlen = i;
-      break;
-    }
-  }
-
-  /* Add yourself to the list. */
-  //if (0 != (options & GNUNET_DHT_RO_RECORD_ROUTE))
-  if (1)
-  {
-    GNUNET_memcpy (pp,
-                   put_path,
-                   putlen * sizeof (struct GNUNET_PeerIdentity));
-    pp[putlen] = my_identity;
-    putlen++;
-  }
-  else
-  {
-    putlen = 0;
-  }
-  GNUNET_memcpy (&key_value,
-                 &put->key,
-                 sizeof (uint64_t));
-  key_value = GNUNET_ntohll (key_value);
-  successor = find_local_best_known_next_hop (key_value,
-                                              GDS_FINGER_TYPE_NON_PREDECESSOR);
-  next_hop = successor.next_hop;
-  intermediate_trail_id = successor.trail_id;
-  best_known_dest = successor.best_known_destination;
-
-  if (0 != (GNUNET_CRYPTO_cmp_peer_identity (&current_best_known_dest,
-                                             &my_identity)))
-  {
-    next_routing_hop = GDS_ROUTING_get_next_hop (&received_intermediate_trail_id,
-                                                 GDS_ROUTING_SRC_TO_DEST);
-    if (NULL != next_routing_hop)
-    {
-      next_hop = *next_routing_hop;
-      intermediate_trail_id = received_intermediate_trail_id;
-      best_known_dest = current_best_known_dest;
-    }
-  }
-
-  GDS_CLIENTS_process_put (options,
-                           ntohl (put->block_type),
-                           hop_count,
-                           ntohl (put->desired_replication_level),
-                           putlen,
-                           pp,
-                           GNUNET_TIME_absolute_ntoh (put->expiration_time),
-                           &put->key,
-                           payload,
-                           payload_size);
-
-  /* I am the final destination */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &best_known_dest))
-  {
-    DEBUG ("\n PUT_REQUEST_SUCCESSFUL for key = %s",
-           GNUNET_h2s(&put->key));
-    GDS_DATACACHE_handle_put (GNUNET_TIME_absolute_ntoh (put->expiration_time),
-                              &put->key,
-                              putlen,
-                              pp,
-                              ntohl (put->block_type),
-                              payload_size,
-                              payload);
-  }
-  GDS_NEIGHBOURS_send_put (&put->key,
-                           ntohl (put->block_type),
-                           ntohl (put->options),
-                           ntohl (put->desired_replication_level),
-                           best_known_dest,
-                           intermediate_trail_id,
-                           &next_hop,
-                           hop_count,
-                           putlen,
-                           pp,
-                           GNUNET_TIME_absolute_ntoh (put->expiration_time),
-                           payload,
-                           payload_size);
-}
-
-
-/**
- * Check integrity of @a get message.
- *
- * @param cls closure
- * @param get the message
- * @return #GNUNET_OK if @a get is well-formed
- */
-static int
-check_dht_p2p_get (void *cls,
-                   const struct PeerGetMessage *get)
-{
-  uint32_t get_length;
-  size_t msize;
-
-  msize = ntohs (get->header.size);
-  get_length = ntohl (get->get_path_length);
-  if ((msize <
-       sizeof (struct PeerGetMessage) +
-       get_length * sizeof (struct GNUNET_PeerIdentity)) ||
-       (get_length >
-        GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * FIXME: Check for loop in the request. If you already are part of get path,
- * then you need to reset the get path length.
- * Core handler for p2p get requests.
- *
- * @param cls closure
- * @param get the message
- */
-static void
-handle_dht_p2p_get (void *cls,
-                    const struct PeerGetMessage *get)
-{
-  const struct GNUNET_PeerIdentity *get_path;
-  struct GNUNET_PeerIdentity best_known_dest;
-  struct GNUNET_PeerIdentity current_best_known_dest;
-  struct GNUNET_HashCode intermediate_trail_id;
-  struct GNUNET_HashCode received_intermediate_trail_id;
-  struct Closest_Peer successor;
-  struct GNUNET_PeerIdentity next_hop;
-  const struct GNUNET_PeerIdentity *next_routing_hop;
-  uint32_t get_length;
-  uint64_t key_value;
-  uint32_t hop_count;
-  size_t msize;
-
-  msize = ntohs (get->header.size);
-  get_length = ntohl (get->get_path_length);
-  current_best_known_dest = get->best_known_destination;
-  received_intermediate_trail_id = get->intermediate_trail_id;
-  get_path = (const struct GNUNET_PeerIdentity *) &get[1];
-  hop_count = get->hop_count;
-  hop_count++;
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-  GNUNET_memcpy (&key_value,
-                 &get->key,
-                 sizeof (uint64_t));
-  key_value = GNUNET_ntohll (key_value);
-
-  /* Check if you are already a part of get path. */
-  for (unsigned int i = 0; i < get_length; i++)
-  {
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                              &get_path[i]))
-    {
-      get_length = i;
-      break;
-    }
-  }
-
-  /* Add yourself in the get path. */
-  struct GNUNET_PeerIdentity gp[get_length + 1];
-  GNUNET_memcpy (gp,
-                 get_path,
-                 get_length * sizeof (struct GNUNET_PeerIdentity));
-  gp[get_length] = my_identity;
-  get_length = get_length + 1;
-  GDS_CLIENTS_process_get (get->options,
-                           get->block_type,
-                           hop_count,
-                           get->desired_replication_level,
-                           get->get_path_length,
-                           gp,
-                           &get->key);
-
-
-  successor = find_local_best_known_next_hop (key_value,
-                                              GDS_FINGER_TYPE_NON_PREDECESSOR);
-  next_hop = successor.next_hop;
-  best_known_dest = successor.best_known_destination;
-  intermediate_trail_id = successor.trail_id;
-  /* I am not the final destination. I am part of trail to reach final dest. */
-  if (0 != (GNUNET_CRYPTO_cmp_peer_identity (&current_best_known_dest, &my_identity)))
-  {
-    next_routing_hop = GDS_ROUTING_get_next_hop (&received_intermediate_trail_id,
-                                                 GDS_ROUTING_SRC_TO_DEST);
-    if (NULL != next_routing_hop)
-    {
-      next_hop = *next_routing_hop;
-      best_known_dest = current_best_known_dest;
-      intermediate_trail_id = received_intermediate_trail_id;
-    }
-  }
-
-  /* I am the final destination. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &best_known_dest))
-  {
-    if (1 == get_length)
-    {
-      DEBUG ("\n GET_REQUEST DONE for key = %s",
-             GNUNET_h2s(&get->key));
-      GDS_DATACACHE_handle_get (&get->key,
-                                get->block_type, /* FIXME: endianess? */
-                                NULL,
-                                0,
-                                NULL,
-                                0,
-                                &get_cb,
-                                NULL);
-    }
-    else
-    {
-      GDS_DATACACHE_handle_get (&get->key,
-                                get->block_type, /* FIXME: endianess? */
-                                NULL,
-                                0,
-                                NULL,
-                                0,
-                                &get_cb,
-                                &gp[get_length - 2]);
-    }
-  }
-  else
-  {
-    GDS_NEIGHBOURS_send_get (&get->key,
-                             get->block_type, /* FIXME: endianess? */
-                             get->options,
-                             get->desired_replication_level,
-                             &best_known_dest,
-                             &intermediate_trail_id,
-                             &next_hop,
-                             hop_count,
-                             get_length,
-                             gp);
-  }
-}
-
-
-/**
- * Check validity of @a get_result message.
- *
- * @param cls closure
- * @param get_result the message
- * @return #GNUNET_OK if @a get_result is well-formed
- */
-static int
-check_dht_p2p_get_result (void *cls,
-                          const struct PeerGetResultMessage *get_result)
-{
-  size_t msize;
-  unsigned int getlen;
-  unsigned int putlen;
-
-  msize = ntohs (get_result->header.size);
-  getlen = ntohl (get_result->get_path_length);
-  putlen = ntohl (get_result->put_path_length);
-  if ((msize <
-       sizeof (struct PeerGetResultMessage) +
-       getlen * sizeof (struct GNUNET_PeerIdentity) +
-       putlen * sizeof (struct GNUNET_PeerIdentity)) ||
-      (getlen >
-       GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity) ||
-      (putlen >
-         GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity))))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handler for get result
- *
- * @param cls closure
- * @param get_result the message
- */
-static void
-handle_dht_p2p_get_result (void *cls,
-                           const struct PeerGetResultMessage *get_result)
-{
-  const struct GNUNET_PeerIdentity *get_path;
-  const struct GNUNET_PeerIdentity *put_path;
-  const void *payload;
-  size_t payload_size;
-  size_t msize;
-  unsigned int getlen;
-  unsigned int putlen;
-  int current_path_index;
-
-  msize = ntohs (get_result->header.size);
-  getlen = ntohl (get_result->get_path_length);
-  putlen = ntohl (get_result->put_path_length);
-  DEBUG ("GET_RESULT  FOR DATA_SIZE = %u\n",
-         (unsigned int) msize);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-  put_path = (const struct GNUNET_PeerIdentity *) &get_result[1];
-  get_path = &put_path[putlen];
-  payload = (const void *) &get_path[getlen];
-  payload_size = msize - (sizeof (struct PeerGetResultMessage) +
-                         (getlen + putlen) * sizeof (struct GNUNET_PeerIdentity));
-
-  if (0 == (GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                             &get_path[0])))
-  {
-    GDS_CLIENTS_handle_reply (GNUNET_TIME_absolute_ntoh (get_result->expiration_time),
-                              &get_result->key,
-                              getlen,
-                              get_path,
-                              putlen,
-                              put_path,
-                              get_result->type,
-                              payload_size,
-                              payload);
-    return;
-  }
-  current_path_index = search_my_index (get_path,
-                                        getlen);
-  if (-1 == current_path_index)
-  {
-    DEBUG ("No entry found in get path.\n");
-    GNUNET_break (0);
-    return;
-  }
-  if ((getlen + 1) == current_path_index)
-  {
-    DEBUG("Present twice in get path. Not allowed. \n");
-    GNUNET_break (0);
-    return;
-  }
-  GDS_NEIGHBOURS_send_get_result (&get_result->key,
-                                  get_result->type, /* FIXME: endianess? */
-                                  &get_path[current_path_index - 1],
-                                  &get_result->querying_peer,
-                                  putlen,
-                                  put_path,
-                                  getlen,
-                                  get_path,
-                                  GNUNET_TIME_absolute_ntoh (get_result->expiration_time),
-                                  payload,
-                                  payload_size);
-}
-
-
-/**
- * Find the next hop to pass trail setup message. First find the local best known
- * hop from your own identity, friends and finger. If you were part of trail,
- * then get the next hop from routing table. Compare next_hop from routing table
- * and local best known hop, and return the closest one to final_dest_finger_val
- * @param final_dest_finger_val 64 bit value of finger identity
- * @param intermediate_trail_id If you are part of trail to reach to some other
- *                              finger, then it is the trail id to reach to
- *                              that finger, else set to 0.
- * @param is_predecessor Are we looking for closest successor or predecessor.
- * @param source Source of trail setup message.
- * @param current_dest In case you are part of trail, then finger to which
- *                     we should forward the message. Else my own identity
- * @return Closest Peer for @a final_dest_finger_val
- */
-static struct Closest_Peer
-get_local_best_known_next_hop (uint64_t final_dest_finger_val,
-                               const struct GNUNET_HashCode *intermediate_trail_id,
-                               unsigned int is_predecessor,
-                               const struct GNUNET_PeerIdentity *source,
-                               const struct GNUNET_PeerIdentity *current_dest)
-{
-  struct Closest_Peer peer;
-
-  peer = find_local_best_known_next_hop (final_dest_finger_val,
-                                         is_predecessor);
-
-  /* Am I just a part of a trail towards a finger (current_destination)? */
-  if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            current_dest) &&
-      0 != GNUNET_CRYPTO_cmp_peer_identity (&peer.best_known_destination,
-                                            current_dest))
-  {
-    const struct GNUNET_PeerIdentity *closest_peer;
-
-    /* Select best successor among one found locally and current_destination
-     * that we got from network.*/
-    closest_peer = select_closest_peer (&peer.best_known_destination,
-                                        current_dest,
-                                        final_dest_finger_val,
-                                        is_predecessor);
-
-    /* Is current dest (end point of the trail of which I am a part) closest_peer? */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (current_dest,
-                                              closest_peer))
-    {
-      const struct GNUNET_PeerIdentity *next_hop;
-
-      next_hop = GDS_ROUTING_get_next_hop (intermediate_trail_id,
-                                           GDS_ROUTING_SRC_TO_DEST);
-      /* next_hop NULL is a valid case. This intermediate trail id is set by
-       some other finger, and while this trail setup is in progress, that other
-       peer might have found a better trail ,and send trail teardown message
-       across the network. In case we got the trail teardown message first,
-       then next_hop will be NULL. A possible solution could be to keep track
-       * of all removed trail id, and be sure that there is no other reason . */
-      if(NULL != next_hop)
-      {
-         peer.next_hop = *next_hop;
-         peer.best_known_destination =  *current_dest;
-         peer.trail_id = *intermediate_trail_id;
-      }
-    }
-  }
-  return peer;
-}
-
-
-/**
- * Check format of a PeerTrailSetupMessage.
- *
- * @param cls closure
- * @param trail_setup the message
- * @return #GNUNET_OK if @a trail_setup is well-formed
- */
-static int
-check_dht_p2p_trail_setup (void *cls,
-                            const struct PeerTrailSetupMessage *trail_setup)
-{
-  size_t msize;
-
-  msize = ntohs (trail_setup->header.size);
-  if ((msize - sizeof (struct PeerTrailSetupMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for PeerTrailSetupMessage.
- *
- * @param cls closure
- * @param trail_setup the message
- */
-static void
-handle_dht_p2p_trail_setup (void *cls,
-                            const struct PeerTrailSetupMessage *trail_setup)
-{
-  struct FriendInfo *friend = cls;
-  const struct GNUNET_PeerIdentity *trail_peer_list;
-  struct GNUNET_PeerIdentity current_dest;
-  struct FriendInfo *target_friend;
-  struct GNUNET_PeerIdentity source;
-  struct GNUNET_HashCode intermediate_trail_id;
-  struct GNUNET_HashCode trail_id;
-  unsigned int is_predecessor;
-  uint32_t trail_length;
-  uint64_t final_dest_finger_val;
-  int i;
-  size_t msize;
-
-  msize = ntohs (trail_setup->header.size);
-  trail_length = (msize - sizeof (struct PeerTrailSetupMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-  trail_peer_list = (const struct GNUNET_PeerIdentity *) &trail_setup[1];
-  current_dest = trail_setup->best_known_destination;
-  trail_id = trail_setup->trail_id;
-  final_dest_finger_val
-    = GNUNET_ntohll (trail_setup->final_destination_finger_value);
-  source = trail_setup->source_peer;
-  is_predecessor = ntohl (trail_setup->is_predecessor);
-  intermediate_trail_id = trail_setup->intermediate_trail_id;
-
-  /* Did the friend insert its ID in the trail list? */
-  if ( (trail_length > 0) &&
-       (0 != memcmp (&trail_peer_list[trail_length-1],
-                     friend->id,
-                     sizeof (struct GNUNET_PeerIdentity))) )
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-   /* If I was the source and got the message back, then set trail length to 0.*/
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &source))
-  {
-    trail_length = 0;
-  }
-
-  /* Check if you are present in the trail seen so far? */
-  for (i = 0; i < trail_length; i++)
-  {
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&trail_peer_list[i],
-                                              &my_identity))
-    {
-      /* We will add ourself later in code, if NOT destination. */
-      trail_length = i;
-      break;
-    }
-  }
-
-  /* Is my routing table full?  */
-  if (GNUNET_YES == GDS_ROUTING_threshold_reached ())
-  {
-    target_friend
-      = (trail_length > 0)
-      ? GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                           &trail_peer_list[trail_length - 1])
-      : GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                           &source);
-    if (NULL == target_friend)
-    {
-      DEBUG ("\n friend not found");
-      GNUNET_break(0);
-      return;
-    }
-    GDS_NEIGHBOURS_send_trail_rejection (&source,
-                                         final_dest_finger_val,
-                                         &my_identity,
-                                         is_predecessor,
-                                         trail_peer_list,
-                                         trail_length,
-                                         &trail_id,
-                                         target_friend,
-                                         CONGESTION_TIMEOUT);
-    return;
-  }
-
-  /* Get the next hop to forward the trail setup request. */
-  struct Closest_Peer next_peer
-    = get_local_best_known_next_hop (final_dest_finger_val,
-                                     &intermediate_trail_id,
-                                     is_predecessor,
-                                     &source,
-                                     &current_dest);
-
-  /* Am I the final destination? */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&next_peer.best_known_destination,
-                                            &my_identity))
-  {
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&source,
-                                              &my_identity))
-    {
-      finger_table_add (&my_identity,
-                        NULL,
-                        0,
-                        is_predecessor,
-                        final_dest_finger_val,
-                        &trail_id);
-      return;
-    }
-
-    target_friend
-      = (trail_length > 0)
-      ? GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                           &trail_peer_list[trail_length-1])
-      : GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                           &source);
-    if (NULL == target_friend)
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    GDS_ROUTING_add (&trail_id,
-                     target_friend->id,
-                     &my_identity);
-    GDS_NEIGHBOURS_send_trail_setup_result (&source,
-                                            &my_identity,
-                                            target_friend,
-                                            trail_length,
-                                            trail_peer_list,
-                                            is_predecessor,
-                                            final_dest_finger_val,
-                                            &trail_id);
-    return;
-  }
-  /* I'm not the final destination. */
-  target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &next_peer.next_hop);
-  if (NULL == target_friend)
-  {
-    DEBUG ("\n target friend not found for peer = %s",
-           GNUNET_i2s(&next_peer.next_hop));
-    GNUNET_break (0);
-    return;
-  }
-  if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &source))
-  {
-    /* Add yourself to list of peers. */
-    struct GNUNET_PeerIdentity peer_list[trail_length + 1];
-
-    GNUNET_memcpy (peer_list,
-                   trail_peer_list,
-                   trail_length * sizeof (struct GNUNET_PeerIdentity));
-    peer_list[trail_length] = my_identity;
-    GDS_NEIGHBOURS_send_trail_setup (&source,
-                                     final_dest_finger_val,
-                                     &next_peer.best_known_destination,
-                                     target_friend,
-                                     trail_length + 1,
-                                     peer_list,
-                                     is_predecessor,
-                                     &trail_id,
-                                     &next_peer.trail_id);
-    return;
-  }
-  GDS_NEIGHBOURS_send_trail_setup (&source,
-                                   final_dest_finger_val,
-                                   &next_peer.best_known_destination,
-                                   target_friend,
-                                   0,
-                                   NULL,
-                                   is_predecessor,
-                                   &trail_id,
-                                   &next_peer.trail_id);
-}
-
-
-/**
- * Validate format of trail setup result messages.
- *
- * @param closure
- * @param trail_result the message
- * @return #GNUNET_OK if @a trail_result is well-formed
- */
-static int
-check_dht_p2p_trail_setup_result (void *cls,
-                                  const struct PeerTrailSetupResultMessage *trail_result)
-{
-  size_t msize;
-
-  msize = ntohs (trail_result->header.size);
-  if ((msize - sizeof (struct PeerTrailSetupResultMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for p2p trail setup result messages.
- *
- * @param closure
- * @param trail_result the message
- */
-static void
-handle_dht_p2p_trail_setup_result (void *cls,
-                                   const struct PeerTrailSetupResultMessage *trail_result)
-{
-  struct FriendInfo *friend = cls;
-  const struct GNUNET_PeerIdentity *trail_peer_list;
-  struct GNUNET_PeerIdentity next_hop;
-  struct FriendInfo *target_friend;
-  struct GNUNET_PeerIdentity querying_peer;
-  struct GNUNET_PeerIdentity finger_identity;
-  uint32_t trail_length;
-  uint64_t ultimate_destination_finger_value;
-  uint32_t is_predecessor;
-  struct GNUNET_HashCode trail_id;
-  int my_index;
-  size_t msize;
-
-  msize = ntohs (trail_result->header.size);
-  trail_length = (msize - sizeof (struct PeerTrailSetupResultMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-
-  is_predecessor = ntohl (trail_result->is_predecessor);
-  querying_peer = trail_result->querying_peer;
-  finger_identity = trail_result->finger_identity;
-  trail_id = trail_result->trail_id;
-  trail_peer_list = (const struct GNUNET_PeerIdentity *) &trail_result[1];
-  ultimate_destination_finger_value
-    = GNUNET_ntohll (trail_result->ultimate_destination_finger_value);
-
-  /* Am I the one who initiated the query? */
-  if (0 == (GNUNET_CRYPTO_cmp_peer_identity (&querying_peer,
-                                             &my_identity)))
-  {
-    /* Check that you got the message from the correct peer. */
-    if (trail_length > 0)
-    {
-      GNUNET_assert (0 == GNUNET_CRYPTO_cmp_peer_identity (&trail_peer_list[0],
-                                                           friend->id));
-    }
-    else
-    {
-      GNUNET_assert (0 == GNUNET_CRYPTO_cmp_peer_identity (&finger_identity,
-                                                           friend->id));
-    }
-    GDS_ROUTING_add (&trail_id,
-                     &my_identity,
-                     friend->id);
-    finger_table_add (&finger_identity,
-                      trail_peer_list,
-                      trail_length,
-                      is_predecessor,
-                      ultimate_destination_finger_value,
-                      &trail_id);
-    return;
-  }
-
-  /* Get my location in the trail. */
-  my_index = search_my_index (trail_peer_list,
-                              trail_length);
-  if (-1 == my_index)
-  {
-    DEBUG ("Not found in trail\n");
-    GNUNET_break_op(0);
-    return;
-  }
-  //TODO; return -2.
-  if ((trail_length + 1) == my_index)
-  {
-    DEBUG ("Found twice in trail.\n");
-    GNUNET_break_op(0);
-    return;
-  }
-
-  //TODO; Refactor code here and above to check if sender peer is correct
-  if (my_index == 0)
-  {
-    if (trail_length > 1)
-      GNUNET_assert (0 == GNUNET_CRYPTO_cmp_peer_identity (&trail_peer_list[1],
-                                                           friend->id));
-    else
-      GNUNET_assert (0 == GNUNET_CRYPTO_cmp_peer_identity (&finger_identity,
-                                                           friend->id));
-    next_hop = trail_result->querying_peer;
-  }
-  else
-  {
-    if (my_index == trail_length - 1)
-    {
-      GNUNET_assert (0 ==
-                     GNUNET_CRYPTO_cmp_peer_identity (&finger_identity,
-                                                      friend->id));
-    }
-    else
-      GNUNET_assert (0 ==
-                     GNUNET_CRYPTO_cmp_peer_identity (&trail_peer_list[my_index + 1],
-                                                      friend->id));
-    next_hop = trail_peer_list[my_index - 1];
-  }
-
-  target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &next_hop);
-  if (NULL == target_friend)
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  GDS_ROUTING_add (&trail_id,
-                   &next_hop,
-                   friend->id);
-  GDS_NEIGHBOURS_send_trail_setup_result (&querying_peer,
-                                          &finger_identity,
-                                          target_friend,
-                                          trail_length,
-                                          trail_peer_list,
-                                          is_predecessor,
-                                          ultimate_destination_finger_value,
-                                          &trail_id);
-}
-
-
-/**
- * Invert the trail.
- *
- * @param trail Trail to be inverted
- * @param trail_length Total number of peers in the trail.
- * @return Updated trail
- */
-static struct GNUNET_PeerIdentity *
-invert_trail (const struct GNUNET_PeerIdentity *trail,
-              unsigned int trail_length)
-{
-  int i;
-  int j;
-  struct GNUNET_PeerIdentity *inverted_trail;
-
-  inverted_trail = GNUNET_new_array (trail_length,
-                                     struct GNUNET_PeerIdentity);
-  i = 0;
-  j = trail_length - 1;
-  while (i < trail_length)
-  {
-    inverted_trail[i] = trail[j];
-    i++;
-    j--;
-  }
-
-  GNUNET_assert (NULL !=
-                 GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                    &inverted_trail[0]));
-  return inverted_trail;
-}
-
-
-/**
- * Return the shortest trail among all the trails to reach to finger from me.
- *
- * @param finger Finger
- * @param shortest_trail_length[out] Trail length of shortest trail from me
- *                                   to @a finger
- * @return Shortest trail.
- */
-static struct GNUNET_PeerIdentity *
-get_shortest_trail (struct FingerInfo *finger,
-                    unsigned int *trail_length)
-{
-  struct Trail *trail;
-  unsigned int flag = 0;
-  unsigned int shortest_trail_index = 0;
-  int shortest_trail_length = -1;
-  struct Trail_Element *trail_element;
-  struct GNUNET_PeerIdentity *trail_list;
-  unsigned int i;
-
-  /* Get the shortest trail to reach to current successor. */
-  for (i = 0; i < finger->trails_count; i++)
-  {
-    trail = &finger->trail_list[i];
-
-    if (0 == flag)
-    {
-      shortest_trail_index = i;
-      shortest_trail_length = trail->trail_length;
-      flag = 1;
-      continue;
-    }
-
-    if (shortest_trail_length > trail->trail_length)
-    {
-      shortest_trail_index = i;
-      shortest_trail_length = trail->trail_length;
-    }
-    continue;
-  }
-
-  /* Copy the shortest trail and return. */
-  trail = &finger->trail_list[shortest_trail_index];
-  trail_element = trail->trail_head;
-
-  trail_list = GNUNET_new_array (shortest_trail_length,
-                                 struct GNUNET_PeerIdentity);
-
-  for (i = 0; i < shortest_trail_length; i++,trail_element = trail_element->next)
-  {
-    trail_list[i] = trail_element->peer;
-  }
-
-  GNUNET_assert(shortest_trail_length != -1);
-
-  *trail_length = shortest_trail_length;
-  return trail_list;
-}
-
-
-/**
- * Check if @a trail_1 and @a trail_2 have any common element. If yes then join
- * them at common element. @a trail_1 always preceeds @a trail_2 in joined trail.
- *
- * @param trail_1 Trail from source to me, NOT including endpoints.
- * @param trail_1_len Total number of peers @a trail_1
- * @param trail_2 Trail from me to current predecessor, NOT including endpoints.
- * @param trail_2_len Total number of peers @a trail_2
- * @param joined_trail_len Total number of peers in combined trail of @a trail_1
- *                          @a trail_2.
- * @return Joined trail.
- */
-static struct GNUNET_PeerIdentity *
-check_for_duplicate_entries (const struct GNUNET_PeerIdentity *trail_1,
-                             unsigned int trail_1_len,
-                             const struct GNUNET_PeerIdentity *trail_2,
-                             unsigned int trail_2_len,
-                             unsigned int *joined_trail_len)
-{
-  struct GNUNET_PeerIdentity *joined_trail;
-  unsigned int i;
-  unsigned int j;
-  unsigned int k;
-
-  for (i = 0; i < trail_1_len; i++)
-  {
-    for (j = 0; j < trail_2_len; j++)
-    {
-      if (0 != GNUNET_CRYPTO_cmp_peer_identity (&trail_1[i],
-                                                &trail_2[j]))
-        continue;
-
-      *joined_trail_len = i + (trail_2_len - j);
-      joined_trail = GNUNET_new_array (*joined_trail_len,
-                                       struct GNUNET_PeerIdentity);
-
-
-      /* Copy all the elements from 0 to i into joined_trail. */
-      for(k = 0; k < ( i+1); k++)
-      {
-        joined_trail[k] = trail_1[k];
-      }
-
-      /* Increment j as entry stored is same as entry stored at i*/
-      j = j+1;
-
-      /* Copy all the elements from j to trail_2_len-1 to joined trail.*/
-      while(k <= (*joined_trail_len - 1))
-      {
-        joined_trail[k] = trail_2[j];
-        j++;
-        k++;
-      }
-
-      return joined_trail;
-    }
-  }
-
-  /* Here you should join the  trails. */
-  *joined_trail_len = trail_1_len + trail_2_len + 1;
-  joined_trail = GNUNET_new_array (*joined_trail_len,
-                                   struct GNUNET_PeerIdentity);
-
-
-  for(i = 0; i < trail_1_len;i++)
-  {
-    joined_trail[i] = trail_1[i];
-  }
-
-  joined_trail[i] = my_identity;
-  i++;
-
-  for (j = 0; i < *joined_trail_len; i++,j++)
-  {
-    joined_trail[i] = trail_2[j];
-  }
-
-  return joined_trail;
-}
-
-
-/**
- * Return the trail from source to my current predecessor. Check if source
- * is already part of the this trail, if yes then return the shorten trail.
- *
- * @param current_trail Trail from source to me, NOT including the endpoints.
- * @param current_trail_length Number of peers in @a current_trail.
- * @param trail_src_to_curr_pred_length[out] Number of peers in trail from
- *                                           source to my predecessor, NOT including
- *                                           the endpoints.
- * @return Trail from source to my predecessor.
- */
-static struct GNUNET_PeerIdentity *
-get_trail_src_to_curr_pred (struct GNUNET_PeerIdentity source_peer,
-                            const struct GNUNET_PeerIdentity *trail_src_to_me,
-                            unsigned int trail_src_to_me_len,
-                            unsigned int *trail_src_to_curr_pred_length)
-{
-  struct GNUNET_PeerIdentity *trail_me_to_curr_pred;
-  struct GNUNET_PeerIdentity *trail_src_to_curr_pred;
-  unsigned int trail_me_to_curr_pred_length;
-  struct FingerInfo *current_predecessor;
-  int i;
-  unsigned int j;
-  unsigned int len;
-
-  current_predecessor = &finger_table[PREDECESSOR_FINGER_ID];
-
-  /* Check if trail_src_to_me contains current_predecessor. */
-  for (i = 0; i < trail_src_to_me_len; i++)
-  {
-    if (0 != GNUNET_CRYPTO_cmp_peer_identity(&trail_src_to_me[i],
-                                             &current_predecessor->finger_identity))
-      continue;
-
-
-    *trail_src_to_curr_pred_length = i;
-
-    if(0 == i)
-      return NULL;
-
-    trail_src_to_curr_pred = GNUNET_new_array (*trail_src_to_curr_pred_length,
-                                               struct GNUNET_PeerIdentity);
-     for (j = 0; j < i; j++)
-       trail_src_to_curr_pred[j] = trail_src_to_me[j];
-     return trail_src_to_curr_pred;
-  }
-
-
-  trail_me_to_curr_pred = get_shortest_trail (current_predecessor,
-                                              &trail_me_to_curr_pred_length);
-
-  /* Check if trail contains the source_peer. */
-  for (i = trail_me_to_curr_pred_length - 1; i >= 0; i--)
-  {
-    if (0 != GNUNET_CRYPTO_cmp_peer_identity (&source_peer,
-                                              &trail_me_to_curr_pred[i]))
-      continue;
-
-    /* Source is NOT part of trail. */
-    i++;
-
-    /* Source is the last element in the trail to reach to my pred.
-       Source is direct friend of the pred. */
-    if (trail_me_to_curr_pred_length == i)
-    {
-      *trail_src_to_curr_pred_length = 0;
-      GNUNET_free_non_null (trail_me_to_curr_pred);
-      return NULL;
-    }
-
-    *trail_src_to_curr_pred_length = trail_me_to_curr_pred_length - i;
-    trail_src_to_curr_pred = GNUNET_new_array (*trail_src_to_curr_pred_length,
-                                               struct GNUNET_PeerIdentity);
-
-
-    for (j = 0; j < *trail_src_to_curr_pred_length; i++,j++)
-      trail_src_to_curr_pred[j] = trail_me_to_curr_pred[i];
-    GNUNET_free_non_null (trail_me_to_curr_pred);
-    return trail_src_to_curr_pred;
-  }
-
-  trail_src_to_curr_pred = check_for_duplicate_entries (trail_src_to_me,
-                                                        trail_src_to_me_len,
-                                                        trail_me_to_curr_pred,
-                                                        trail_me_to_curr_pred_length,
-                                                        &len);
-  *trail_src_to_curr_pred_length = len;
-  GNUNET_free_non_null(trail_me_to_curr_pred);
-  return trail_src_to_curr_pred;
-}
-
-
-/**
- * Add finger as your predecessor. To add, first generate a new trail id, invert
- * the trail to get the trail from me to finger, add an entry in your routing
- * table, send add trail message to peers which are part of trail from me to
- * finger and add finger in finger table.
- *
- * @param finger
- * @param trail
- * @param trail_length
- */
-static void
-update_predecessor (const struct GNUNET_PeerIdentity *finger,
-                    const struct GNUNET_PeerIdentity *trail,
-                    unsigned int trail_length)
-{
-  struct GNUNET_HashCode trail_to_new_predecessor_id;
-  struct GNUNET_PeerIdentity *trail_to_new_predecessor;
-  struct FriendInfo *target_friend;
-
-  /* Generate trail id for trail from me to new predecessor = finger. */
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_STRONG,
-                              &trail_to_new_predecessor_id,
-                              sizeof (trail_to_new_predecessor_id));
-
-  if (0 == trail_length)
-  {
-    trail_to_new_predecessor = NULL;
-    GDS_ROUTING_add (&trail_to_new_predecessor_id,
-                     &my_identity,
-                     finger);
-    target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                       finger);
-    if (NULL == target_friend)
-    {
-      GNUNET_break (0);
-      return;
-    }
-  }
-  else
-  {
-    /* Invert the trail to get the trail from me to finger, NOT including the
-       endpoints.*/
-    GNUNET_assert(NULL != GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                             &trail[trail_length-1]));
-    trail_to_new_predecessor = invert_trail (trail,
-                                             trail_length);
-
-    /* Add an entry in your routing table. */
-    GDS_ROUTING_add (&trail_to_new_predecessor_id,
-                     &my_identity,
-                     &trail_to_new_predecessor[0]);
-
-    GNUNET_assert (NULL != (target_friend =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      &trail_to_new_predecessor[0])));
-  }
-
-  /* Add entry in routing table of all peers that are part of trail from me
-     to finger, including finger. */
-  GDS_NEIGHBOURS_send_add_trail (&my_identity,
-                                 finger,
-                                 &trail_to_new_predecessor_id,
-                                 trail_to_new_predecessor,
-                                 trail_length,
-                                 target_friend);
-
-  add_new_finger (finger,
-                  trail_to_new_predecessor,
-                  trail_length,
-                  &trail_to_new_predecessor_id,
-                  PREDECESSOR_FINGER_ID);
-  GNUNET_free_non_null (trail_to_new_predecessor);
-}
-
-
-/**
- * Check if you already have a predecessor. If not then add finger as your
- * predecessor. If you have predecessor, then compare two peer identites.
- * If finger is correct predecessor, then remove the old entry, add finger in
- * finger table and send add_trail message to add the trail in the routing
- * table of all peers which are part of trail to reach from me to finger.
- * @param finger New peer which may be our predecessor.
- * @param trail List of peers to reach from @finger to me.
- * @param trail_length Total number of peer in @a trail.
- */
-static void
-compare_and_update_predecessor (const struct GNUNET_PeerIdentity *finger,
-                                const struct GNUNET_PeerIdentity *trail,
-                                unsigned int trail_length)
-{
-  struct FingerInfo *current_predecessor;
-  const struct GNUNET_PeerIdentity *closest_peer;
-  uint64_t predecessor_value;
-  unsigned int is_predecessor = 1;
-
-  current_predecessor = &finger_table[PREDECESSOR_FINGER_ID];
-  GNUNET_assert (0 != GNUNET_CRYPTO_cmp_peer_identity (finger,
-                                                       &my_identity));
-
-  /* No predecessor. Add finger as your predecessor. */
-  if (GNUNET_NO == current_predecessor->is_present)
-  {
-    update_predecessor (finger,
-                        trail,
-                        trail_length);
-    return;
-  }
-
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&current_predecessor->finger_identity,
-                                            finger))
-  {
-    return;
-  }
-
-  predecessor_value = compute_finger_identity_value (PREDECESSOR_FINGER_ID);
-  closest_peer = select_closest_peer (finger,
-                                      &current_predecessor->finger_identity,
-                                      predecessor_value,
-                                      is_predecessor);
-
-  /* Finger is the closest predecessor. Remove the existing one and add the new
-     one. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (closest_peer,
-                                            finger))
-  {
-    remove_existing_finger (current_predecessor,
-                            PREDECESSOR_FINGER_ID);
-    update_predecessor (finger,
-                        trail,
-                        trail_length);
-    return;
-  }
-}
-
-
-/**
- * Check format of a p2p verify successor messages.
- *
- * @param cls closure
- * @param vsm the message
- * @return #GNUNET_OK if @a vsm is well-formed
- */
-static int
-check_dht_p2p_verify_successor (void *cls,
-                                const struct PeerVerifySuccessorMessage *vsm)
-{
-  size_t msize;
-
-  msize = ntohs (vsm->header.size);
-  if ((msize - sizeof (struct PeerVerifySuccessorMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for p2p verify successor messages.
- *
- * @param cls closure
- * @param vsm the message
- */
-static void
-handle_dht_p2p_verify_successor (void *cls,
-                                 const struct PeerVerifySuccessorMessage *vsm)
-{
-  struct FriendInfo *friend = cls;
-  struct GNUNET_HashCode trail_id;
-  struct GNUNET_PeerIdentity successor;
-  struct GNUNET_PeerIdentity source_peer;
-  struct GNUNET_PeerIdentity *trail;
-  const struct GNUNET_PeerIdentity *next_hop;
-  struct FingerInfo current_predecessor;
-  struct FriendInfo *target_friend;
-  unsigned int trail_src_to_curr_pred_len = 0;
-  struct GNUNET_PeerIdentity *trail_src_to_curr_pred;
-  unsigned int trail_length;
-  size_t msize;
-
-  msize = ntohs (vsm->header.size);
-  trail_length = (msize - sizeof (struct PeerVerifySuccessorMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-
-  trail_id = vsm->trail_id;
-  source_peer = vsm->source_peer;
-  successor = vsm->successor;
-  trail = (struct GNUNET_PeerIdentity *)&vsm[1];
-
-  /* I am NOT the successor of source_peer. Pass the message to next_hop on
-   * the trail. */
-  if (0 != (GNUNET_CRYPTO_cmp_peer_identity (&successor,
-                                             &my_identity)))
-  {
-    next_hop = GDS_ROUTING_get_next_hop (&trail_id,
-                                         GDS_ROUTING_SRC_TO_DEST);
-    if (NULL == next_hop)
-      return;
-
-    target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                       next_hop);
-    if (NULL == target_friend)
-    {
-      GNUNET_break_op(0);
-      return;
-    }
-    GDS_NEIGHBOURS_send_verify_successor_message (&source_peer,
-                                                  &successor,
-                                                  &trail_id,
-                                                  trail,
-                                                  trail_length,
-                                                  target_friend);
-    return;
-  }
-
-  /* I am the destination of this message. */
-  /* Check if the source_peer could be our predecessor and if yes then update
-   * it.  */
-  compare_and_update_predecessor (&source_peer,
-                                  trail,
-                                  trail_length);
-  current_predecessor = finger_table[PREDECESSOR_FINGER_ID];
-
-  /* Is source of this message NOT my predecessor. */
-  if (0 != (GNUNET_CRYPTO_cmp_peer_identity (&current_predecessor.finger_identity,
-                                             &source_peer)))
-  {
-    trail_src_to_curr_pred
-      = get_trail_src_to_curr_pred (source_peer,
-                                    trail,
-                                    trail_length,
-                                    &trail_src_to_curr_pred_len);
-  }
-  else
-  {
-    trail_src_to_curr_pred_len = trail_length;
-    trail_src_to_curr_pred = GNUNET_new_array (trail_src_to_curr_pred_len,
-                                               struct GNUNET_PeerIdentity);
-
-    for (unsigned int i = 0; i < trail_src_to_curr_pred_len; i++)
-    {
-      trail_src_to_curr_pred[i] = trail[i];
-    }
-  }
-
-  GNUNET_assert (NULL !=
-                (target_friend =
-                 GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                    friend->id)));
-  GDS_NEIGHBOURS_send_verify_successor_result (&source_peer,
-                                               &my_identity,
-                                               &current_predecessor.finger_identity,
-                                               &trail_id,
-                                               trail_src_to_curr_pred,
-                                               trail_src_to_curr_pred_len,
-                                               GDS_ROUTING_DEST_TO_SRC,
-                                               target_friend);
-  GNUNET_free_non_null (trail_src_to_curr_pred);
-}
-
-
-/**
- * If the trail from me to my probable successor contains a friend not
- * at index 0, then we can shorten the trail.
- *
- * @param probable_successor Peer which is our probable successor
- * @param trail_me_to_probable_successor Peers in path from me to my probable
- *                                       successor, NOT including the endpoints.
- * @param trail_me_to_probable_successor_len Total number of peers in
- *                                           @a trail_me_to_probable_succesor.
- * @return Updated trail, if any friend found.
- *         Else the trail_me_to_probable_successor.
- */
-const struct GNUNET_PeerIdentity *
-check_trail_me_to_probable_succ (const struct GNUNET_PeerIdentity *probable_successor,
-                                 const struct GNUNET_PeerIdentity *trail_me_to_probable_successor,
-                                 unsigned int trail_me_to_probable_successor_len,
-                                 unsigned int *trail_to_new_successor_length)
-{
-  unsigned int i;
-  unsigned int j;
-  struct GNUNET_PeerIdentity *trail_to_new_successor;
-
-  /* Probable successor is  a friend */
-  if (NULL != GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                 probable_successor))
-  {
-    trail_to_new_successor = NULL;
-    *trail_to_new_successor_length = 0;
-    return trail_to_new_successor;
-  }
-
-  /* Is there any friend of yours in this trail. */
-  if (trail_me_to_probable_successor_len > 1)
-  {
-    for (i = trail_me_to_probable_successor_len - 1; i > 0; i--)
-    {
-      if (NULL == GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &trail_me_to_probable_successor[i]))
-        continue;
-
-      *trail_to_new_successor_length = (trail_me_to_probable_successor_len - i);
-      trail_to_new_successor = GNUNET_new_array (*trail_to_new_successor_length,
-                                                 struct GNUNET_PeerIdentity);
-      for (j = 0; j < *trail_to_new_successor_length; i++,j++)
-      {
-        trail_to_new_successor[j] = trail_me_to_probable_successor[i];
-      }
-
-      return trail_to_new_successor;
-    }
-  }
-
-  *trail_to_new_successor_length = trail_me_to_probable_successor_len;
-  return trail_me_to_probable_successor;
-}
-
-
-// TODO: Move up
-struct SendNotifyContext
-{
-  struct GNUNET_PeerIdentity source_peer;
-  struct GNUNET_PeerIdentity successor;
-  struct GNUNET_PeerIdentity *successor_trail;
-  unsigned int successor_trail_length;
-  struct GNUNET_HashCode succesor_trail_id;
-  struct FriendInfo *target_friend;
-  unsigned int num_retries_scheduled;
-};
-
-
-void
-send_notify_new_successor (void *cls);
-
-
-/**
- * Check if the peer which sent us verify successor result message is still ours
- * successor or not. If not, then compare existing successor and probable successor.
- * In case probable successor is the correct successor, remove the existing
- * successor. Add probable successor as new successor. Send notify new successor
- * message to new successor.
- * @param curr_succ Peer to which we sent the verify successor message. It may
- * or may not be our real current successor, as we may have few iterations of
- * find finger trail task.
- * @param probable_successor Peer which should be our successor accroding to @a
- *                           curr_succ
- * @param trail List of peers to reach from me to @a probable successor, NOT including
- *              endpoints.
- * @param trail_length Total number of peers in @a trail.
- */
-static void
-compare_and_update_successor (const struct GNUNET_PeerIdentity *curr_succ,
-                              const struct GNUNET_PeerIdentity *probable_successor,
-                              const struct GNUNET_PeerIdentity *trail,
-                              unsigned int trail_length)
-{
-  struct FingerInfo *current_successor;
-  const struct GNUNET_PeerIdentity *closest_peer;
-  struct GNUNET_HashCode trail_id;
-  const struct GNUNET_PeerIdentity *trail_me_to_probable_succ;
-  struct FriendInfo *target_friend;
-  unsigned int trail_me_to_probable_succ_len;
-  unsigned int is_predecessor = 0;
-  uint64_t successor_value;
-  struct SendNotifyContext *notify_ctx;
-
-  current_successor = &finger_table[0];
-  successor_value = compute_finger_identity_value(0);
-
-  /* If probable successor is same as current_successor, do nothing. */
-  if(0 == GNUNET_CRYPTO_cmp_peer_identity (probable_successor,
-                                           &current_successor->finger_identity))
-  {
-    if ((NULL != GDS_stats))
-    {
-      char *my_id_str;
-      uint64_t succ;
-      char *key;
-      uint64_t my_id;
-      GNUNET_memcpy (&my_id, &my_identity, sizeof(uint64_t));
-      my_id_str = GNUNET_strdup (GNUNET_i2s_full (&my_identity));
-      GNUNET_memcpy (&succ,
-                     &current_successor->finger_identity,
-                     sizeof(uint64_t));
-      succ = GNUNET_ntohll(succ);
-      GNUNET_asprintf (&key,
-                       "XDHT:%s:",
-                       my_id_str);
-      GNUNET_free (my_id_str);
-
-      GNUNET_STATISTICS_set (GDS_stats, key, succ, 0);
-      GNUNET_free (key);
-    }
-    if (send_verify_successor_task == NULL)
-      send_verify_successor_task =
-              GNUNET_SCHEDULER_add_delayed(verify_successor_next_send_time,
-                                           &send_verify_successor_message,
-                                           NULL);
-    return;
-  }
-  closest_peer = select_closest_peer (probable_successor,
-                                      &current_successor->finger_identity,
-                                      successor_value,
-                                      is_predecessor);
-
-  /* If the current_successor in the finger table is closest, then do nothing. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (closest_peer,
-                                            &current_successor->finger_identity))
-  {
-    //FIXME: Is this a good place to return the stats.
-    if ((NULL != GDS_stats))
-    {
-      char *my_id_str;
-      uint64_t succ;
-      char *key;
-
-      my_id_str = GNUNET_strdup (GNUNET_i2s_full (&my_identity));
-      GNUNET_memcpy(&succ, &current_successor->finger_identity, sizeof(uint64_t));
-      GNUNET_asprintf (&key, "XDHT:%s:", my_id_str);
-      GNUNET_free (my_id_str);
-      GNUNET_STATISTICS_set (GDS_stats, key, succ, 0);
-      GNUNET_free (key);
-    }
-
-    if(0 == successor_times)
-    {
-//      successor_times = 3;
-      verify_successor_next_send_time =
-              GNUNET_TIME_STD_BACKOFF (verify_successor_next_send_time);
-    }
-    else
-      successor_times--;
-
-
-    if (send_verify_successor_task == NULL)
-      send_verify_successor_task =
-              GNUNET_SCHEDULER_add_delayed (verify_successor_next_send_time,
-                                            &send_verify_successor_message,
-                                            NULL);
-    return;
-  }
-
-  /* Probable successor is the closest peer.*/
-  if(trail_length > 0)
-  {
-    GNUNET_assert(NULL != GNUNET_CONTAINER_multipeermap_get  (friend_peermap,
-                                                              &trail[0]));
-  }
-  else
-  {
-    GNUNET_assert(NULL != GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                             probable_successor));
-  }
-
-  trail_me_to_probable_succ_len = 0;
-  trail_me_to_probable_succ = check_trail_me_to_probable_succ (probable_successor,
-                                                               trail,
-                                                               trail_length,
-                                                               &trail_me_to_probable_succ_len);
-
-  /* Remove the existing successor. */
-  remove_existing_finger (current_successor, 0);
-   /* Generate a new trail id to reach to your new successor. */
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_STRONG,
-                              &trail_id,
-                              sizeof (trail_id));
-
-  if (trail_me_to_probable_succ_len > 0)
-  {
-    GDS_ROUTING_add (&trail_id,
-                     &my_identity,
-                     &trail_me_to_probable_succ[0]);
-    GNUNET_assert (NULL !=
-                  (target_friend =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      &trail_me_to_probable_succ[0])));
-  }
-  else
-  {
-    GDS_ROUTING_add (&trail_id,
-                     &my_identity,
-                     probable_successor);
-    GNUNET_assert (NULL !=
-                  (target_friend =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      probable_successor)));
-  }
-
-  add_new_finger (probable_successor,
-                  trail_me_to_probable_succ,
-                  trail_me_to_probable_succ_len,
-                  &trail_id,
-                  0);
-
-  notify_ctx = GNUNET_new (struct SendNotifyContext);
-
-  notify_ctx->source_peer = my_identity;
-  notify_ctx->successor = *probable_successor;
-  notify_ctx->successor_trail = GNUNET_new_array (trail_me_to_probable_succ_len,
-                                                  struct GNUNET_PeerIdentity);
-  GNUNET_memcpy (notify_ctx->successor_trail,
-                 trail_me_to_probable_succ,
-                 sizeof(struct GNUNET_PeerIdentity) * trail_me_to_probable_succ_len);
-  notify_ctx->successor_trail_length = trail_me_to_probable_succ_len;
-  notify_ctx->succesor_trail_id = trail_id;
-  notify_ctx->target_friend = target_friend;
-  notify_ctx->num_retries_scheduled = 0;
-
-  // TODO: Check if we should verify before schedule if already scheduled.
-  GNUNET_SCHEDULER_add_now (&send_notify_new_successor,
-                            notify_ctx);
-}
-
-
-void
-send_notify_new_successor (void *cls)
-{
-  struct SendNotifyContext *ctx = cls;
-
-  GDS_NEIGHBOURS_send_notify_new_successor (&ctx->source_peer,
-                                            &ctx->successor,
-                                            ctx->successor_trail,
-                                            ctx->successor_trail_length,
-                                            &ctx->succesor_trail_id,
-                                            ctx->target_friend);
-
-  if ( (0 == ctx->num_retries_scheduled) &&
-       (send_notify_new_successor_retry_task != NULL) )
-  {
-    // Result from previous notify successos hasn't arrived, so the retry task
-    // hasn't been cancelled! Already a new notify successor must be called.
-    // We will cancel the retry request.
-    struct SendNotifyContext *old_notify_ctx;
-
-    old_notify_ctx = GNUNET_SCHEDULER_cancel(send_notify_new_successor_retry_task);
-    GNUNET_free (old_notify_ctx->successor_trail);
-    GNUNET_free (old_notify_ctx);
-    send_notify_new_successor_retry_task = NULL;
-  }
-
-  ctx->num_retries_scheduled++;
-  send_notify_new_successor_retry_task
-    = GNUNET_SCHEDULER_add_delayed (notify_successor_retry_time,
-                                    &send_notify_new_successor,
-                                    cls);
-}
-
-
-/**
- * Check integrity of verify successor result messages.
- *
- * @param cls closure
- * @param vsrm the message
- * @return #GNUNET_OK if @a vrsm is well-formed
- */
-static int
-check_dht_p2p_verify_successor_result (void *cls,
-                                       const struct PeerVerifySuccessorResultMessage *vsrm)
-{
-  size_t msize;
-
-  msize = ntohs (vsrm->header.size);
-  if ((msize - sizeof (struct PeerVerifySuccessorResultMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for p2p verify successor result messages.
- *
- * @param cls closure
- * @param vsrm the message
- */
-static void
-handle_dht_p2p_verify_successor_result (void *cls,
-                                        const struct PeerVerifySuccessorResultMessage *vsrm)
-{
-  enum GDS_ROUTING_trail_direction trail_direction;
-  struct GNUNET_PeerIdentity querying_peer;
-  struct GNUNET_HashCode trail_id;
-  const struct GNUNET_PeerIdentity *next_hop;
-  struct FriendInfo *target_friend;
-  struct GNUNET_PeerIdentity probable_successor;
-  struct GNUNET_PeerIdentity current_successor;
-  const struct GNUNET_PeerIdentity *trail;
-  unsigned int trail_length;
-  size_t msize;
-
-  msize = ntohs (vsrm->header.size);
-  trail_length = (msize - sizeof (struct PeerVerifySuccessorResultMessage))
-    / sizeof (struct GNUNET_PeerIdentity);
-
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-
-  trail = (const struct GNUNET_PeerIdentity *) &vsrm[1];
-  querying_peer = vsrm->querying_peer;
-  trail_direction = ntohl (vsrm->trail_direction);
-  trail_id = vsrm->trail_id;
-  probable_successor = vsrm->probable_successor;
-  current_successor = vsrm->current_successor;
-
-  /* Am I the querying_peer? */
-  if (0 == (GNUNET_CRYPTO_cmp_peer_identity (&querying_peer,
-                                             &my_identity)))
-  {
-    /* Cancel Retry Task */
-    if (NULL != send_verify_successor_retry_task)
-    {
-      struct VerifySuccessorContext *ctx;
-
-      ctx = GNUNET_SCHEDULER_cancel (send_verify_successor_retry_task);
-      GNUNET_free (ctx);
-      send_verify_successor_retry_task = NULL;
-    }
-    compare_and_update_successor (&current_successor,
-                                  &probable_successor,
-                                  trail,
-                                  trail_length);
-    return;
-  }
-
-  /*If you are not the querying peer then pass on the message */
-  if(NULL == (next_hop =
-              GDS_ROUTING_get_next_hop (&trail_id,
-                                        trail_direction)))
-  {
-    /* Here it may happen that source peer has found a new successor, and removed
-     the trail, Hence no entry found in the routing table. Fail silently.*/
-    DEBUG (" NO ENTRY FOUND IN %s ROUTING TABLE for trail id %s, line %u",
-           GNUNET_i2s (&my_identity),
-           GNUNET_h2s (&trail_id),
-           __LINE__);
-    GNUNET_break_op(0);
-    return;
-  }
-  if (NULL == (target_friend =
-               GNUNET_CONTAINER_multipeermap_get (friend_peermap, next_hop)))
-  {
-    GNUNET_break_op(0);
-    return;
-  }
-  GDS_NEIGHBOURS_send_verify_successor_result (&querying_peer,
-                                               &vsrm->current_successor,
-                                               &probable_successor,
-                                               &trail_id,
-                                               trail,
-                                               trail_length,
-                                               trail_direction,
-                                               target_friend);
-}
-
-
-/**
- * Check integrity of p2p notify new successor messages.
- *
- * @param cls closure
- * @param nsm the message
- * @return #GNUNET_OK if @a nsm is well-formed
- */
-static int
-check_dht_p2p_notify_new_successor (void *cls,
-                                    const struct PeerNotifyNewSuccessorMessage *nsm)
-{
-  size_t msize;
-
-  msize = ntohs (nsm->header.size);
-  if ((msize - sizeof (struct PeerNotifyNewSuccessorMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for p2p notify new successor messages.
- *
- * @param cls closure
- * @param nsm the message
- */
-static void
-handle_dht_p2p_notify_new_successor (void *cls,
-                                     const struct PeerNotifyNewSuccessorMessage *nsm)
-{
-  struct FriendInfo *friend = cls;
-  const struct GNUNET_PeerIdentity *trail;
-  struct GNUNET_PeerIdentity source;
-  struct GNUNET_PeerIdentity new_successor;
-  struct GNUNET_HashCode trail_id;
-  struct GNUNET_PeerIdentity next_hop;
-  struct FriendInfo *target_friend;
-  int my_index;
-  size_t msize;
-  uint32_t trail_length;
-
-  msize = ntohs (nsm->header.size);
-  trail_length = (msize - sizeof (struct PeerNotifyNewSuccessorMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-  trail = (const struct GNUNET_PeerIdentity *) &nsm[1];
-  source = nsm->source_peer;
-  new_successor = nsm->new_successor;
-  trail_id = nsm->trail_id;
-
-  /* I am the new_successor to source_peer. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &new_successor))
-  {
-    if (trail_length > 0)
-      GNUNET_assert(0 == GNUNET_CRYPTO_cmp_peer_identity (&trail[trail_length - 1],
-                                                          friend->id));
-    else
-      GNUNET_assert(0 == GNUNET_CRYPTO_cmp_peer_identity (&source,
-                                                          friend->id));
-
-    compare_and_update_predecessor (&source,
-                                    trail,
-                                    trail_length);
-    target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                       friend->id);
-    GNUNET_assert (NULL != target_friend);
-    GDS_NEIGHBOURS_send_notify_succcessor_confirmation (&trail_id,
-                                                        GDS_ROUTING_DEST_TO_SRC,
-                                                        target_friend);
-    return;
-  }
-
-  GNUNET_assert(trail_length > 0);
-  /* I am part of trail to reach to successor. */
-  my_index = search_my_index (trail, trail_length);
-  if (-1 == my_index)
-  {
-    DEBUG ("No entry found in trail\n");
-    GNUNET_break_op (0);
-    return;
-  }
-  if((trail_length + 1) == my_index)
-  {
-    DEBUG ("Found twice in trail.\n");
-    GNUNET_break_op (0);
-    return;
-  }
-  if ((trail_length-1) == my_index)
-    next_hop = new_successor;
-  else
-    next_hop = trail[my_index + 1];
-
-  GDS_ROUTING_add (&trail_id,
-                   friend->id,
-                   &next_hop);
-  target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &next_hop);
-  if (NULL == target_friend)
-  {
-    GNUNET_break(0);
-    return;
-  }
-  GDS_NEIGHBOURS_send_notify_new_successor (&source,
-                                            &new_successor,
-                                            trail,
-                                            trail_length,
-                                            &trail_id,
-                                            target_friend);
-}
-
-
-/**
- * Core handler for P2P notify successor message
- *
- * @param cls closure
- * @param notify_confirmation the message
- */
-static void
-handle_dht_p2p_notify_succ_confirmation (void *cls,
-                                         const struct PeerNotifyConfirmationMessage *notify_confirmation)
-{
-  enum GDS_ROUTING_trail_direction trail_direction;
-  struct GNUNET_HashCode trail_id;
-  struct FriendInfo *target_friend;
-  const struct GNUNET_PeerIdentity *next_hop;
-
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            ntohs (notify_confirmation->header.size),
-                            GNUNET_NO);
-  trail_direction = ntohl (notify_confirmation->trail_direction);
-  trail_id = notify_confirmation->trail_id;
-
-  next_hop = GDS_ROUTING_get_next_hop (&trail_id,
-                                       trail_direction);
-  if (NULL == next_hop)
-  {
-    /* The source of notify new successor, might have found even a better
-     successor. In that case it send a trail teardown message, and hence,
-     the next hop is NULL. */
-    //Fixme: Add some print to confirm the above theory.
-    return;
-  }
-
-  /* I peer which sent the notify successor message to the successor. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (next_hop,
-                                            &my_identity))
-  {
-   /*
-    * Schedule another round of verify sucessor with your current successor
-    * which may or may not be source of this message. This message is used
-    * only to ensure that we have a path setup to reach to our successor.
-    */
-
-    // TODO: cancel schedule of notify_successor_retry_task
-    if (send_notify_new_successor_retry_task != NULL)
-    {
-      struct SendNotifyContext *notify_ctx;
-      notify_ctx = GNUNET_SCHEDULER_cancel(send_notify_new_successor_retry_task);
-      GNUNET_free (notify_ctx->successor_trail);
-      GNUNET_free (notify_ctx);
-      send_notify_new_successor_retry_task = NULL;
-    }
-    if (send_verify_successor_task == NULL)
-    {
-      verify_successor_next_send_time.rel_value_us =
-      DHT_SEND_VERIFY_SUCCESSOR_INTERVAL.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_SEND_VERIFY_SUCCESSOR_INTERVAL.rel_value_us);
-      send_verify_successor_task
-        = GNUNET_SCHEDULER_add_delayed(verify_successor_next_send_time,
-                                       &send_verify_successor_message,
-                                       NULL);
-    }
-  }
-  else
-  {
-    target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                       next_hop);
-    if (NULL == target_friend)
-      {
-      DEBUG ("\n friend not found, line number = %d",
-             __LINE__);
-      return;
-    }
-    GDS_NEIGHBOURS_send_notify_succcessor_confirmation  (&trail_id,
-                                                         GDS_ROUTING_DEST_TO_SRC,
-                                                         target_friend);
-  }
-}
-
-
-/**
- * Check integrity of P2P trail rejection message
- *
- * @param cls closure
- * @param trail_rejection the message
- * @return #GNUNET_OK if @a trail_rejection is well-formed
- */
-static int
-check_dht_p2p_trail_setup_rejection (void *cls,
-                                     const struct PeerTrailRejectionMessage *trail_rejection)
-{
-  size_t msize;
-
-  msize = ntohs (trail_rejection->header.size);
-  if ((msize - sizeof (struct PeerTrailRejectionMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handler for P2P trail rejection message
- *
- * @param cls closure
- * @param trail_rejection the message
- */
-static void
-handle_dht_p2p_trail_setup_rejection (void *cls,
-                                      const struct PeerTrailRejectionMessage *trail_rejection)
-{
-  struct FriendInfo *friend = cls;
-  unsigned int trail_length;
-  const struct GNUNET_PeerIdentity *trail_peer_list;
-  struct FriendInfo *target_friend;
-  struct GNUNET_TIME_Relative congestion_timeout;
-  struct GNUNET_HashCode trail_id;
-  struct GNUNET_PeerIdentity next_peer;
-  struct GNUNET_PeerIdentity source;
-  uint64_t ultimate_destination_finger_value;
-  unsigned int is_predecessor;
-  struct Closest_Peer successor;
-  size_t msize;
-
-  msize = ntohs (trail_rejection->header.size);
-  trail_length = (msize - sizeof (struct PeerTrailRejectionMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-
-  trail_peer_list = (const struct GNUNET_PeerIdentity *) &trail_rejection[1];
-  is_predecessor = ntohl (trail_rejection->is_predecessor);
-  congestion_timeout = trail_rejection->congestion_time;
-  source = trail_rejection->source_peer;
-  trail_id = trail_rejection->trail_id;
-  ultimate_destination_finger_value
-    = GNUNET_ntohll (trail_rejection->ultimate_destination_finger_value);
-  /* First set the congestion time of the friend that sent you this message. */
-  target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     friend->id);
-  if (NULL == target_friend)
-  {
-    DEBUG ("\nLINE = %d ,No friend found.",__LINE__);
-    GNUNET_break(0);
-    return;
-  }
-  target_friend->congestion_timestamp
-    = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(),
-                                congestion_timeout);
-
-  /* I am the source peer which wants to setup the trail. Do nothing.
-   * send_find_finger_trail_task is scheduled periodically.*/
-  if(0 == (GNUNET_CRYPTO_cmp_peer_identity (&my_identity, &source)))
-    return;
-
-  /* If I am congested then pass this message to peer before me in trail. */
-  if (GNUNET_YES == GDS_ROUTING_threshold_reached())
-  {
-    /* First remove yourself from the trail. */
-    unsigned int new_trail_length = trail_length - 1;
-    struct GNUNET_PeerIdentity trail[new_trail_length];
-
-    GNUNET_memcpy (trail,
-                   trail_peer_list,
-                   new_trail_length * sizeof(struct GNUNET_PeerIdentity));
-    if (0 == trail_length)
-      next_peer = source;
-    else
-      next_peer = trail[new_trail_length-1];
-
-    target_friend
-      = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                           &next_peer);
-    if (NULL == target_friend)
-    {
-      DEBUG ("\nLINE = %d ,No friend found.",
-             __LINE__);
-      GNUNET_break(0);
-      return;
-    }
-    GDS_NEIGHBOURS_send_trail_rejection (&source,
-                                         ultimate_destination_finger_value,
-                                         &my_identity,
-                                         is_predecessor,
-                                         trail,
-                                         new_trail_length,
-                                         &trail_id,
-                                         target_friend,
-                                         CONGESTION_TIMEOUT);
-    return;
-  }
-
-  successor = find_local_best_known_next_hop (ultimate_destination_finger_value,
-                                              is_predecessor);
-
-  /* Am I the final destination? */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&successor.best_known_destination,
-                                            &my_identity))
-  {
-     /*Here you are already part of trail. Copy the trail removing yourself. */
-    unsigned int new_trail_length = trail_length - 1;
-    struct GNUNET_PeerIdentity trail[new_trail_length];
-
-    GNUNET_memcpy (trail,
-                   trail_peer_list,
-                   new_trail_length * sizeof(struct GNUNET_PeerIdentity));
-
-    if (0 == new_trail_length)
-      next_peer = source;
-    else
-    {
-      next_peer = trail[new_trail_length-1];
-    }
-    target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                       &next_peer);
-
-    if (NULL == target_friend)
-    {
-      DEBUG ("\nLINE = %d ,No friend found.",
-             __LINE__);
-      GNUNET_break(0);
-      return;
-    }
-    GDS_NEIGHBOURS_send_trail_setup_result (&source,
-                                            &my_identity,
-                                            target_friend,
-                                            new_trail_length,
-                                            trail,
-                                            is_predecessor,
-                                            ultimate_destination_finger_value,
-                                            &trail_id);
-    return;
-  }
-  /* Here I was already part of trail. So no need to add. */
-  target_friend = GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                     &successor.next_hop);
-  if (NULL == target_friend)
-  {
-    DEBUG ("\nLINE = %d ,No friend found.",__LINE__);
-    GNUNET_break (0);
-    return;
-  }
-  GDS_NEIGHBOURS_send_trail_setup (&source,
-                                   ultimate_destination_finger_value,
-                                   &successor.best_known_destination,
-                                   target_friend,
-                                   trail_length,
-                                   trail_peer_list,
-                                   is_predecessor,
-                                   &trail_id,
-                                   &successor.trail_id);
-}
-
-
-/**
- * Core handler for trail teardown message.
- *
- * @param cls closure
- * @param trail_teardown the message
- */
-static void
-handle_dht_p2p_trail_teardown (void *cls,
-                               const struct PeerTrailTearDownMessage *trail_teardown)
-{
-  enum GDS_ROUTING_trail_direction trail_direction;
-  struct GNUNET_HashCode trail_id;
-  const struct GNUNET_PeerIdentity *next_hop;
-  size_t msize;
-
-  msize = ntohs (trail_teardown->header.size);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-  trail_direction = ntohl (trail_teardown->trail_direction);
-  trail_id = trail_teardown->trail_id;
-
-  /* Check if peer is the real peer from which we should get this message.*/
-  /* Get the prev_hop for this trail by getting the next hop in opposite direction. */
-#if 0
-  GNUNET_assert (NULL != (prev_hop =
-                 GDS_ROUTING_get_next_hop (trail_id, ! trail_direction)));
-  if (0 != GNUNET_CRYPTO_cmp_peer_identity (prev_hop,
-                                            friend->id))
-  {
-    GNUNET_break (0);
-    return;
-  }
-#endif
-
-  next_hop = GDS_ROUTING_get_next_hop (&trail_id,
-                                       trail_direction);
-  if (NULL == next_hop)
-  {
-    DEBUG(" NO ENTRY FOUND IN %s ROUTING TABLE for trail id %s, line %u",
-          GNUNET_i2s (&my_identity),
-          GNUNET_h2s (&trail_id),
-          __LINE__);
-    GNUNET_break (0);
-    return;
-  }
-
-  /* I am the next hop, which means I am the final destination. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (next_hop, &my_identity))
-  {
-    GNUNET_assert (GNUNET_YES ==
-                   GDS_ROUTING_remove_trail (&trail_id));
-    return;
-  }
-  /* If not final destination, then send a trail teardown message to next hop.*/
-  GNUNET_assert (NULL != GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                            next_hop));
-  GNUNET_assert (GNUNET_YES ==
-                 GDS_ROUTING_remove_trail (&trail_id));
-  GDS_NEIGHBOURS_send_trail_teardown (&trail_id,
-                                      trail_direction,
-                                      next_hop);
-}
-
-
-/**
- * Check validity of p2p add trail message.
- *
- * @param cls closure
- * @param add_trail the message
- * @return #GNUNET_OK if @a add_trail is well-formed
- */
-static int
-check_dht_p2p_add_trail (void *cls,
-                         const struct PeerAddTrailMessage *add_trail)
-{
-  size_t msize;
-
-  msize = ntohs (add_trail->header.size);
-  if ((msize - sizeof (struct PeerAddTrailMessage)) %
-      sizeof (struct GNUNET_PeerIdentity) != 0)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Core handle for p2p add trail message.
- *
- * @param cls closure
- * @param add_trail the message
- */
-static void
-handle_dht_p2p_add_trail (void *cls,
-                          const struct PeerAddTrailMessage *add_trail)
-{
-  struct FriendInfo *friend = cls;
-  const struct GNUNET_PeerIdentity *trail;
-  struct GNUNET_HashCode trail_id;
-  struct GNUNET_PeerIdentity destination_peer;
-  struct GNUNET_PeerIdentity source_peer;
-  struct GNUNET_PeerIdentity next_hop;
-  unsigned int trail_length;
-  unsigned int my_index;
-  size_t msize;
-
-  msize = ntohs (add_trail->header.size);
-  /* In this message we pass the whole trail from source to destination as we
-   * are adding that trail.*/
-  //FIXME: failed when run with 1000 pears. check why.
-  trail_length = (msize - sizeof (struct PeerAddTrailMessage))/
-                  sizeof (struct GNUNET_PeerIdentity);
-  GNUNET_STATISTICS_update (GDS_stats,
-                            gettext_noop ("# Bytes received from other peers"),
-                            msize,
-                            GNUNET_NO);
-
-  trail = (const struct GNUNET_PeerIdentity *) &add_trail[1];
-  destination_peer = add_trail->destination_peer;
-  source_peer = add_trail->source_peer;
-  trail_id = add_trail->trail_id;
-
-  /* I am not the destination of the trail. */
-  if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                            &destination_peer))
-  {
-    struct FriendInfo *target_friend;
-
-    /* Get my location in the trail. */
-    my_index = search_my_index (trail, trail_length);
-    if (-1 == my_index)
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    if((trail_length + 1) == my_index)
-    {
-      DEBUG ("Found twice in trail.\n");
-      GNUNET_break_op (0);
-      return;
-    }
-    if ((trail_length - 1) == my_index)
-    {
-      next_hop = destination_peer;
-    }
-    else
-    {
-      next_hop = trail[my_index + 1];
-    }
-    /* Add in your routing table. */
-    GNUNET_assert (GNUNET_OK == GDS_ROUTING_add (&trail_id,
-                                                 friend->id,
-                                                 &next_hop));
-    //GNUNET_assert (GNUNET_OK == GDS_ROUTING_add (trail_id, next_hop, *peer));
-    GNUNET_assert (NULL !=
-                  (target_friend =
-                   GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                      &next_hop)));
-    GDS_NEIGHBOURS_send_add_trail (&source_peer,
-                                   &destination_peer,
-                                   &trail_id,
-                                   trail,
-                                   trail_length,
-                                   target_friend);
-    return;
-  }
-  /* I am the destination. Add an entry in routing table. */
-  GNUNET_assert (GNUNET_OK == GDS_ROUTING_add (&trail_id,
-                                               friend->id,
-                                               &my_identity));
-}
-
-
-/**
- * Free the finger trail in which the first friend to reach to a finger is
- * disconnected_friend. Also remove entry from routing table for that particular
- * trail id.
- * @param disconnected_friend PeerIdentity of friend which got disconnected
- * @param remove_finger Finger whose trail we need to check if it has
- *                      disconnected_friend as the first hop.
- * @return Total number of trails in which disconnected_friend was the first
- *         hop.
- */
-static int
-remove_matching_trails (const struct GNUNET_PeerIdentity *disconnected_friend,
-                        struct FingerInfo *finger)
-{
-  const struct GNUNET_PeerIdentity *next_hop;
-  struct FriendInfo *remove_friend;
-  struct Trail *current_trail;
-  unsigned int matching_trails_count = 0;
-  int i;
-
-  /* Iterate over all the trails of finger. */
-  for (i = 0; i < finger->trails_count; i++)
-  {
-    current_trail = &finger->trail_list[i];
-    if (GNUNET_NO == current_trail->is_present)
-      continue;
-
-    /* First friend to reach to finger is disconnected_peer. */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (&current_trail->trail_head->peer,
-                                              disconnected_friend))
-    {
-      remove_friend =
-                     GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                        disconnected_friend);
-      GNUNET_assert (NULL != remove_friend);
-      next_hop = GDS_ROUTING_get_next_hop (&current_trail->trail_id,
-                                           GDS_ROUTING_SRC_TO_DEST);
-
-      /* Here it may happen that as all the peers got disconnected, the entry in
-       routing table for that particular trail has been removed, because the
-       previously disconnected peer was either a next hop or prev hop of that
-       peer. */
-      if (NULL != next_hop)
-      {
-        GNUNET_assert (0 == (GNUNET_CRYPTO_cmp_peer_identity (disconnected_friend,
-                                                              next_hop)));
-        GNUNET_assert (GNUNET_YES ==
-                       GDS_ROUTING_remove_trail (&current_trail->trail_id));
-      }
-      matching_trails_count++;
-      free_trail (current_trail);
-      current_trail->is_present = GNUNET_NO;
-    }
-  }
-  return matching_trails_count;
-}
-
-
-/**
- * Iterate over finger_table entries.
- * 0. Ignore finger which is my_identity or if no valid entry present at
- *    that finger index.
- * 1. If disconnected_friend is a finger, then remove the routing entry from
-      your own table. Free the trail.
- * 2. Check if disconnected_friend is the first friend in the trail to reach to a finger.
- *   2.1 Remove all the trails and entry from routing table in which disconnected
- *       friend is the first friend in the trail. If disconnected_friend is the
- *       first friend in all the trails to reach finger, then remove the finger.
- * @param disconnected_friend Peer identity of friend which got disconnected.
- */
-static void
-remove_matching_fingers (const struct GNUNET_PeerIdentity *disconnected_peer)
-{
-  struct FingerInfo *current_finger;
-  int removed_trails_count;
-  int i;
-
-  /* Iterate over finger table entries. */
-  for (i = 0; i < MAX_FINGERS; i++)
-  {
-    current_finger = &finger_table[i];
-
-    /* No finger stored at this trail index or I am the finger. */
-    if ((GNUNET_NO == current_finger->is_present) ||
-        (0 == GNUNET_CRYPTO_cmp_peer_identity (&current_finger->finger_identity,
-                                               &my_identity)))
-      continue;
-
-    /* Is disconnected_peer a finger? */
-    if (0 == GNUNET_CRYPTO_cmp_peer_identity (disconnected_peer,
-                                              &current_finger->finger_identity))
-    {
-      remove_existing_finger (current_finger, i);
-    }
-
-    /* If finger is a friend but not disconnected_friend, then continue. */
-    if (NULL != GNUNET_CONTAINER_multipeermap_get (friend_peermap,
-                                                   &current_finger->finger_identity))
-      continue;
-
-    /* Iterate over the list of trails to reach remove_finger. Check if
-     * disconnected_friend is the first friend in any of the trail. */
-    removed_trails_count = remove_matching_trails (disconnected_peer,
-                                                   current_finger);
-    current_finger->trails_count =
-            current_finger->trails_count - removed_trails_count;
-    if (0 == current_finger->trails_count)
-    {
-      current_finger->is_present = GNUNET_NO;
-      memset (&finger_table[i],
-              0,
-              sizeof (finger_table[i]));
-    }
-  }
-}
-
-
-/**
- * Method called whenever a peer disconnects.
- *
- * @param cls closure
- * @param peer peer identity this notification is about
- * @param internal_cls our `struct FriendInfo` for @a peer
- */
-static void
-handle_core_disconnect (void *cls,
-                        const struct GNUNET_PeerIdentity *peer,
-                        void *internal_cls)
-{
-  struct FriendInfo *remove_friend = internal_cls;
-
-  /* If disconnected to own identity, then return. */
-  if (NULL == remove_friend)
-    return;
-  remove_matching_fingers (peer);
-  GNUNET_assert (GNUNET_SYSERR !=
-                 GDS_ROUTING_remove_trail_by_peer (peer));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_remove (friend_peermap,
-                                                       peer,
-                                                       remove_friend));
-  if (0 != GNUNET_CONTAINER_multipeermap_size (friend_peermap))
-    return;
-
-  if (NULL != find_finger_trail_task)
-  {
-    GNUNET_SCHEDULER_cancel (find_finger_trail_task);
-    find_finger_trail_task = NULL;
-  }
-  else
-    GNUNET_break (0);
-}
-
-
-/**
- * Method called whenever a peer connects.
- *
- * @param cls closure
- * @param peer_identity peer identity this notification is about
- * @param mq message queue for sending data to @a peer
- * @return our `struct FriendInfo` for this peer
- */
-static void *
-handle_core_connect (void *cls,
-                     const struct GNUNET_PeerIdentity *peer_identity,
-                     struct GNUNET_MQ_Handle *mq)
-{
-  struct FriendInfo *friend;
-
-  /* Check for connect to self message */
-  if (0 == memcmp (&my_identity,
-                   peer_identity,
-                   sizeof (struct GNUNET_PeerIdentity)))
-    return NULL;
-  friend = GNUNET_new (struct FriendInfo);
-  friend->id = peer_identity;
-  friend->mq = mq;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multipeermap_put (friend_peermap,
-                                                    friend->id,
-                                                    friend,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  /* FIXME: now we are not making a distinction between fingers which are friends
-   * also.But later, we should add a congestion timestamp on the friend, so that it is
-   * selected after some time out. This is to ensure that both peers have added
-   * each other as their friend. */
-  /* Got a first connection, good time to start with FIND FINGER TRAIL requests...*/
-  if (NULL == find_finger_trail_task)
-  {
-    find_finger_trail_task
-      = GNUNET_SCHEDULER_add_now (&send_find_finger_trail_message,
-                                  NULL);
-  }
-  return friend;
-}
-
-
-/**
- * To be called on core init/fail.
- *
- * @param cls service closure
- * @param identity the public identity of this peer
- */
-static void
-core_init (void *cls,
-           const struct GNUNET_PeerIdentity *identity)
-{
-  my_identity = *identity;
-}
-
-
-/**
- * Initialize finger table entries.
- */
-static void
-finger_table_init ()
-{
-  memset (&finger_table, 0, sizeof (finger_table));
-}
-
-
-/**
- * Initialize neighbours subsystem.
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-int
-GDS_NEIGHBOURS_init (void)
-{
-  struct GNUNET_MQ_MessageHandler core_handlers[] = {
-    GNUNET_MQ_hd_var_size (dht_p2p_put,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_PUT,
-                           struct PeerPutMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_get,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_GET,
-                           struct PeerGetMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_get_result,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_GET_RESULT,
-                           struct PeerGetResultMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_trail_setup,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP,
-                           struct PeerTrailSetupMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_trail_setup_result,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_RESULT,
-                           struct PeerTrailSetupResultMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_verify_successor,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR,
-                           struct PeerVerifySuccessorMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_verify_successor_result,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_VERIFY_SUCCESSOR_RESULT,
-                           struct PeerVerifySuccessorResultMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_notify_new_successor,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_NOTIFY_NEW_SUCCESSOR,
-                           struct PeerNotifyNewSuccessorMessage,
-                           NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_trail_setup_rejection,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_SETUP_REJECTION,
-                           struct PeerTrailRejectionMessage,
-                           NULL),
-    GNUNET_MQ_hd_fixed_size (dht_p2p_trail_teardown,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_TRAIL_TEARDOWN,
-                             struct PeerTrailTearDownMessage,
-                             NULL),
-    GNUNET_MQ_hd_var_size (dht_p2p_add_trail,
-                           GNUNET_MESSAGE_TYPE_XDHT_P2P_ADD_TRAIL,
-                           struct PeerAddTrailMessage,
-                           NULL),
-    GNUNET_MQ_hd_fixed_size (dht_p2p_notify_succ_confirmation,
-                             GNUNET_MESSAGE_TYPE_XDHT_P2P_NOTIFY_SUCCESSOR_CONFIRMATION,
-                             struct PeerNotifyConfirmationMessage,
-                             NULL),
-    GNUNET_MQ_handler_end ()
-  };
-
-  core_api = GNUNET_CORE_connect (GDS_cfg,
-                                  NULL,
-                                  &core_init,
-                                  &handle_core_connect,
-                                  &handle_core_disconnect,
-                                  core_handlers);
-  if (NULL == core_api)
-    return GNUNET_SYSERR;
-  friend_peermap = GNUNET_CONTAINER_multipeermap_create (256,
-                                                         GNUNET_YES);
-  finger_table_init ();
-  successor_times = 10;
-  fingers_round_count = 5;
-  find_finger_trail_task_next_send_time.rel_value_us =
-      DHT_FIND_FINGER_TRAIL_INTERVAL.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_FIND_FINGER_TRAIL_INTERVAL.rel_value_us);
-
-  verify_successor_next_send_time.rel_value_us =
-      DHT_SEND_VERIFY_SUCCESSOR_INTERVAL.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_SEND_VERIFY_SUCCESSOR_INTERVAL.rel_value_us);
-
-  verify_successor_retry_time.rel_value_us =
-      DHT_SEND_VERIFY_SUCCESSOR_RETRY_INTERVAL.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_SEND_VERIFY_SUCCESSOR_RETRY_INTERVAL.rel_value_us);
-
-  notify_successor_retry_time.rel_value_us =
-      DHT_SEND_NOTIFY_SUCCESSOR_RETRY_INTERVAL.rel_value_us +
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                DHT_SEND_NOTIFY_SUCCESSOR_RETRY_INTERVAL.rel_value_us);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Free the memory held up by trails of a finger.
- */
-static void
-delete_finger_table_entries()
-{
-  for (unsigned int i = 0; i < MAX_FINGERS; i++)
-  {
-    if (GNUNET_YES != finger_table[i].is_present)
-      continue;
-    for (unsigned int j = 0; j < finger_table[i].trails_count; j++)
-      free_trail(&finger_table[i].trail_list[j]);
-  }
-}
-
-
-/**
- * Shutdown neighbours subsystem.
- */
-void
-GDS_NEIGHBOURS_done (void)
-{
-  if (NULL == core_api)
-    return;
-
-  GNUNET_CORE_disconnect (core_api);
-  core_api = NULL;
-
-  delete_finger_table_entries();
-  GNUNET_assert (0 == GNUNET_CONTAINER_multipeermap_size (friend_peermap));
-  GNUNET_CONTAINER_multipeermap_destroy (friend_peermap);
-  friend_peermap = NULL;
-
-  if (NULL != find_finger_trail_task)
-  {
-    GNUNET_SCHEDULER_cancel (find_finger_trail_task);
-    find_finger_trail_task = NULL;
-  }
-
-  if (NULL != send_verify_successor_task)
-  {
-    GNUNET_SCHEDULER_cancel (send_verify_successor_task);
-    send_verify_successor_task = NULL;
-  }
-  if (NULL != send_verify_successor_retry_task)
-  {
-    struct VerifySuccessorContext *ctx;
-
-    ctx = GNUNET_SCHEDULER_cancel (send_verify_successor_retry_task);
-    GNUNET_free (ctx);
-    send_verify_successor_retry_task = NULL;
-  }
-  if (NULL != send_notify_new_successor_retry_task)
-  {
-    struct SendNotifyContext *notify_ctx;
-
-    notify_ctx = GNUNET_SCHEDULER_cancel (send_notify_new_successor_retry_task);
-    GNUNET_free (notify_ctx->successor_trail);
-    GNUNET_free (notify_ctx);
-    send_notify_new_successor_retry_task = NULL;
-  }
-}
-
-
-/**
- * Get my identity
- *
- * @return my identity
- */
-struct GNUNET_PeerIdentity *
-GDS_NEIGHBOURS_get_id (void)
-{
-  return &my_identity;
-}
-
-/* end of gnunet-service-xdht_neighbours.c */
diff --git a/src/dht/gnunet-service-xdht_neighbours.h b/src/dht/gnunet-service-xdht_neighbours.h
deleted file mode 100644
index 2fb118448..000000000
--- a/src/dht/gnunet-service-xdht_neighbours.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_neighbours.h
- * @brief GNUnet DHT routing code
- * @author Supriti Singh
- */
-
-#ifndef GNUNET_SERVICE_XDHT_NEIGHBOURS_H
-#define GNUNET_SERVICE_XDHT_NEIGHBOURS_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_dht_service.h"
-
-
-/**
- * Construct a trail teardown message and forward it to target friend.
- * @param trail_id Unique identifier of the trail.
- * @param trail_direction Direction of trail.
- * @param target_friend Friend to get this message.
- */
-void
-GDS_NEIGHBOURS_send_trail_teardown (const struct GNUNET_HashCode *trail_id,
-                                    unsigned int trail_direction,
-                                    const struct GNUNET_PeerIdentity *peer);
-
-
-#endif
diff --git a/src/dht/gnunet-service-xdht_routing.c b/src/dht/gnunet-service-xdht_routing.c
deleted file mode 100644
index ec7361579..000000000
--- a/src/dht/gnunet-service-xdht_routing.c
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 - 2014 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_routing.c
- * @brief GNUnet DHT tracking of requests for routing replies
- * @author Supriti Singh
- */
-#include "platform.h"
-#include "gnunet-service-dht_neighbours.h"
-#include "gnunet-service-xdht_neighbours.h"
-#include "gnunet-service-xdht_routing.h"
-#include "gnunet-service-dht.h"
-
-
-/**
- * FIXME: Check if its better to store pointer to friend rather than storing
- * peer identity next_hop or prev_hop.
- * keep entries in destnation and source peer also. so when we send the trail
- * teardown message then we don't know the source but if source gets the message
- * then it shold remove that trail id from its finger table. But how does
- * source know what is the desination finger ? It will whenevr contact a trail
- * will do a lookup in routing table and if no trail id present the remove
- * that trail of the finger and if only one trail then remove the finger.
- * because of this use case of trail teardown I think trail compression
- * and trail teardown should not be merged.
- * 2. store a pointer to friendInfo in place o peer identity.
- */
-/**
- * Maximum number of entries in routing table.
- */
-#define ROUTING_TABLE_THRESHOLD 80000
-
-/**
- * FIXME: Store friend pointer instead of peer identifier.
- * Routing table entry .
- */
-struct RoutingTrail
-{
-  /**
-   * Global Unique identifier of the trail.
-   */
-  struct GNUNET_HashCode trail_id;
-
-  /**
-   * The peer to which this request should be passed to.
-   */
-  struct GNUNET_PeerIdentity next_hop;
-
-  /**
-   * Peer just before next hop in the trail.
-   */
-  struct GNUNET_PeerIdentity prev_hop;
-};
-
-/**
- * Routing table of the peer
- */
-static struct GNUNET_CONTAINER_MultiHashMap *routing_table;
-
-/**
- * Update the prev. hop of the trail. Call made by trail compression where
- * if you are the first friend now in the trail then you need to update
- * your prev. hop.
- * @param trail_id
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case no matching entry found in routing table.
- */
-int
-GDS_ROUTING_update_trail_prev_hop (const struct GNUNET_HashCode *trail_id,
-                                   const struct GNUNET_PeerIdentity *prev_hop)
-{
-  struct RoutingTrail *trail;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (routing_table,
-                                             trail_id);
-
-  if (NULL == trail)
-    return GNUNET_SYSERR;
-  trail->prev_hop = *prev_hop;
-  return GNUNET_OK;
-}
-
-
-/**
- * Update the next hop of the trail. Call made by trail compression where
- * if you are source of the trail and now you have a new first friend, then
- * you should update the trail.
- * @param trail_id
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case no matching entry found in routing table.
- */
-int
-GDS_ROUTING_update_trail_next_hop (const struct GNUNET_HashCode *trail_id,
-                                   const struct GNUNET_PeerIdentity *next_hop)
-{
-  struct RoutingTrail *trail;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (routing_table,
-                                             trail_id);
-  if (NULL == trail)
-    return GNUNET_SYSERR;
-  trail->next_hop = *next_hop;
-  return GNUNET_OK;
-}
-
-
-/**
- * Get the next hop for trail corresponding to trail_id
- *
- * @param trail_id Trail id to be searched.
- * @return Next_hop if found
- *         NULL If next hop not found.
- */
-const struct GNUNET_PeerIdentity *
-GDS_ROUTING_get_next_hop (const struct GNUNET_HashCode *trail_id,
-                          enum GDS_ROUTING_trail_direction trail_direction)
-{
-  struct RoutingTrail *trail;
-
-  trail = GNUNET_CONTAINER_multihashmap_get (routing_table,
-                                             trail_id);
-  if (NULL == trail)
-  {
-    /* If a friend got disconnected and we removed all the entry from the
-     routing table, then trail will be deleted and my identity will not know
-     and when it tries to reach to that finger it fails. thats why
-     assertion always fails in*/
-    return NULL;
-  }
-  switch (trail_direction)
-  {
-    case GDS_ROUTING_SRC_TO_DEST:
-      return &trail->next_hop;
-    case GDS_ROUTING_DEST_TO_SRC:
-      return &trail->prev_hop;
-  }
-  return NULL;
-}
-
-
-/**
- * Remove trail with trail_id
- * @param trail_id Trail id to be removed
- * @return #GNUNET_YES success
- *         #GNUNET_NO if entry not found.
- */
-int
-GDS_ROUTING_remove_trail (const struct GNUNET_HashCode *remove_trail_id)
-{
-  struct RoutingTrail *remove_entry;
-
-  remove_entry = GNUNET_CONTAINER_multihashmap_get (routing_table,
-                                                    remove_trail_id);
-  if (NULL == remove_entry)
-    return GNUNET_NO;
-
-  if (GNUNET_YES ==
-      GNUNET_CONTAINER_multihashmap_remove (routing_table,
-                                            remove_trail_id,
-                                            remove_entry))
-  {
-    GNUNET_free (remove_entry);
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * Iterate over routing table and remove entries with value as part of any trail.
- *
- * @param cls closure
- * @param key current public key
- * @param value value in the hash map
- * @return #GNUNET_YES if we should continue to iterate,
- *         #GNUNET_NO if not.
- */
-static int
-remove_matching_trails (void *cls,
-                        const struct GNUNET_HashCode *key,
-                        void *value)
-{
-  struct RoutingTrail *remove_trail = value;
-  struct GNUNET_PeerIdentity *disconnected_peer = cls;
-  struct GNUNET_HashCode trail_id = *key;
-  struct GNUNET_PeerIdentity my_identity;
-
-  /* If disconnected_peer is next_hop, then send a trail teardown message through
-   * prev_hop in direction from destination to source. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&remove_trail->next_hop,
-                                            disconnected_peer))
-  {
-    my_identity = *GDS_NEIGHBOURS_get_id ();
-    if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                              &remove_trail->prev_hop))
-    {
-      GDS_NEIGHBOURS_send_trail_teardown (&trail_id,
-                                          GDS_ROUTING_DEST_TO_SRC,
-                                          &remove_trail->prev_hop);
-    }
-  }
-
-  /* If disconnected_peer is prev_hop, then send a trail teardown through
-   * next_hop in direction from Source to Destination. */
-  if (0 == GNUNET_CRYPTO_cmp_peer_identity (&remove_trail->prev_hop,
-                                            disconnected_peer))
-  {
-    my_identity = *GDS_NEIGHBOURS_get_id ();
-
-    if (0 != GNUNET_CRYPTO_cmp_peer_identity (&my_identity,
-                                              &remove_trail->next_hop))
-    {
-      GDS_NEIGHBOURS_send_trail_teardown (&trail_id,
-                                          GDS_ROUTING_SRC_TO_DEST,
-                                          &remove_trail->next_hop);
-    }
-  }
-
-  GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (routing_table,
-                                                         &trail_id,
-                                                         remove_trail));
-  GNUNET_free (remove_trail);
-  return GNUNET_YES;
-}
-
-#if 0
-/**
- * TEST FUNCTION
- * Remove after using.
- */
-void
-GDS_ROUTING_test_print (void)
-{
-  struct GNUNET_CONTAINER_MultiHashMapIterator *iter;
-  struct RoutingTrail *trail;
-  struct GNUNET_PeerIdentity print_peer;
-  struct GNUNET_HashCode key_ret;
-  int i;
-
-  struct GNUNET_PeerIdentity my_identity = *GDS_NEIGHBOURS_get_id();
-  print_peer = my_identity;
-   FPRINTF (stderr,_("\nSUPU ***PRINTING ROUTING TABLE ***** of =%s"),GNUNET_i2s(&print_peer));
-  iter =GNUNET_CONTAINER_multihashmap_iterator_create (routing_table);
-  for (i = 0; i < GNUNET_CONTAINER_multihashmap_size(routing_table); i++)
-  {
-    if(GNUNET_YES == GNUNET_CONTAINER_multihashmap_iterator_next (iter,
-                                                                  &key_ret,
-                                                                  (const void **)&trail))
-    {
-      FPRINTF (stderr,_("\nSUPU %s, %s, %d, trail->trail_id = %s"),
-              __FILE__, __func__,__LINE__, GNUNET_h2s(&trail->trail_id));
-      GNUNET_memcpy (&print_peer, &trail->next_hop, sizeof (struct GNUNET_PeerIdentity));
-      FPRINTF (stderr,_("\nSUPU %s, %s, %d, trail->next_hop = %s"),
-              __FILE__, __func__,__LINE__, GNUNET_i2s(&print_peer));
-      GNUNET_memcpy (&print_peer, &trail->prev_hop, sizeof (struct GNUNET_PeerIdentity));
-      FPRINTF (stderr,_("\nSUPU %s, %s, %d, trail->prev_hop = %s"),
-              __FILE__, __func__,__LINE__, GNUNET_i2s(&print_peer));
-    }
-  }
-}
-#endif
-
-/**
- * Remove every trail where peer is either next_hop or prev_hop. Also send a
- * trail teardown message in direction of hop which is not disconnected.
- * @param peer Peer identity. Trail containing this peer should be removed.
- */
-int
-GDS_ROUTING_remove_trail_by_peer (const struct GNUNET_PeerIdentity *peer)
-{
-  int ret;
-
-
-  /* No entries in my routing table. */
-  if (0 == GNUNET_CONTAINER_multihashmap_size(routing_table))
-    return GNUNET_YES;
-
-  ret = GNUNET_CONTAINER_multihashmap_iterate (routing_table,
-                                               &remove_matching_trails,
-                                               (void *)peer);
-  return ret;
-}
-
-
-/**
- * Add a new entry in routing table
- * @param new_trail_id
- * @param prev_hop
- * @param next_hop
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case new_trail_id already exists in the network
- *                         but with different prev_hop/next_hop
- */
-int
-GDS_ROUTING_add (const struct GNUNET_HashCode *new_trail_id,
-                 const struct GNUNET_PeerIdentity *prev_hop,
-                 const struct GNUNET_PeerIdentity *next_hop)
-{
-  struct RoutingTrail *new_entry;
-
-  new_entry = GNUNET_new (struct RoutingTrail);
-  new_entry->trail_id = *new_trail_id;
-  new_entry->next_hop = *next_hop;
-  new_entry->prev_hop = *prev_hop;
-
-  // FIXME: this leaks memory if the put fails!
-  return GNUNET_CONTAINER_multihashmap_put (routing_table,
-                                            &new_entry->trail_id,
-                                            new_entry,
-                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
-}
-
-
-/**
- * Check if the size of routing table has crossed ROUTING_TABLE_THRESHOLD.
- * It means that I don't have any more space in my routing table and I can not
- * be part of any more trails till there is free space in my routing table.
- * @return #GNUNET_YES, if threshold crossed else #GNUNET_NO.
- */
-int
-GDS_ROUTING_threshold_reached (void)
-{
-  return (GNUNET_CONTAINER_multihashmap_size(routing_table) >
-          ROUTING_TABLE_THRESHOLD) ? GNUNET_YES:GNUNET_NO;
-}
-
-
-/**
- * Initialize routing subsystem.
- */
-void
-GDS_ROUTING_init (void)
-{
-  routing_table = GNUNET_CONTAINER_multihashmap_create (ROUTING_TABLE_THRESHOLD * 4 / 3,
-                                                        GNUNET_NO);
-}
-
-
-/**
- * Shutdown routing subsystem.
- */
-void
-GDS_ROUTING_done (void)
-{
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (routing_table));
-  GNUNET_CONTAINER_multihashmap_destroy (routing_table);
-}
-
-/* end of gnunet-service-xdht_routing.c */
diff --git a/src/dht/gnunet-service-xdht_routing.h b/src/dht/gnunet-service-xdht_routing.h
deleted file mode 100644
index 69ab1ff78..000000000
--- a/src/dht/gnunet-service-xdht_routing.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 - 2014 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file dht/gnunet-service-xdht_routing.h
- * @brief GNUnet DHT tracking of requests for routing replies
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_XDHT_ROUTING_H
-#define GNUNET_SERVICE_XDHT_ROUTING_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet_block_lib.h"
-#include "gnunet_dht_service.h"
-
-/**
- * To understand the direction in which trial should be read. 
- */
-enum GDS_ROUTING_trail_direction 
-{
-  GDS_ROUTING_SRC_TO_DEST,
-  GDS_ROUTING_DEST_TO_SRC
-};
-
-
-/**
- * Update the prev. hop of the trail. Call made by trail teardown where
- * if you are the first friend now in the trail then you need to update
- * your prev. hop.
- * @param trail_id
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case no matching entry found in routing table. 
- */
-int
-GDS_ROUTING_update_trail_prev_hop (const struct GNUNET_HashCode *trail_id,
-                                   const struct GNUNET_PeerIdentity *prev_hop);
-
-
-/**
- * Update the next hop of the trail. Call made by trail compression where
- * if you are source of the trail and now you have a new first friend, then
- * you should update the trail. 
- * @param trail_id
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case no matching entry found in routing table.
- */
-int
-GDS_ROUTING_update_trail_next_hop (const struct GNUNET_HashCode *trail_id,
-                                   const struct GNUNET_PeerIdentity *next_hop);
-
-/**
- * Get the next hop for trail corresponding to trail_id
- * @param trail_id Trail id to be searched. 
- * @return Next_hop if found
- *         NULL If next hop not found. 
- */
-const struct GNUNET_PeerIdentity *
-GDS_ROUTING_get_next_hop (const struct GNUNET_HashCode *trail_id,
-                          enum GDS_ROUTING_trail_direction trail_direction);
-
-
-/**
- * Remove every trail where peer is either next_hop or prev_hop 
- * @param peer Peer to be searched.
- */
-int
-GDS_ROUTING_remove_trail_by_peer (const struct GNUNET_PeerIdentity *peer);
-
-
-/**
- * Remove trail with trail_id
- *
- * @param trail_id Trail id to be removed
- * @return #GNUNET_YES success 
- *         #GNUNET_NO if entry not found.
- */
-int
-GDS_ROUTING_remove_trail (const struct GNUNET_HashCode *remove_trail_id);
-
-
-/**
- * Add a new entry in routing table
- * @param new_trail_id
- * @param prev_hop
- * @param next_hop
- * @return #GNUNET_OK success
- *         #GNUNET_SYSERR in case new_trail_id already exists in the network
- *                         but with different prev_hop/next_hop
- */
-int
-GDS_ROUTING_add (const struct GNUNET_HashCode *new_trail_id, 
-                 const struct GNUNET_PeerIdentity *prev_hop,
-                 const struct GNUNET_PeerIdentity *next_hop);
-
-
-/**
- * Check if the size of routing table has crossed threshold. 
- * @return #GNUNET_YES, if threshold crossed 
- *         #GNUNET_NO, if size is within threshold 
- */
-int
-GDS_ROUTING_threshold_reached (void);
-
-#if 0
-/**
- * Test function. Remove afterwards. 
- */
-void 
-GDS_ROUTING_test_print (void);
-#endif
-
-/**
- * Initialize routing subsystem.
- */
-void
-GDS_ROUTING_init (void);
-
-/**
- * Shutdown routing subsystem.
- */
-void
-GDS_ROUTING_done (void);
-
-#endif
diff --git a/src/dht/gnunet_dht_profiler.c b/src/dht/gnunet_dht_profiler.c
index 460eaa572..a8807bea8 100644
--- a/src/dht/gnunet_dht_profiler.c
+++ b/src/dht/gnunet_dht_profiler.c
@@ -1181,31 +1181,54 @@ main (int argc, char *const *argv)
 {
   int rc;
 
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "peers", "COUNT",
-     gettext_noop ("number of peers to start"),
-     1, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'s', "searches", "COUNT",
-     gettext_noop ("maximum number of times we try to search for successor circle formation (0 for R5N)"),
-     1, &GNUNET_GETOPT_set_uint, &max_searches},
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     1, &GNUNET_GETOPT_set_string, &hosts_file},
-    {'D', "delay", "DELAY",
-     gettext_noop ("delay between rounds for collecting statistics (default: 30 sec)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_stats},
-    {'P', "PUT-delay", "DELAY",
-     gettext_noop ("delay to start doing PUTs (default: 1 sec)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_put},
-    {'G', "GET-delay", "DELAY",
-     gettext_noop ("delay to start doing GETs (default: 5 min)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_get},
-    {'r', "replication", "DEGREE",
-     gettext_noop ("replication degree for DHT PUTs"),
-     1, &GNUNET_GETOPT_set_uint, &replication},
-    {'t', "timeout", "TIMEOUT",
-     gettext_noop ("timeout for DHT PUT and GET requests (default: 1 min)"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_uint ('n',
+                                   "peers",
+                                   "COUNT",
+                                   gettext_noop ("number of peers to start"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_option_uint ('s',
+                                   "searches",
+                                   "COUNT",
+                                   gettext_noop ("maximum number of times we try to search for successor circle formation (0 for R5N)"),
+                                   &max_searches),
+
+    GNUNET_GETOPT_option_string ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
+
+    GNUNET_GETOPT_option_relative_time ('D',
+                                            "delay",
+                                            "DELAY",
+                                            gettext_noop ("delay between rounds for collecting statistics (default: 30 sec)"),
+                                            &delay_stats),
+
+    GNUNET_GETOPT_option_relative_time ('P',
+                                            "PUT-delay",
+                                            "DELAY",
+                                            gettext_noop ("delay to start doing PUTs (default: 1 sec)"),
+                                            &delay_put),
+
+    GNUNET_GETOPT_option_relative_time ('G',
+                                            "GET-delay",
+                                            "DELAY",
+                                            gettext_noop ("delay to start doing GETs (default: 5 min)"),
+                                            &delay_get),
+    GNUNET_GETOPT_option_uint ('r',
+                                   "replication",
+                                   "DEGREE",
+                                   gettext_noop ("replication degree for DHT PUTs"),
+                                   &replication),
+
+
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("timeout for DHT PUT and GET requests (default: 1 min)"),
+                                            &timeout),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dht/plugin_block_dht.c b/src/dht/plugin_block_dht.c
index 8bb533961..24f8b21b2 100644
--- a/src/dht/plugin_block_dht.c
+++ b/src/dht/plugin_block_dht.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2010 GNUnet e.V.
+     Copyright (C) 2010, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -25,25 +25,78 @@
  *        DHT (see fs block plugin)
  * @author Christian Grothoff
  */
-
 #include "platform.h"
 #include "gnunet_constants.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
 
 #define DEBUG_DHT GNUNET_EXTRA_LOGGING
 
+/**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_dht_create_group (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               uint32_t nonce,
+                               const void *raw_data,
+                               size_t raw_data_size,
+                               va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = 8;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
 
 /**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the @a reply_block.
  *
  * @param cls closure
+ * @param ctx context
  * @param type block type
+ * @param group block group to check against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -52,17 +105,16 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_dht_evaluate (void *cls,
+                           struct GNUNET_BLOCK_Context *ctx,
                            enum GNUNET_BLOCK_Type type,
+                           struct GNUNET_BLOCK_Group *group,
                            enum GNUNET_BLOCK_EvaluationOptions eo,
                            const struct GNUNET_HashCode *query,
-                           struct GNUNET_CONTAINER_BloomFilter **bf,
-                           int32_t bf_mutator,
                            const void *xquery,
                            size_t xquery_size,
                            const void *reply_block,
                            size_t reply_block_size)
 {
-  struct GNUNET_HashCode mhash;
   const struct GNUNET_HELLO_Message *hello;
   struct GNUNET_PeerIdentity pid;
   const struct GNUNET_MessageHeader *msg;
@@ -75,8 +127,7 @@ block_plugin_dht_evaluate (void *cls,
     GNUNET_break_op (0);
     return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID;
   }
-  if ( (NULL == reply_block) ||
-       (0 == reply_block_size) )
+  if (NULL == reply_block)
     return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
   if (reply_block_size < sizeof (struct GNUNET_MessageHeader))
   {
@@ -95,22 +146,13 @@ block_plugin_dht_evaluate (void *cls,
     GNUNET_break_op (0);
     return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
   }
-  if (NULL != bf)
-  {
-    GNUNET_CRYPTO_hash (&pid, sizeof (pid), &phash);
-    GNUNET_BLOCK_mingle_hash (&phash, bf_mutator, &mhash);
-    if (NULL != *bf)
-    {
-      if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-        return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-    }
-    else
-    {
-      *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8,
-                                               GNUNET_CONSTANTS_BLOOMFILTER_K);
-    }
-    GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-  }
+  GNUNET_CRYPTO_hash (&pid,
+                      sizeof (pid),
+                      &phash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (group,
+                                          &phash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_OK_MORE;
 }
 
@@ -183,6 +225,7 @@ libgnunet_plugin_block_dht_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_dht_evaluate;
   api->get_key = &block_plugin_dht_get_key;
+  api->create_group = &block_plugin_dht_create_group;
   api->types = types;
   return api;
 }
@@ -194,7 +237,7 @@ libgnunet_plugin_block_dht_init (void *cls)
 void *
 libgnunet_plugin_block_dht_done (void *cls)
 {
-  struct GNUNET_TRANSPORT_PluginFunctions *api = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
 
   GNUNET_free (api);
   return NULL;
diff --git a/src/dht/test_dht_api.c b/src/dht/test_dht_api.c
index 99f17699c..8f4e0ed31 100644
--- a/src/dht/test_dht_api.c
+++ b/src/dht/test_dht_api.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009, 2015 GNUnet e.V.
+     Copyright (C) 2009, 2015, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,6 +20,7 @@
 /**
  * @file dht/test_dht_api.c
  * @brief base test case for dht api
+ * @author Christian Grothoff
  *
  * This test case tests DHT api to DUMMY DHT service communication.
  */
@@ -33,116 +34,70 @@
 /**
  * How long until we really give up on a particular testcase portion?
  */
-#define TOTAL_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 600)
-
-/**
- * How long until we give up on any particular operation (and retry)?
- */
-#define BASE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)
-
-#define MTYPE 12345
-
-
-struct RetryContext
-{
-  /**
-   * When to really abort the operation.
-   */
-  struct GNUNET_TIME_Absolute real_timeout;
-
-  /**
-   * What timeout to set for the current attempt (increases)
-   */
-  struct GNUNET_TIME_Relative next_timeout;
-
-  /**
-   * The task identifier of the retry task, so it can be cancelled.
-   */
-  struct GNUNET_SCHEDULER_Task * retry_task;
-
-};
-
+#define TOTAL_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
 
 static struct GNUNET_DHT_Handle *dht_handle;
 
 static struct GNUNET_DHT_GetHandle *get_handle;
 
-struct RetryContext retry_context;
+static struct GNUNET_DHT_PutHandle *put_handle;
 
 static int ok = 1;
 
-static struct GNUNET_SCHEDULER_Task * die_task;
-
-
-#if VERBOSE
-#define OKPP do { ok++; FPRINTF (stderr, "Now at stage %u at %s:%u\n", ok, __FILE__, __LINE__); } while (0)
-#else
-#define OKPP do { ok++; } while (0)
-#endif
+static struct GNUNET_SCHEDULER_Task *die_task;
 
 
 static void
-end (void *cls)
+do_shutdown (void *cls)
 {
-  GNUNET_SCHEDULER_cancel (die_task);
-  die_task = NULL;
-  GNUNET_DHT_disconnect (dht_handle);
-  dht_handle = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "DHT disconnected, returning success!\n");
-  ok = 0;
-}
-
-
-static void
-end_badly ()
-{
-  /* do work here */
-  FPRINTF (stderr, "%s",  "Ending on an unhappy note.\n");
-  if (get_handle != NULL)
+  if (NULL != die_task)
+  {
+    GNUNET_SCHEDULER_cancel (die_task);
+    die_task = NULL;
+  }
+  if (NULL != put_handle)
+  {
+    GNUNET_DHT_put_cancel (put_handle);
+    put_handle = NULL;
+  }
+  if (NULL != get_handle)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Stopping get request!\n");
     GNUNET_DHT_get_stop (get_handle);
+    get_handle = NULL;
   }
-  if (retry_context.retry_task != NULL)
-    GNUNET_SCHEDULER_cancel (retry_context.retry_task);
   GNUNET_DHT_disconnect (dht_handle);
   dht_handle = NULL;
-  ok = 1;
 }
 
 
-/**
- * Signature of the main function of a task.
- *
- * @param cls closure
- */
 static void
-test_get_stop (void *cls)
+end_badly (void *cls)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Called test_get_stop!\n");
-  GNUNET_assert (NULL != dht_handle);
-  GNUNET_DHT_get_stop (get_handle);
-  get_handle = NULL;
-  GNUNET_SCHEDULER_add_now (&end, NULL);
+  die_task = NULL;
+  FPRINTF (stderr,
+           "%s",
+           "Ending on an unhappy note.\n");
+  GNUNET_SCHEDULER_shutdown ();
+  ok = 1;
 }
 
 
 static void
-test_get_iterator (void *cls, struct GNUNET_TIME_Absolute exp,
-                   const struct GNUNET_HashCode * key,
+test_get_iterator (void *cls,
+                   struct GNUNET_TIME_Absolute exp,
+                   const struct GNUNET_HashCode *key,
                    const struct GNUNET_PeerIdentity *get_path,
                    unsigned int get_path_length,
                    const struct GNUNET_PeerIdentity *put_path,
                    unsigned int put_path_length,
                    enum GNUNET_BLOCK_Type type,
-                   size_t size, const void *data)
+                   size_t size,
+                   const void *data)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "test_get_iterator called (we got a result), stopping get request!\n");
-  GNUNET_SCHEDULER_add_now (&test_get_stop,
-                            NULL);
+  GNUNET_SCHEDULER_shutdown ();
+  ok = 0;
 }
 
 
@@ -153,31 +108,33 @@ test_get_iterator (void *cls, struct GNUNET_TIME_Absolute exp,
  * @param success result of PUT
  */
 static void
-test_get (void *cls, int success)
+test_get (void *cls,
+          int success)
 {
   struct GNUNET_HashCode hash;
 
+  put_handle = NULL;
   memset (&hash,
           42,
           sizeof (struct GNUNET_HashCode));
-
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Called test_get!\n");
   GNUNET_assert (dht_handle != NULL);
-  retry_context.real_timeout = GNUNET_TIME_relative_to_absolute (TOTAL_TIMEOUT);
-  retry_context.next_timeout = BASE_TIMEOUT;
-
-  get_handle =
-      GNUNET_DHT_get_start (dht_handle,
-                            GNUNET_BLOCK_TYPE_TEST, &hash, 1,
-                            GNUNET_DHT_RO_NONE, NULL, 0, &test_get_iterator,
-                            NULL);
-
-  if (get_handle == NULL)
+  get_handle = GNUNET_DHT_get_start (dht_handle,
+                                     GNUNET_BLOCK_TYPE_TEST,
+                                     &hash,
+                                     1,
+                                     GNUNET_DHT_RO_NONE,
+                                     NULL,
+                                     0,
+                                     &test_get_iterator,
+                                     NULL);
+
+  if (NULL == get_handle)
   {
     GNUNET_break (0);
-    GNUNET_SCHEDULER_cancel (die_task);
-    die_task = GNUNET_SCHEDULER_add_now (&end_badly, NULL);
+    ok = 1;
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
 }
@@ -193,29 +150,38 @@ run (void *cls,
   size_t data_size = 42;
 
   GNUNET_assert (ok == 1);
-  OKPP;
-  die_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_MINUTES, 1), &end_badly,
-                                    NULL);
-
-
-  memset (&hash, 42, sizeof (struct GNUNET_HashCode));
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
+  die_task = GNUNET_SCHEDULER_add_delayed (TOTAL_TIMEOUT,
+                                           &end_badly,
+                                           NULL);
+  memset (&hash,
+          42,
+          sizeof (struct GNUNET_HashCode));
   data = GNUNET_malloc (data_size);
   memset (data, 43, data_size);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Called test_put!\n");
-  dht_handle = GNUNET_DHT_connect (cfg, 100);
-  GNUNET_assert (dht_handle != NULL);
-  GNUNET_DHT_put (dht_handle, &hash, 1, GNUNET_DHT_RO_NONE,
-                  GNUNET_BLOCK_TYPE_TEST, data_size, data,
-                  GNUNET_TIME_relative_to_absolute (TOTAL_TIMEOUT),
-                  &test_get, NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Called test_put!\n");
+  dht_handle = GNUNET_DHT_connect (cfg,
+                                   100);
+  GNUNET_assert (NULL != dht_handle);
+  put_handle = GNUNET_DHT_put (dht_handle,
+                               &hash,
+                               1,
+                               GNUNET_DHT_RO_NONE,
+                               GNUNET_BLOCK_TYPE_TEST,
+                               data_size,
+                               data,
+                               GNUNET_TIME_relative_to_absolute (TOTAL_TIMEOUT),
+                               &test_get,
+                               NULL);
   GNUNET_free (data);
 }
 
 
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   if (0 != GNUNET_TESTING_peer_run ("test-dht-api",
                                     "test_dht_api_data.conf",
diff --git a/src/dht/test_dht_monitor.c b/src/dht/test_dht_monitor.c
index 5e6fc074a..3de800144 100644
--- a/src/dht/test_dht_monitor.c
+++ b/src/dht/test_dht_monitor.c
@@ -90,7 +90,7 @@ static unsigned int NUM_PEERS = 3;
 /**
  * Task called to disconnect peers.
  */
-static struct GNUNET_SCHEDULER_Task * timeout_task;
+static struct GNUNET_SCHEDULER_Task *timeout_task;
 
 /**
  * Task to do DHT_puts
@@ -107,7 +107,7 @@ static unsigned int monitor_counter;
  * Terminates active get operations and shuts down
  * the testbed.
  *
- * @param cls the 'struct GNUNET_DHT_TestContext'
+ * @param cls the `struct GNUNET_DHT_TEST_Context`
  */
 static void
 shutdown_task (void *cls)
@@ -133,6 +133,26 @@ shutdown_task (void *cls)
   GNUNET_free (monitors);
   GNUNET_SCHEDULER_cancel (put_task);
   GNUNET_DHT_TEST_cleanup (ctx);
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+/**
+ * Task run on success or timeout to clean up.
+ * Terminates active get operations and shuts down
+ * the testbed.
+ *
+ * @param cls NULL
+ */
+static void
+timeout_task_cb (void *cls)
+{
+  timeout_task = NULL;
+  GNUNET_SCHEDULER_shutdown ();
 }
 
 
@@ -157,12 +177,12 @@ dht_get_handler (void *cls, struct GNUNET_TIME_Absolute exp,
                  const struct GNUNET_PeerIdentity *get_path,
                  unsigned int get_path_length,
                  const struct GNUNET_PeerIdentity *put_path,
-                 unsigned int put_path_length, enum GNUNET_BLOCK_Type type,
+                 unsigned int put_path_length,
+                 enum GNUNET_BLOCK_Type type,
                  size_t size, const void *data)
 {
   struct GetOperation *get_op = cls;
   struct GNUNET_HashCode want;
-  struct GNUNET_DHT_TestContext *ctx;
 
   if (sizeof (struct GNUNET_HashCode) != size)
   {
@@ -186,8 +206,7 @@ dht_get_handler (void *cls, struct GNUNET_TIME_Absolute exp,
     return;
   /* all DHT GET operations successful; terminate! */
   ok = 0;
-  ctx = GNUNET_SCHEDULER_cancel (timeout_task);
-  timeout_task = GNUNET_SCHEDULER_add_now (&shutdown_task, ctx);
+  GNUNET_SCHEDULER_shutdown ();
 }
 
 
@@ -202,11 +221,10 @@ do_puts (void *cls)
   struct GNUNET_DHT_Handle **hs = cls;
   struct GNUNET_HashCode key;
   struct GNUNET_HashCode value;
-  unsigned int i;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Putting values into DHT\n");
-  for (i = 0; i < NUM_PEERS; i++)
+  for (unsigned int i = 0; i < NUM_PEERS; i++)
   {
     GNUNET_CRYPTO_hash (&i, sizeof (i), &key);
     GNUNET_CRYPTO_hash (&key, sizeof (key), &value);
@@ -360,7 +378,8 @@ run (void *cls,
 
   GNUNET_assert (NUM_PEERS == num_peers);
   my_peers = peers;
-  monitors = GNUNET_malloc (num_peers * sizeof (struct GNUNET_DHT_MonitorHandle *));
+  monitors = GNUNET_new_array (num_peers,
+                               struct GNUNET_DHT_MonitorHandle *);
   for (i = 0; i < num_peers; i++)
     monitors[i] = GNUNET_DHT_monitor_start (dhts[i],
                                             GNUNET_BLOCK_TYPE_ANY,
@@ -392,7 +411,10 @@ run (void *cls,
     }
   }
   timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
-                                               &shutdown_task, ctx);
+                                               &timeout_task_cb,
+                                               NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 ctx);
 }
 
 
diff --git a/src/dht/test_dht_tools.py.in b/src/dht/test_dht_tools.py.in
index 0f48d7f76..5ceabbfad 100644
--- a/src/dht/test_dht_tools.py.in
+++ b/src/dht/test_dht_tools.py.in
@@ -1,4 +1,16 @@
 #!@PYTHON@
+#
+# This testcase simply checks that the DHT command-line tools work.
+# It launches a single peer, stores a value "testdata" under "testkey",
+# and then gives the system 50 ms to fetch it.
+#
+# This could fail if
+# - command line tool interfaces fail
+# - DHT plugins for storage are not installed / working
+# - block plugins for verification (the test plugin) is not installed
+#
+# The code does NOT depend on DHT routing or any actual P2P functionality.
+#
 from __future__ import print_function
 import os
 import sys
@@ -8,6 +20,8 @@ import subprocess
 import time
 import tempfile
 
+os.environ["PATH"] = "@bindir@" + ":" + os.environ["PATH"];
+
 if os.name == "nt":
   tmp = os.getenv ("TEMP")
 else:
@@ -103,7 +117,7 @@ print ("PASS")
 time.sleep (1)
 
 print ("TEST: Testing get...", end='')
-rc, stdo, stde = r_get (['-k', 'testkey', '-T', '5 ms', '-t', '8'], want_stdo = True, failer = end_arm_failer)
+rc, stdo, stde = r_get (['-k', 'testkey', '-T', '50 ms', '-t', '8'], want_stdo = True, failer = end_arm_failer)
 stdo = stdo.replace ('\r', '').splitlines ()
 expect = "Result 0, type 8:\ntestdata".splitlines()
 if len (stdo) != 2 or len (expect) != 2 or stdo[0] != expect[0] or stdo[1] != expect[1]:
diff --git a/src/dns/Makefile.am b/src/dns/Makefile.am
index f5fcf3782..5af228121 100644
--- a/src/dns/Makefile.am
+++ b/src/dns/Makefile.am
@@ -106,13 +106,14 @@ libgnunetdns_la_LDFLAGS = \
 libgnunet_plugin_block_dns_la_SOURCES = \
   plugin_block_dns.c
 libgnunet_plugin_block_dns_la_LIBADD = \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
   $(top_builddir)/src/util/libgnunetutil.la
 libgnunet_plugin_block_dns_la_LDFLAGS = \
   $(top_builddir)/src/block/$(GN_PLUGIN_LDFLAGS)
 
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)  $(check_SCRIPTS)
 endif
 
diff --git a/src/dns/dns_api.c b/src/dns/dns_api.c
index e7450a1d4..9f0dee9a9 100644
--- a/src/dns/dns_api.c
+++ b/src/dns/dns_api.c
@@ -317,7 +317,7 @@ GNUNET_DNS_request_answer (struct GNUNET_DNS_RequestHandle *rh,
       return;
   }
   if (reply_length + sizeof (struct GNUNET_DNS_Response)
-      >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     GNUNET_free (rh);
diff --git a/src/dns/gnunet-dns-monitor.c b/src/dns/gnunet-dns-monitor.c
index 5e6f90555..fb5c768ac 100644
--- a/src/dns/gnunet-dns-monitor.c
+++ b/src/dns/gnunet-dns-monitor.c
@@ -52,7 +52,7 @@ static int ret;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -346,14 +346,19 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "inbound-only", NULL,
-     gettext_noop ("only monitor DNS queries"),
-     0, &GNUNET_GETOPT_set_one, &inbound_only},
-    {'o', "outbound-only", NULL,
-     gettext_noop ("only monitor DNS replies"),
-     0, &GNUNET_GETOPT_set_one, &outbound_only},
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_flag ('i',
+                                  "inbound-only",
+                                  gettext_noop ("only monitor DNS queries"),
+                                  &inbound_only),
+
+    GNUNET_GETOPT_option_flag ('o',
+                                  "outbound-only",
+                                  gettext_noop ("only monitor DNS queries"),
+                                  &outbound_only),
+
+    GNUNET_GETOPT_option_verbose (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dns/gnunet-dns-redirector.c b/src/dns/gnunet-dns-redirector.c
index 89929815a..44d3d0b6c 100644
--- a/src/dns/gnunet-dns-redirector.c
+++ b/src/dns/gnunet-dns-redirector.c
@@ -52,7 +52,7 @@ static int ret;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -230,14 +230,20 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'4', "ipv4", "IPV4",
-     gettext_noop ("set A records"),
-     1, &GNUNET_GETOPT_set_string, &n4},
-    {'6', "ipv4", "IPV6",
-     gettext_noop ("set AAAA records"),
-     1, &GNUNET_GETOPT_set_string, &n6},
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_string ('4',
+                                 "ipv4",
+                                 "IPV4",
+                                 gettext_noop ("set A records"),
+                                 &n4),
+
+    GNUNET_GETOPT_option_string ('6',
+                                 "ipv4",
+                                 "IPV6",
+                                 gettext_noop ("set AAAA records"),
+                                 &n6),
+
+    GNUNET_GETOPT_option_verbose (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dns/gnunet-helper-dns.c b/src/dns/gnunet-helper-dns.c
index 1c5744002..5cf1967f5 100644
--- a/src/dns/gnunet-helper-dns.c
+++ b/src/dns/gnunet-helper-dns.c
@@ -79,7 +79,7 @@
 #include "gnunet_protocols.h"
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
@@ -794,6 +794,8 @@ main (int argc, char *const*argv)
       sbin_ip = "/sbin/ip";
     else if (0 == access ("/usr/sbin/ip", X_OK))
       sbin_ip = "/usr/sbin/ip";
+    else if (0 == access ("/bin/ip", X_OK)) /* gentoo has it there */
+      sbin_ip = "/bin/ip";
     else
     {
       fprintf (stderr,
diff --git a/src/dns/gnunet-service-dns.c b/src/dns/gnunet-service-dns.c
index 74f595c5e..ffc94afb7 100644
--- a/src/dns/gnunet-service-dns.c
+++ b/src/dns/gnunet-service-dns.c
@@ -347,7 +347,7 @@ request_done (struct RequestRecord *rr)
   }
   reply_len += sizeof (struct GNUNET_TUN_UdpHeader);
   reply_len += rr->payload_length;
-  if (reply_len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (reply_len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     /* response too big, drop */
     GNUNET_break (0); /* how can this be? */
@@ -481,7 +481,7 @@ send_request_to_client (struct RequestRecord *rr,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_DNS_Request *req;
 
-  if (sizeof (struct GNUNET_DNS_Request) + rr->payload_length >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (sizeof (struct GNUNET_DNS_Request) + rr->payload_length >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     cleanup_rr (rr);
@@ -882,11 +882,10 @@ handle_client_response (void *cls,
  * message is received by the tokenizer from the DNS hijack process.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message, a DNS request we should handle
  */
 static int
-process_helper_messages (void *cls GNUNET_UNUSED, void *client,
+process_helper_messages (void *cls,
                          const struct GNUNET_MessageHeader *message)
 {
   uint16_t msize;
diff --git a/src/dns/plugin_block_dns.c b/src/dns/plugin_block_dns.c
index dc339dd25..ca6ea84c4 100644
--- a/src/dns/plugin_block_dns.c
+++ b/src/dns/plugin_block_dns.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -31,6 +31,61 @@
 #include "gnunet_block_plugin.h"
 #include "block_dns.h"
 #include "gnunet_signatures.h"
+#include "gnunet_block_group_lib.h"
+
+
+/**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_dns_create_group (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               uint32_t nonce,
+                               const void *raw_data,
+                               size_t raw_data_size,
+                               va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = 8;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
 
 
 /**
@@ -38,11 +93,11 @@
  * request evaluation, simply pass "NULL" for the reply_block.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param bg group to evaluate against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for bf
  * @param xquery extended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -51,17 +106,18 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_dns_evaluate (void *cls,
+                           struct GNUNET_BLOCK_Context *ctx,
                            enum GNUNET_BLOCK_Type type,
+                           struct GNUNET_BLOCK_Group *bg,
                            enum GNUNET_BLOCK_EvaluationOptions eo,
                            const struct GNUNET_HashCode * query,
-                           struct GNUNET_CONTAINER_BloomFilter **bf,
-                           int32_t bf_mutator,
                            const void *xquery,
                            size_t xquery_size,
                            const void *reply_block,
                            size_t reply_block_size)
 {
   const struct GNUNET_DNS_Advertisement *ad;
+  struct GNUNET_HashCode phash;
 
   switch (type)
   {
@@ -69,7 +125,7 @@ block_plugin_dns_evaluate (void *cls,
     if (0 != xquery_size)
       return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID;
 
-    if (0 == reply_block_size)
+    if (NULL == reply_block)
       return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
 
     if (sizeof (struct GNUNET_DNS_Advertisement) != reply_block_size)
@@ -96,13 +152,20 @@ block_plugin_dns_evaluate (void *cls,
     }
     if (GNUNET_OK !=
         GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_DNS_RECORD,
-                                  &ad->purpose,
-                                  &ad->signature,
-                                  &ad->peer.public_key))
+                                    &ad->purpose,
+                                    &ad->signature,
+                                    &ad->peer.public_key))
     {
       GNUNET_break_op (0);
       return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
     }
+    GNUNET_CRYPTO_hash (reply_block,
+                        reply_block_size,
+                        &phash);
+    if (GNUNET_YES ==
+        GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                            &phash))
+      return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
     return GNUNET_BLOCK_EVALUATION_OK_MORE;
   default:
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -149,6 +212,7 @@ libgnunet_plugin_block_dns_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_dns_evaluate;
   api->get_key = &block_plugin_dns_get_key;
+  api->create_group = &block_plugin_dns_create_group;
   api->types = types;
   return api;
 }
diff --git a/src/dv/Makefile.am b/src/dv/Makefile.am
index c6a620343..c44321151 100644
--- a/src/dv/Makefile.am
+++ b/src/dv/Makefile.am
@@ -73,7 +73,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = $(check_PROGRAMS) 
 endif
 
diff --git a/src/dv/dv_api.c b/src/dv/dv_api.c
index 062f9a95f..ab521ec1f 100644
--- a/src/dv/dv_api.c
+++ b/src/dv/dv_api.c
@@ -437,7 +437,7 @@ GNUNET_DV_send (struct GNUNET_DV_ServiceHandle *sh,
   struct ConnectedPeer *peer;
   struct GNUNET_MQ_Envelope *env;
 
-  if (ntohs (msg->size) + sizeof (*sm) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (ntohs (msg->size) + sizeof (*sm) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/dv/gnunet-dv.c b/src/dv/gnunet-dv.c
index d0917d363..d99097a15 100644
--- a/src/dv/gnunet-dv.c
+++ b/src/dv/gnunet-dv.c
@@ -34,7 +34,7 @@ static struct GNUNET_DV_ServiceHandle *sh;
 /**
  * Was verbose specified?
  */
-static int verbose;
+static unsigned int verbose;
 
 
 /**
@@ -161,10 +161,10 @@ main (int argc, char *const *argv)
 {
   int res;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'V', "verbose", NULL,
-     gettext_noop ("verbose output"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dv/gnunet-service-dv.c b/src/dv/gnunet-service-dv.c
index 6adaa04d9..fa1c16be8 100644
--- a/src/dv/gnunet-service-dv.c
+++ b/src/dv/gnunet-service-dv.c
@@ -399,7 +399,7 @@ send_data_to_plugin (const struct GNUNET_MessageHeader *message,
               (unsigned int) distance);
   size = sizeof (struct GNUNET_DV_ReceivedMessage) +
     ntohs (message->size);
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0); /* too big */
     return;
@@ -537,7 +537,7 @@ forward_payload (struct DirectNeighbor *target,
     return;
   }
   if (sizeof (struct RouteMessage) + ntohs (payload->size)
-      >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1346,11 +1346,13 @@ check_target_added (void *cls,
  *
  * @param cls the `struct DirectNeighbor` we're building the consensus with
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status see `enum GNUNET_SET_Status`
  */
 static void
 handle_set_union_result (void *cls,
                          const struct GNUNET_SET_Element *element,
+                         uint64_t current_size,
                          enum GNUNET_SET_Status status)
 {
   struct DirectNeighbor *neighbor = cls;
@@ -1528,6 +1530,7 @@ listen_set_union (void *cls,
                                         GNUNET_SET_OPERATION_UNION);
   neighbor->set_op = GNUNET_SET_accept (request,
                                         GNUNET_SET_RESULT_ADDED,
+                                        (struct GNUNET_SET_Option[]) {{ 0 }},
                                         &handle_set_union_result,
                                         neighbor);
   neighbor->consensus_insertion_offset = 0;
@@ -1558,6 +1561,7 @@ initiate_set_union (void *cls)
                                          &neighbor->real_session_id,
                                          NULL,
                                          GNUNET_SET_RESULT_ADDED,
+                                         (struct GNUNET_SET_Option[]) {{ 0 }},
                                          &handle_set_union_result,
                                          neighbor);
   neighbor->consensus_insertion_offset = 0;
diff --git a/src/exit/Makefile.am b/src/exit/Makefile.am
index 6c4cbf114..aa1210269 100644
--- a/src/exit/Makefile.am
+++ b/src/exit/Makefile.am
@@ -30,11 +30,11 @@ endif
 
 libexec_PROGRAMS = \
   gnunet-daemon-exit \
-  $(EXITBIN) 
+  $(EXITBIN)
 
 if MINGW
   gnunet_helper_exit_LDFLAGS = \
-    -no-undefined -Wl,--export-all-symbols 
+    -no-undefined -Wl,--export-all-symbols
 
   gnunet_helper_exit_LDADD = \
     -lsetupapi -lnewdev -lshell32 -liconv -lstdc++ \
@@ -47,7 +47,7 @@ else
    gnunet-helper-exit.c
 endif
 gnunet_daemon_exit_SOURCES = \
- gnunet-daemon-exit.c exit.h 
+ gnunet-daemon-exit.c exit.h
 gnunet_daemon_exit_LDADD = \
   $(top_builddir)/src/dns/libgnunetdnsstub.la \
   $(top_builddir)/src/dht/libgnunetdht.la \
diff --git a/src/exit/gnunet-daemon-exit.c b/src/exit/gnunet-daemon-exit.c
index 68699db71..87b30c04a 100644
--- a/src/exit/gnunet-daemon-exit.c
+++ b/src/exit/gnunet-daemon-exit.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2010-2013 Christian Grothoff
+     Copyright (C) 2010-2013, 2017 Christian Grothoff
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -195,33 +195,6 @@ struct RedirectInformation
 
 
 /**
- * Queue of messages to a channel.
- */
-struct ChannelMessageQueue
-{
-  /**
-   * This is a doubly-linked list.
-   */
-  struct ChannelMessageQueue *next;
-
-  /**
-   * This is a doubly-linked list.
-   */
-  struct ChannelMessageQueue *prev;
-
-  /**
-   * Payload to send via the channel.
-   */
-  const void *payload;
-
-  /**
-   * Number of bytes in @e payload.
-   */
-  size_t len;
-};
-
-
-/**
  * This struct is saved into #connections_map to allow finding the
  * right channel given an IP packet from TUN.  It is also associated
  * with the channel's closure so we can find it again for the next
@@ -241,11 +214,6 @@ struct ChannelState
   struct GNUNET_PeerIdentity peer;
 
   /**
-   * Active channel transmission request (or NULL).
-   */
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  /**
    * #GNUNET_NO if this is a channel for TCP/UDP,
    * #GNUNET_YES if this is a channel for DNS,
    * #GNUNET_SYSERR if the channel is not yet initialized.
@@ -273,16 +241,6 @@ struct ChannelState
       struct LocalService *serv;
 
       /**
-       * Head of DLL of messages for this channel.
-       */
-      struct ChannelMessageQueue *head;
-
-      /**
-       * Tail of DLL of messages for this channel.
-       */
-      struct ChannelMessageQueue *tail;
-
-      /**
        * Primary redirection information for this connection.
        */
       struct RedirectInformation ri;
@@ -292,22 +250,12 @@ struct ChannelState
     {
 
       /**
-       * DNS reply ready for transmission.
-       */
-      char *reply;
-
-      /**
        * Socket we are using to transmit this request (must match if we receive
        * a response).
        */
       struct GNUNET_DNSSTUB_RequestSocket *rs;
 
       /**
-       * Number of bytes in 'reply'.
-       */
-      size_t reply_length;
-
-      /**
        * Original DNS request ID as used by the client.
        */
       uint16_t original_id;
@@ -428,7 +376,7 @@ static struct GNUNET_DHT_Handle *dht;
 /**
  * Task for doing DHT PUTs to advertise exit service.
  */
-static struct GNUNET_SCHEDULER_Task * dht_task;
+static struct GNUNET_SCHEDULER_Task *dht_task;
 
 /**
  * Advertisement message we put into the DHT to advertise us
@@ -447,6 +395,21 @@ static struct GNUNET_HashCode dht_put_key;
 static struct GNUNET_CRYPTO_EddsaPrivateKey *peer_key;
 
 /**
+ * Port for DNS exit.
+ */
+static struct GNUNET_CADET_Port *dns_port;
+
+/**
+ * Port for IPv4 exit.
+ */
+static struct GNUNET_CADET_Port *cadet_port4;
+
+/**
+ * Port for IPv6 exit.
+ */
+static struct GNUNET_CADET_Port *cadet_port6;
+
+/**
  * Are we an IPv4-exit?
  */
 static int ipv4_exit;
@@ -467,51 +430,27 @@ static int ipv4_enabled;
 static int ipv6_enabled;
 
 
+GNUNET_NETWORK_STRUCT_BEGIN
+
 /**
- * We got a reply from DNS for a request of a CADET channel.  Send it
- * via the channel (after changing the request ID back).
- *
- * @param cls the `struct ChannelState`
- * @param size number of bytes available in @a buf
- * @param buf where to copy the reply
- * @return number of bytes written to @a buf
- */
-static size_t
-transmit_reply_to_cadet (void *cls,
-                        size_t size,
-                        void *buf)
+ * Message with a DNS response.
+ */
+struct DnsResponseMessage
 {
-  struct ChannelState *ts = cls;
-  size_t off;
-  size_t ret;
-  char *cbuf = buf;
-  struct GNUNET_MessageHeader hdr;
+  /**
+   * GNUnet header, of type #GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * DNS header.
+   */
   struct GNUNET_TUN_DnsHeader dns;
 
-  GNUNET_assert (GNUNET_YES == ts->is_dns);
-  ts->th = NULL;
-  GNUNET_assert (ts->specifics.dns.reply != NULL);
-  if (size == 0)
-    return 0;
-  ret = sizeof (struct GNUNET_MessageHeader) + ts->specifics.dns.reply_length;
-  GNUNET_assert (ret <= size);
-  hdr.size = htons (ret);
-  hdr.type = htons (GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET);
-  GNUNET_memcpy (&dns, ts->specifics.dns.reply, sizeof (dns));
-  dns.id = ts->specifics.dns.original_id;
-  off = 0;
-  GNUNET_memcpy (&cbuf[off], &hdr, sizeof (hdr));
-  off += sizeof (hdr);
-  GNUNET_memcpy (&cbuf[off], &dns, sizeof (dns));
-  off += sizeof (dns);
-  GNUNET_memcpy (&cbuf[off], &ts->specifics.dns.reply[sizeof (dns)], ts->specifics.dns.reply_length - sizeof (dns));
-  off += ts->specifics.dns.reply_length - sizeof (dns);
-  GNUNET_free (ts->specifics.dns.reply);
-  ts->specifics.dns.reply = NULL;
-  ts->specifics.dns.reply_length = 0;
-  GNUNET_assert (ret == off);
-  return ret;
-}
+  /* Followed by more DNS payload */
+};
+
+GNUNET_NETWORK_STRUCT_END
 
 
 /**
@@ -521,7 +460,7 @@ transmit_reply_to_cadet (void *cls,
  * @param cls NULL
  * @param rs the socket that received the response
  * @param dns the response itself
- * @param r number of bytes in dns
+ * @param r number of bytes in @a dns
  */
 static void
 process_dns_result (void *cls,
@@ -530,6 +469,8 @@ process_dns_result (void *cls,
                     size_t r)
 {
   struct ChannelState *ts;
+  struct GNUNET_MQ_Envelope *env;
+  struct DnsResponseMessage *resp;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Processing DNS result from stub resolver\n");
@@ -542,48 +483,35 @@ process_dns_result (void *cls,
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Got a response from the stub resolver for DNS request received via CADET!\n");
   channels[dns->id] = NULL;
-  GNUNET_free_non_null (ts->specifics.dns.reply);
-  ts->specifics.dns.reply = GNUNET_malloc (r);
-  ts->specifics.dns.reply_length = r;
-  GNUNET_memcpy (ts->specifics.dns.reply, dns, r);
-  if (NULL != ts->th)
-    GNUNET_CADET_notify_transmit_ready_cancel (ts->th);
-  ts->th = GNUNET_CADET_notify_transmit_ready (ts->channel,
-                                              GNUNET_NO,
-                                              GNUNET_TIME_UNIT_FOREVER_REL,
-                                              sizeof (struct GNUNET_MessageHeader) + r,
-                                              &transmit_reply_to_cadet,
-                                              ts);
+  env = GNUNET_MQ_msg_extra (resp,
+                             r - sizeof (struct GNUNET_TUN_DnsHeader),
+                             GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET);
+  GNUNET_memcpy (&resp->dns,
+                 dns,
+                 r);
+  resp->dns.id = ts->specifics.dns.original_id;
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (ts->channel),
+                  env);
 }
 
 
 /**
- * Process a request via cadet to perform a DNS query.
- *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct ChannelState *`
- * @param message the actual message
+ * Check a request via cadet to perform a DNS query.
  *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_dns_request (void *cls GNUNET_UNUSED,
-                     struct GNUNET_CADET_Channel *channel,
-                     void **channel_ctx,
-                     const struct GNUNET_MessageHeader *message)
+check_dns_request (void *cls,
+                   const struct DnsResponseMessage *msg)
 {
-  struct ChannelState *ts = *channel_ctx;
-  const struct GNUNET_TUN_DnsHeader *dns;
-  size_t mlen = ntohs (message->size);
-  size_t dlen = mlen - sizeof (struct GNUNET_MessageHeader);
-  char buf[dlen] GNUNET_ALIGN;
-  struct GNUNET_TUN_DnsHeader *dout;
+  struct ChannelState *ts = cls;
 
   if (NULL == dnsstub)
   {
-    GNUNET_break_op (0);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (GNUNET_NO == ts->is_dns)
@@ -591,34 +519,53 @@ receive_dns_request (void *cls GNUNET_UNUSED,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request via cadet to perform a DNS query.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ */
+static void
+handle_dns_request (void *cls,
+                     const struct DnsResponseMessage *msg)
+{
+  struct ChannelState *ts = cls;
+  size_t mlen = ntohs (msg->header.size);
+  size_t dlen = mlen - sizeof (struct GNUNET_MessageHeader);
+  char buf[dlen] GNUNET_ALIGN;
+  struct GNUNET_TUN_DnsHeader *dout;
+
   if (GNUNET_SYSERR == ts->is_dns)
   {
     /* channel is DNS from now on */
     ts->is_dns = GNUNET_YES;
   }
-  if (dlen < sizeof (struct GNUNET_TUN_DnsHeader))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  dns = (const struct GNUNET_TUN_DnsHeader *) &message[1];
-  ts->specifics.dns.original_id = dns->id;
+  ts->specifics.dns.original_id = msg->dns.id;
   if (channels[ts->specifics.dns.my_id] == ts)
     channels[ts->specifics.dns.my_id] = NULL;
   ts->specifics.dns.my_id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                                   UINT16_MAX + 1);
+                                                                 UINT16_MAX + 1);
   channels[ts->specifics.dns.my_id] = ts;
-  GNUNET_memcpy (buf, dns, dlen);
+  GNUNET_memcpy (buf,
+                 &msg->dns,
+                 dlen);
   dout = (struct GNUNET_TUN_DnsHeader *) buf;
   dout->id = ts->specifics.dns.my_id;
   ts->specifics.dns.rs = GNUNET_DNSSTUB_resolve2 (dnsstub,
-                                                  buf, dlen,
+                                                  buf,
+                                                  dlen,
                                                   &process_dns_result,
                                                   NULL);
   if (NULL == ts->specifics.dns.rs)
-    return GNUNET_SYSERR;
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  GNUNET_CADET_receive_done (ts->channel);
 }
 
 
@@ -753,706 +700,445 @@ get_redirect_state (int af,
 }
 
 
-/**
- * Free memory associated with a service record.
- *
- * @param cls unused
- * @param key service descriptor
- * @param value service record to free
- * @return #GNUNET_OK
- */
-static int
-free_service_record (void *cls,
-                     const struct GNUNET_HashCode *key,
-                     void *value)
-{
-  struct LocalService *service = value;
-
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (services,
-                                                       key,
-                                                       service));
-  GNUNET_CADET_close_port (service->port);
-  GNUNET_free_non_null (service->name);
-  GNUNET_free (service);
-  return GNUNET_OK;
-}
-
-
-/**
- * Callback from CADET for new channels.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port destination port
- * @param options channel options flags
- * @return initial channel context for the channel
- */
-static void *
-new_service_channel (void *cls,
-                     struct GNUNET_CADET_Channel *channel,
-                     const struct GNUNET_PeerIdentity *initiator,
-                     const struct GNUNET_HashCode *port,
-                     enum GNUNET_CADET_ChannelOption options)
-{
-  struct LocalService *ls = cls;
-  struct ChannelState *s = GNUNET_new (struct ChannelState);
-
-  s->peer = *initiator;
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Inbound CADET channels created"),
-                            1,
-                            GNUNET_NO);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received inbound channel from `%s'\n",
-              GNUNET_i2s (initiator));
-  s->channel = channel;
-  s->specifics.tcp_udp.serv = ls;
-  s->specifics.tcp_udp.ri.remote_address = ls->address;
-  return s;
-}
-
 
 /**
- * Given a service descriptor and a destination port, find the
- * respective service entry.
+ * Check a request via cadet to send a request to a TCP service
+ * offered by this system.
  *
- * @param proto IPPROTO_TCP or IPPROTO_UDP
- * @param name name of the service
- * @param destination_port destination port
- * @param service service information record to store (service->name will be set).
+ * @param cls our `struct ChannelState *`
+ * @param start the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
  */
-static void
-store_service (int proto,
-               const char *name,
-               uint16_t destination_port,
-               struct LocalService *service)
+static int
+check_tcp_service (void *cls,
+                   const struct GNUNET_EXIT_TcpServiceStartMessage *start)
 {
-  struct GNUNET_HashCode cadet_port;
+  struct ChannelState *state = cls;
 
-  service->name = GNUNET_strdup (name);
-  GNUNET_TUN_service_name_to_hash (name,
-                                   &service->descriptor);
-  GNUNET_TUN_compute_service_cadet_port (&service->descriptor,
-                                         destination_port,
-                                         &cadet_port);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Opening CADET port %s for SERVICE exit %s on port %u\n",
-              GNUNET_h2s (&cadet_port),
-              name,
-              (unsigned int) destination_port);
-  service->port = GNUNET_CADET_open_port (cadet_handle,
-                                          &cadet_port,
-                                          &new_service_channel,
-                                          service);
-  service->is_udp = (IPPROTO_UDP == proto);
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (services,
-                                         &cadet_port,
-                                         service,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  if (NULL == state)
   {
-    GNUNET_CADET_close_port (service->port);
-    GNUNET_free_non_null (service->name);
-    GNUNET_free (service);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Got duplicate service records for `%s:%u'\n"),
-                name,
-                (unsigned int) destination_port);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
+  if (GNUNET_YES == state->is_dns)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (NULL == state->specifics.tcp_udp.serv)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (NULL != state->specifics.tcp_udp.heap_node)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (start->tcp_header.off * 4 < sizeof (struct GNUNET_TUN_TcpHeader))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
 }
 
 
 /**
- * CADET is ready to receive a message for the channel.  Transmit it.
- *
- * @param cls the `struct ChannelState`.
- * @param size number of bytes available in @a buf
- * @param buf where to copy the message
- * @return number of bytes copied to @a buf
- */
-static size_t
-send_to_peer_notify_callback (void *cls,
-                              size_t size,
-                              void *buf)
-{
-  struct ChannelState *s = cls;
-  struct GNUNET_CADET_Channel *channel = s->channel;
-  struct ChannelMessageQueue *tnq;
-
-  s->th = NULL;
-  tnq = s->specifics.tcp_udp.head;
-  if (NULL == tnq)
-    return 0;
-  if (0 == size)
-  {
-    s->th = GNUNET_CADET_notify_transmit_ready (channel,
-                                               GNUNET_NO /* corking */,
-                                               GNUNET_TIME_UNIT_FOREVER_REL,
-                                               tnq->len,
-                                               &send_to_peer_notify_callback,
-                                               s);
-    return 0;
-  }
-  GNUNET_assert (size >= tnq->len);
-  GNUNET_memcpy (buf, tnq->payload, tnq->len);
-  size = tnq->len;
-  GNUNET_CONTAINER_DLL_remove (s->specifics.tcp_udp.head,
-                               s->specifics.tcp_udp.tail,
-                               tnq);
-  GNUNET_free (tnq);
-  if (NULL != (tnq = s->specifics.tcp_udp.head))
-    s->th = GNUNET_CADET_notify_transmit_ready (channel,
-                                               GNUNET_NO /* corking */,
-                                               GNUNET_TIME_UNIT_FOREVER_REL,
-                                               tnq->len,
-                                               &send_to_peer_notify_callback,
-                                               s);
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes transmitted via cadet channels"),
-                            size, GNUNET_NO);
-  return size;
-}
-
-
-/**
- * Send the given packet via the cadet channel.
- *
- * @param s channel destination
- * @param tnq message to queue
- */
-static void
-send_packet_to_cadet_channel (struct ChannelState *s,
-                              struct ChannelMessageQueue *tnq)
-{
-  struct GNUNET_CADET_Channel *cadet_channel;
-
-  cadet_channel = s->channel;
-  GNUNET_assert (NULL != s);
-  GNUNET_CONTAINER_DLL_insert_tail (s->specifics.tcp_udp.head,
-                                    s->specifics.tcp_udp.tail,
-                                    tnq);
-  if (NULL == s->th)
-    s->th = GNUNET_CADET_notify_transmit_ready (cadet_channel,
-                                                GNUNET_NO /* cork */,
-                                                GNUNET_TIME_UNIT_FOREVER_REL,
-                                                tnq->len,
-                                                &send_to_peer_notify_callback,
-                                                s);
-}
-
-
-/**
- * @brief Handles an ICMP packet received from the helper.
+ * Prepare an IPv4 packet for transmission via the TUN interface.
+ * Initializes the IP header and calculates checksums (IP+UDP/TCP).
+ * For UDP, the UDP header will be fully created, whereas for TCP
+ * only the ports and checksum will be filled in.  So for TCP,
+ * a skeleton TCP header must be part of the provided payload.
  *
- * @param icmp A pointer to the Packet
- * @param pktlen number of bytes in @a icmp
- * @param af address family (AFINET or AF_INET6)
- * @param destination_ip destination IP-address of the IP packet (should
- *                       be our local address)
- * @param source_ip original source IP-address of the IP packet (should
- *                       be the original destination address)
+ * @param payload payload of the packet (starting with UDP payload or
+ *                TCP header, depending on protocol)
+ * @param payload_length number of bytes in @a payload
+ * @param protocol IPPROTO_UDP or IPPROTO_TCP
+ * @param tcp_header skeleton of the TCP header, NULL for UDP
+ * @param src_address source address to use (IP and port)
+ * @param dst_address destination address to use (IP and port)
+ * @param pkt4 where to write the assembled packet; must
+ *        contain enough space for the IP header, UDP/TCP header
+ *        AND the payload
  */
 static void
-icmp_from_helper (const struct GNUNET_TUN_IcmpHeader *icmp,
-                  size_t pktlen,
-                  int af,
-                  const void *destination_ip,
-                  const void *source_ip)
+prepare_ipv4_packet (const void *payload,
+                     size_t payload_length,
+                     int protocol,
+                     const struct GNUNET_TUN_TcpHeader *tcp_header,
+                     const struct SocketAddress *src_address,
+                     const struct SocketAddress *dst_address,
+                     struct GNUNET_TUN_IPv4Header *pkt4)
 {
-  struct ChannelState *state;
-  struct ChannelMessageQueue *tnq;
-  struct GNUNET_EXIT_IcmpToVPNMessage *i2v;
-  const struct GNUNET_TUN_IPv4Header *ipv4;
-  const struct GNUNET_TUN_IPv6Header *ipv6;
-  const struct GNUNET_TUN_UdpHeader *udp;
-  size_t mlen;
-  uint16_t source_port;
-  uint16_t destination_port;
-  uint8_t protocol;
+  size_t len;
 
+  len = payload_length;
+  switch (protocol)
   {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received ICMP packet going from %s to %s\n",
-                inet_ntop (af,
-                           source_ip,
-                           sbuf, sizeof (sbuf)),
-                inet_ntop (af,
-                           destination_ip,
-                           dbuf, sizeof (dbuf)));
+  case IPPROTO_UDP:
+    len += sizeof (struct GNUNET_TUN_UdpHeader);
+    break;
+  case IPPROTO_TCP:
+    len += sizeof (struct GNUNET_TUN_TcpHeader);
+    GNUNET_assert (NULL != tcp_header);
+    break;
+  default:
+    GNUNET_break (0);
+    return;
   }
-  if (pktlen < sizeof (struct GNUNET_TUN_IcmpHeader))
+  if (len + sizeof (struct GNUNET_TUN_IPv4Header) > UINT16_MAX)
   {
-    /* blame kernel */
     GNUNET_break (0);
     return;
   }
 
-  /* Find out if this is an ICMP packet in response to an existing
-     TCP/UDP packet and if so, figure out ports / protocol of the
-     existing session from the IP data in the ICMP payload */
-  source_port = 0;
-  destination_port = 0;
-  switch (af)
-  {
-  case AF_INET:
-    protocol = IPPROTO_ICMP;
-    switch (icmp->type)
-      {
-      case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
-      case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
-        break;
-      case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
-      case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
-      case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
-        if (pktlen <
-            sizeof (struct GNUNET_TUN_IcmpHeader) +
-            sizeof (struct GNUNET_TUN_IPv4Header) + 8)
-        {
-          /* blame kernel */
-          GNUNET_break (0);
-          return;
-        }
-        ipv4 = (const struct GNUNET_TUN_IPv4Header *) &icmp[1];
-        protocol = ipv4->protocol;
-        /* could be TCP or UDP, but both have the ports in the right
-           place, so that doesn't matter here */
-        udp = (const struct GNUNET_TUN_UdpHeader *) &ipv4[1];
-        /* swap ports, as they are from the original message */
-        destination_port = ntohs (udp->source_port);
-        source_port = ntohs (udp->destination_port);
-        /* throw away ICMP payload, won't be useful for the other side anyway */
-        pktlen = sizeof (struct GNUNET_TUN_IcmpHeader);
-        break;
-      default:
-        GNUNET_STATISTICS_update (stats,
-                                  gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
-                                  1, GNUNET_NO);
-        return;
-      }
-    break;
-  case AF_INET6:
-    protocol = IPPROTO_ICMPV6;
-    switch (icmp->type)
-      {
-      case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
-      case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
-      case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
-      case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
-        if (pktlen <
-            sizeof (struct GNUNET_TUN_IcmpHeader) +
-            sizeof (struct GNUNET_TUN_IPv6Header) + 8)
-        {
-          /* blame kernel */
-          GNUNET_break (0);
-          return;
-        }
-        ipv6 = (const struct GNUNET_TUN_IPv6Header *) &icmp[1];
-        protocol = ipv6->next_header;
-        /* could be TCP or UDP, but both have the ports in the right
-           place, so that doesn't matter here */
-        udp = (const struct GNUNET_TUN_UdpHeader *) &ipv6[1];
-        /* swap ports, as they are from the original message */
-        destination_port = ntohs (udp->source_port);
-        source_port = ntohs (udp->destination_port);
-        /* throw away ICMP payload, won't be useful for the other side anyway */
-        pktlen = sizeof (struct GNUNET_TUN_IcmpHeader);
-        break;
-      case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
-      case GNUNET_TUN_ICMPTYPE6_ECHO_REPLY:
-        break;
-      default:
-        GNUNET_STATISTICS_update (stats,
-                                  gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
-                                  1, GNUNET_NO);
-        return;
-      }
-    break;
-  default:
-    GNUNET_assert (0);
-  }
+  GNUNET_TUN_initialize_ipv4_header (pkt4,
+                                     protocol,
+                                     len,
+                                     &src_address->address.ipv4,
+                                     &dst_address->address.ipv4);
   switch (protocol)
   {
-  case IPPROTO_ICMP:
-    state = get_redirect_state (af,
-                                IPPROTO_ICMP,
-                                source_ip,
-                                0,
-                                destination_ip,
-                                0,
-                                NULL);
-    break;
-  case IPPROTO_ICMPV6:
-    state = get_redirect_state (af,
-                                IPPROTO_ICMPV6,
-                                source_ip,
-                                0,
-                                destination_ip,
-                                0,
-                                NULL);
-    break;
   case IPPROTO_UDP:
-    state = get_redirect_state (af,
-                                IPPROTO_UDP,
-                                source_ip,
-                                source_port,
-                                destination_ip,
-                                destination_port,
-                                NULL);
+    {
+      struct GNUNET_TUN_UdpHeader *pkt4_udp = (struct GNUNET_TUN_UdpHeader *) &pkt4[1];
+
+      pkt4_udp->source_port = htons (src_address->port);
+      pkt4_udp->destination_port = htons (dst_address->port);
+      pkt4_udp->len = htons ((uint16_t) payload_length);
+      GNUNET_TUN_calculate_udp4_checksum (pkt4,
+                                          pkt4_udp,
+                                          payload,
+                                          payload_length);
+      GNUNET_memcpy (&pkt4_udp[1],
+                     payload,
+                     payload_length);
+    }
     break;
   case IPPROTO_TCP:
-    state = get_redirect_state (af,
-                                IPPROTO_TCP,
-                                source_ip,
-                                source_port,
-                                destination_ip,
-                                destination_port,
-                                NULL);
+    {
+      struct GNUNET_TUN_TcpHeader *pkt4_tcp = (struct GNUNET_TUN_TcpHeader *) &pkt4[1];
+
+      *pkt4_tcp = *tcp_header;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending TCP packet from port %u to port %u\n",
+                  src_address->port,
+                  dst_address->port);
+      pkt4_tcp->source_port = htons (src_address->port);
+      pkt4_tcp->destination_port = htons (dst_address->port);
+      GNUNET_TUN_calculate_tcp4_checksum (pkt4,
+                                          pkt4_tcp,
+                                          payload,
+                                          payload_length);
+      GNUNET_memcpy (&pkt4_tcp[1],
+                     payload,
+                     payload_length);
+    }
     break;
   default:
-    GNUNET_STATISTICS_update (stats,
-                              gettext_noop ("# ICMP packets dropped (not allowed)"),
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  if (NULL == state)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                _("ICMP Packet dropped, have no matching connection information\n"));
-    return;
+    GNUNET_assert (0);
   }
-  mlen = sizeof (struct GNUNET_EXIT_IcmpToVPNMessage) + pktlen - sizeof (struct GNUNET_TUN_IcmpHeader);
-  tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueue) + mlen);
-  tnq->payload = &tnq[1];
-  tnq->len = mlen;
-  i2v = (struct GNUNET_EXIT_IcmpToVPNMessage *) &tnq[1];
-  i2v->header.size = htons ((uint16_t) mlen);
-  i2v->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_VPN);
-  i2v->af = htonl (af);
-  GNUNET_memcpy (&i2v->icmp_header,
-          icmp,
-          pktlen);
-  send_packet_to_cadet_channel (state, tnq);
 }
 
 
 /**
- * @brief Handles an UDP packet received from the helper.
+ * Prepare an IPv6 packet for transmission via the TUN interface.
+ * Initializes the IP header and calculates checksums (IP+UDP/TCP).
+ * For UDP, the UDP header will be fully created, whereas for TCP
+ * only the ports and checksum will be filled in.  So for TCP,
+ * a skeleton TCP header must be part of the provided payload.
  *
- * @param udp A pointer to the Packet
- * @param pktlen number of bytes in 'udp'
- * @param af address family (AFINET or AF_INET6)
- * @param destination_ip destination IP-address of the IP packet (should
- *                       be our local address)
- * @param source_ip original source IP-address of the IP packet (should
- *                       be the original destination address)
+ * @param payload payload of the packet (starting with UDP payload or
+ *                TCP header, depending on protocol)
+ * @param payload_length number of bytes in @a payload
+ * @param protocol IPPROTO_UDP or IPPROTO_TCP
+ * @param tcp_header skeleton TCP header data to send, NULL for UDP
+ * @param src_address source address to use (IP and port)
+ * @param dst_address destination address to use (IP and port)
+ * @param pkt6 where to write the assembled packet; must
+ *        contain enough space for the IP header, UDP/TCP header
+ *        AND the payload
  */
 static void
-udp_from_helper (const struct GNUNET_TUN_UdpHeader *udp,
-                 size_t pktlen,
-                 int af,
-                 const void *destination_ip,
-                 const void *source_ip)
+prepare_ipv6_packet (const void *payload,
+                     size_t payload_length,
+                     int protocol,
+                     const struct GNUNET_TUN_TcpHeader *tcp_header,
+                     const struct SocketAddress *src_address,
+                     const struct SocketAddress *dst_address,
+                     struct GNUNET_TUN_IPv6Header *pkt6)
 {
-  struct ChannelState *state;
-  struct ChannelMessageQueue *tnq;
-  struct GNUNET_EXIT_UdpReplyMessage *urm;
-  size_t mlen;
+  size_t len;
 
+  len = payload_length;
+  switch (protocol)
   {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received UDP packet going from %s:%u to %s:%u\n",
-                inet_ntop (af,
-                           source_ip,
-                           sbuf, sizeof (sbuf)),
-                (unsigned int) ntohs (udp->source_port),
-                inet_ntop (af,
-                           destination_ip,
-                           dbuf, sizeof (dbuf)),
-                (unsigned int) ntohs (udp->destination_port));
-  }
-  if (pktlen < sizeof (struct GNUNET_TUN_UdpHeader))
-  {
-    /* blame kernel */
+  case IPPROTO_UDP:
+    len += sizeof (struct GNUNET_TUN_UdpHeader);
+    break;
+  case IPPROTO_TCP:
+    len += sizeof (struct GNUNET_TUN_TcpHeader);
+    break;
+  default:
     GNUNET_break (0);
     return;
   }
-  if (pktlen != ntohs (udp->len))
+  if (len > UINT16_MAX)
   {
-    /* blame kernel */
     GNUNET_break (0);
     return;
   }
-  state = get_redirect_state (af,
-                              IPPROTO_UDP,
-                              source_ip,
-                              ntohs (udp->source_port),
-                              destination_ip,
-                              ntohs (udp->destination_port),
-                              NULL);
-  if (NULL == state)
+
+  GNUNET_TUN_initialize_ipv6_header (pkt6,
+                                     protocol,
+                                     len,
+                                     &src_address->address.ipv6,
+                                     &dst_address->address.ipv6);
+
+  switch (protocol)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                _("UDP Packet dropped, have no matching connection information\n"));
-    return;
+  case IPPROTO_UDP:
+    {
+      struct GNUNET_TUN_UdpHeader *pkt6_udp = (struct GNUNET_TUN_UdpHeader *) &pkt6[1];
+
+      pkt6_udp->source_port = htons (src_address->port);
+      pkt6_udp->destination_port = htons (dst_address->port);
+      pkt6_udp->len = htons ((uint16_t) payload_length);
+      GNUNET_TUN_calculate_udp6_checksum (pkt6,
+                                          pkt6_udp,
+                                          payload,
+                                          payload_length);
+      GNUNET_memcpy (&pkt6_udp[1],
+                     payload,
+                     payload_length);
+    }
+    break;
+  case IPPROTO_TCP:
+    {
+      struct GNUNET_TUN_TcpHeader *pkt6_tcp = (struct GNUNET_TUN_TcpHeader *) &pkt6[1];
+
+      /* GNUNET_memcpy first here as some TCP header fields are initialized this way! */
+      *pkt6_tcp = *tcp_header;
+      pkt6_tcp->source_port = htons (src_address->port);
+      pkt6_tcp->destination_port = htons (dst_address->port);
+      GNUNET_TUN_calculate_tcp6_checksum (pkt6,
+                                          pkt6_tcp,
+                                          payload,
+                                          payload_length);
+      GNUNET_memcpy (&pkt6_tcp[1],
+                     payload,
+                     payload_length);
+    }
+    break;
+  default:
+    GNUNET_assert (0);
+    break;
   }
-  mlen = sizeof (struct GNUNET_EXIT_UdpReplyMessage) + pktlen - sizeof (struct GNUNET_TUN_UdpHeader);
-  tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueue) + mlen);
-  tnq->payload = &tnq[1];
-  tnq->len = mlen;
-  urm = (struct GNUNET_EXIT_UdpReplyMessage *) &tnq[1];
-  urm->header.size = htons ((uint16_t) mlen);
-  urm->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_UDP_REPLY);
-  urm->source_port = htons (0);
-  urm->destination_port = htons (0);
-  GNUNET_memcpy (&urm[1],
-          &udp[1],
-          pktlen - sizeof (struct GNUNET_TUN_UdpHeader));
-  send_packet_to_cadet_channel (state, tnq);
 }
 
 
 /**
- * @brief Handles a TCP packet received from the helper.
+ * Send a TCP packet via the TUN interface.
  *
- * @param tcp A pointer to the Packet
- * @param pktlen the length of the packet, including its TCP header
- * @param af address family (AFINET or AF_INET6)
- * @param destination_ip destination IP-address of the IP packet (should
- *                       be our local address)
- * @param source_ip original source IP-address of the IP packet (should
- *                       be the original destination address)
+ * @param destination_address IP and port to use for the TCP packet's destination
+ * @param source_address IP and port to use for the TCP packet's source
+ * @param tcp_header header template to use
+ * @param payload payload of the TCP packet
+ * @param payload_length number of bytes in @a payload
  */
 static void
-tcp_from_helper (const struct GNUNET_TUN_TcpHeader *tcp,
-                 size_t pktlen,
-                 int af,
-                 const void *destination_ip,
-                 const void *source_ip)
+send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
+                         const struct SocketAddress *source_address,
+                         const struct GNUNET_TUN_TcpHeader *tcp_header,
+                         const void *payload,
+                         size_t payload_length)
 {
-  struct ChannelState *state;
-  char buf[pktlen] GNUNET_ALIGN;
-  struct GNUNET_TUN_TcpHeader *mtcp;
-  struct GNUNET_EXIT_TcpDataMessage *tdm;
-  struct ChannelMessageQueue *tnq;
-  size_t mlen;
+  size_t len;
 
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# TCP packets sent via TUN"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending packet with %u bytes TCP payload via TUN\n",
+              (unsigned int) payload_length);
+  len = sizeof (struct GNUNET_MessageHeader) + sizeof (struct GNUNET_TUN_Layer2PacketHeader);
+  switch (source_address->af)
   {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received TCP packet with %u bytes going from %s:%u to %s:%u\n",
-                (unsigned int) (pktlen - sizeof (struct GNUNET_TUN_TcpHeader)),
-                inet_ntop (af,
-                           source_ip,
-                           sbuf, sizeof (sbuf)),
-                (unsigned int) ntohs (tcp->source_port),
-                inet_ntop (af,
-                           destination_ip,
-                           dbuf, sizeof (dbuf)),
-                (unsigned int) ntohs (tcp->destination_port));
+  case AF_INET:
+    len += sizeof (struct GNUNET_TUN_IPv4Header);
+    break;
+  case AF_INET6:
+    len += sizeof (struct GNUNET_TUN_IPv6Header);
+    break;
+  default:
+    GNUNET_break (0);
+    return;
   }
-  if (pktlen < sizeof (struct GNUNET_TUN_TcpHeader))
+  len += sizeof (struct GNUNET_TUN_TcpHeader);
+  len += payload_length;
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
-    /* blame kernel */
     GNUNET_break (0);
     return;
   }
-  state = get_redirect_state (af,
-                              IPPROTO_TCP,
-                              source_ip,
-                              ntohs (tcp->source_port),
-                              destination_ip,
-                              ntohs (tcp->destination_port),
-                              NULL);
-  if (NULL == state)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                _("TCP Packet dropped, have no matching connection information\n"));
+    char buf[len] GNUNET_ALIGN;
+    struct GNUNET_MessageHeader *hdr;
+    struct GNUNET_TUN_Layer2PacketHeader *tun;
 
-    return;
-  }
-  /* mug port numbers and crc to avoid information leakage;
-     sender will need to lookup the correct values anyway */
-  GNUNET_memcpy (buf, tcp, pktlen);
-  mtcp = (struct GNUNET_TUN_TcpHeader *) buf;
-  mtcp->source_port = 0;
-  mtcp->destination_port = 0;
-  mtcp->crc = 0;
+    hdr = (struct GNUNET_MessageHeader *) buf;
+    hdr->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+    hdr->size = htons (len);
+    tun = (struct GNUNET_TUN_Layer2PacketHeader*) &hdr[1];
+    tun->flags = htons (0);
+    switch (source_address->af)
+    {
+    case AF_INET:
+      {
+        struct GNUNET_TUN_IPv4Header *ipv4
+          = (struct GNUNET_TUN_IPv4Header*) &tun[1];
 
-  mlen = sizeof (struct GNUNET_EXIT_TcpDataMessage) + (pktlen - sizeof (struct GNUNET_TUN_TcpHeader));
-  if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
+        tun->proto = htons (ETH_P_IPV4);
+        prepare_ipv4_packet (payload,
+                             payload_length,
+                             IPPROTO_TCP,
+                             tcp_header,
+                             source_address,
+                             destination_address,
+                             ipv4);
+      }
+      break;
+    case AF_INET6:
+      {
+        struct GNUNET_TUN_IPv6Header *ipv6
+          = (struct GNUNET_TUN_IPv6Header*) &tun[1];
 
-  tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueue) + mlen);
-  tnq->payload = &tnq[1];
-  tnq->len = mlen;
-  tdm = (struct GNUNET_EXIT_TcpDataMessage *) &tnq[1];
-  tdm->header.size = htons ((uint16_t) mlen);
-  tdm->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_VPN);
-  tdm->reserved = htonl (0);
-  GNUNET_memcpy (&tdm->tcp_header,
-          buf,
-          pktlen);
-  send_packet_to_cadet_channel (state, tnq);
+        tun->proto = htons (ETH_P_IPV6);
+        prepare_ipv6_packet (payload,
+                             payload_length,
+                             IPPROTO_TCP,
+                             tcp_header,
+                             source_address,
+                             destination_address,
+                             ipv6);
+      }
+      break;
+    default:
+      GNUNET_assert (0);
+      break;
+    }
+    if (NULL != helper_handle)
+      (void) GNUNET_HELPER_send (helper_handle,
+                                 (const struct GNUNET_MessageHeader*) buf,
+                                 GNUNET_YES,
+                                 NULL,
+                                 NULL);
+  }
 }
 
 
 /**
- * Receive packets from the helper-process
+ * Send an ICMP packet via the TUN interface.
  *
- * @param cls unused
- * @param client unsued
- * @param message message received from helper
+ * @param destination_address IP to use for the ICMP packet's destination
+ * @param source_address IP to use for the ICMP packet's source
+ * @param icmp_header ICMP header to send
+ * @param payload payload of the ICMP packet (does NOT include ICMP header)
+ * @param payload_length number of bytes of data in @a payload
  */
-static int
-message_token (void *cls GNUNET_UNUSED,
-               void *client GNUNET_UNUSED,
-               const struct GNUNET_MessageHeader *message)
+static void
+send_icmp_packet_via_tun (const struct SocketAddress *destination_address,
+                          const struct SocketAddress *source_address,
+                          const struct GNUNET_TUN_IcmpHeader *icmp_header,
+                          const void *payload, size_t payload_length)
 {
-  const struct GNUNET_TUN_Layer2PacketHeader *pkt_tun;
-  size_t size;
+  size_t len;
+  struct GNUNET_TUN_IcmpHeader *icmp;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got %u-byte message of type %u from gnunet-helper-exit\n",
-              ntohs (message->size),
-              ntohs (message->type));
   GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Packets received from TUN"),
+                            gettext_noop ("# ICMP packets sent via TUN"),
                             1, GNUNET_NO);
-  if (ntohs (message->type) != GNUNET_MESSAGE_TYPE_VPN_HELPER)
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending packet with %u bytes ICMP payload via TUN\n",
+              (unsigned int) payload_length);
+  len = sizeof (struct GNUNET_MessageHeader) + sizeof (struct GNUNET_TUN_Layer2PacketHeader);
+  switch (destination_address->af)
   {
+  case AF_INET:
+    len += sizeof (struct GNUNET_TUN_IPv4Header);
+    break;
+  case AF_INET6:
+    len += sizeof (struct GNUNET_TUN_IPv6Header);
+    break;
+  default:
     GNUNET_break (0);
-    return GNUNET_OK;
+    return;
   }
-  size = ntohs (message->size);
-  if (size < sizeof (struct GNUNET_TUN_Layer2PacketHeader) + sizeof (struct GNUNET_MessageHeader))
+  len += sizeof (struct GNUNET_TUN_IcmpHeader);
+  len += payload_length;
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
-    return GNUNET_OK;
+    return;
   }
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes received from TUN"),
-                            size, GNUNET_NO);
-  pkt_tun = (const struct GNUNET_TUN_Layer2PacketHeader *) &message[1];
-  size -= sizeof (struct GNUNET_TUN_Layer2PacketHeader) + sizeof (struct GNUNET_MessageHeader);
-  switch (ntohs (pkt_tun->proto))
   {
-  case ETH_P_IPV4:
-    {
-      const struct GNUNET_TUN_IPv4Header *pkt4;
-
-      if (size < sizeof (struct GNUNET_TUN_IPv4Header))
-      {
-        /* Kernel to blame? */
-        GNUNET_break (0);
-        return GNUNET_OK;
-      }
-      pkt4 = (const struct GNUNET_TUN_IPv4Header *) &pkt_tun[1];
-      if (size != ntohs (pkt4->total_length))
-      {
-        /* Kernel to blame? */
-        GNUNET_break (0);
-        return GNUNET_OK;
-      }
-      if (pkt4->header_length * 4 != sizeof (struct GNUNET_TUN_IPv4Header))
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    _("IPv4 packet options received.  Ignored.\n"));
-        return GNUNET_OK;
-      }
+    char buf[len] GNUNET_ALIGN;
+    struct GNUNET_MessageHeader *hdr;
+    struct GNUNET_TUN_Layer2PacketHeader *tun;
 
-      size -= sizeof (struct GNUNET_TUN_IPv4Header);
-      switch (pkt4->protocol)
-      {
-      case IPPROTO_UDP:
-        udp_from_helper ((const struct GNUNET_TUN_UdpHeader *) &pkt4[1], size,
-                         AF_INET,
-                         &pkt4->destination_address,
-                         &pkt4->source_address);
-        break;
-      case IPPROTO_TCP:
-        tcp_from_helper ((const struct GNUNET_TUN_TcpHeader *) &pkt4[1], size,
-                         AF_INET,
-                         &pkt4->destination_address,
-                         &pkt4->source_address);
-        break;
-      case IPPROTO_ICMP:
-        icmp_from_helper ((const struct GNUNET_TUN_IcmpHeader *) &pkt4[1], size,
-                          AF_INET,
-                          &pkt4->destination_address,
-                          &pkt4->source_address);
-        break;
-      default:
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    _("IPv4 packet with unsupported next header %u received.  Ignored.\n"),
-                    (int) pkt4->protocol);
-        return GNUNET_OK;
-      }
-    }
-    break;
-  case ETH_P_IPV6:
+    hdr= (struct GNUNET_MessageHeader *) buf;
+    hdr->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+    hdr->size = htons (len);
+    tun = (struct GNUNET_TUN_Layer2PacketHeader*) &hdr[1];
+    tun->flags = htons (0);
+    switch (source_address->af)
     {
-      const struct GNUNET_TUN_IPv6Header *pkt6;
-
-      if (size < sizeof (struct GNUNET_TUN_IPv6Header))
-      {
-        /* Kernel to blame? */
-        GNUNET_break (0);
-        return GNUNET_OK;
-      }
-      pkt6 = (struct GNUNET_TUN_IPv6Header *) &pkt_tun[1];
-      if (size != ntohs (pkt6->payload_length) + sizeof (struct GNUNET_TUN_IPv6Header))
+    case AF_INET:
       {
-        /* Kernel to blame? */
-        GNUNET_break (0);
-        return GNUNET_OK;
+        struct GNUNET_TUN_IPv4Header * ipv4 = (struct GNUNET_TUN_IPv4Header*) &tun[1];
+
+        tun->proto = htons (ETH_P_IPV4);
+        GNUNET_TUN_initialize_ipv4_header (ipv4,
+                                           IPPROTO_ICMP,
+                                           sizeof (struct GNUNET_TUN_IcmpHeader) + payload_length,
+                                           &source_address->address.ipv4,
+                                           &destination_address->address.ipv4);
+        icmp = (struct GNUNET_TUN_IcmpHeader*) &ipv4[1];
       }
-      size -= sizeof (struct GNUNET_TUN_IPv6Header);
-      switch (pkt6->next_header)
+      break;
+    case AF_INET6:
       {
-      case IPPROTO_UDP:
-        udp_from_helper ((const struct GNUNET_TUN_UdpHeader *) &pkt6[1], size,
-                         AF_INET6,
-                         &pkt6->destination_address,
-                         &pkt6->source_address);
-        break;
-      case IPPROTO_TCP:
-        tcp_from_helper ((const struct GNUNET_TUN_TcpHeader *) &pkt6[1], size,
-                         AF_INET6,
-                         &pkt6->destination_address,
-                         &pkt6->source_address);
-        break;
-      case IPPROTO_ICMPV6:
-        icmp_from_helper ((const struct GNUNET_TUN_IcmpHeader *) &pkt6[1], size,
-                          AF_INET6,
-                          &pkt6->destination_address,
-                          &pkt6->source_address);
-        break;
-      default:
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    _("IPv6 packet with unsupported next header %d received.  Ignored.\n"),
-                    pkt6->next_header);
-        return GNUNET_OK;
+        struct GNUNET_TUN_IPv6Header * ipv6 = (struct GNUNET_TUN_IPv6Header*) &tun[1];
+
+        tun->proto = htons (ETH_P_IPV6);
+        GNUNET_TUN_initialize_ipv6_header (ipv6,
+                                           IPPROTO_ICMPV6,
+                                           sizeof (struct GNUNET_TUN_IcmpHeader) + payload_length,
+                                           &source_address->address.ipv6,
+                                           &destination_address->address.ipv6);
+        icmp = (struct GNUNET_TUN_IcmpHeader*) &ipv6[1];
       }
+      break;
+    default:
+      GNUNET_assert (0);
+      break;
     }
-    break;
-  default:
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Packet from unknown protocol %u received.  Ignored.\n"),
-                ntohs (pkt_tun->proto));
-    break;
+    *icmp = *icmp_header;
+    GNUNET_memcpy (&icmp[1],
+            payload,
+            payload_length);
+    GNUNET_TUN_calculate_icmp_checksum (icmp,
+                                        payload,
+                                        payload_length);
+    if (NULL != helper_handle)
+      (void) GNUNET_HELPER_send (helper_handle,
+                                 (const struct GNUNET_MessageHeader*) buf,
+                                 GNUNET_YES,
+                                 NULL, NULL);
   }
-  return GNUNET_OK;
 }
 
 
@@ -1637,223 +1323,25 @@ setup_state_record (struct ChannelState *state)
 
 
 /**
- * Prepare an IPv4 packet for transmission via the TUN interface.
- * Initializes the IP header and calculates checksums (IP+UDP/TCP).
- * For UDP, the UDP header will be fully created, whereas for TCP
- * only the ports and checksum will be filled in.  So for TCP,
- * a skeleton TCP header must be part of the provided payload.
- *
- * @param payload payload of the packet (starting with UDP payload or
- *                TCP header, depending on protocol)
- * @param payload_length number of bytes in @a payload
- * @param protocol IPPROTO_UDP or IPPROTO_TCP
- * @param tcp_header skeleton of the TCP header, NULL for UDP
- * @param src_address source address to use (IP and port)
- * @param dst_address destination address to use (IP and port)
- * @param pkt4 where to write the assembled packet; must
- *        contain enough space for the IP header, UDP/TCP header
- *        AND the payload
- */
-static void
-prepare_ipv4_packet (const void *payload,
-                     size_t payload_length,
-                     int protocol,
-                     const struct GNUNET_TUN_TcpHeader *tcp_header,
-                     const struct SocketAddress *src_address,
-                     const struct SocketAddress *dst_address,
-                     struct GNUNET_TUN_IPv4Header *pkt4)
-{
-  size_t len;
-
-  len = payload_length;
-  switch (protocol)
-  {
-  case IPPROTO_UDP:
-    len += sizeof (struct GNUNET_TUN_UdpHeader);
-    break;
-  case IPPROTO_TCP:
-    len += sizeof (struct GNUNET_TUN_TcpHeader);
-    GNUNET_assert (NULL != tcp_header);
-    break;
-  default:
-    GNUNET_break (0);
-    return;
-  }
-  if (len + sizeof (struct GNUNET_TUN_IPv4Header) > UINT16_MAX)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  GNUNET_TUN_initialize_ipv4_header (pkt4,
-                                     protocol,
-                                     len,
-                                     &src_address->address.ipv4,
-                                     &dst_address->address.ipv4);
-  switch (protocol)
-  {
-  case IPPROTO_UDP:
-    {
-      struct GNUNET_TUN_UdpHeader *pkt4_udp = (struct GNUNET_TUN_UdpHeader *) &pkt4[1];
-
-      pkt4_udp->source_port = htons (src_address->port);
-      pkt4_udp->destination_port = htons (dst_address->port);
-      pkt4_udp->len = htons ((uint16_t) payload_length);
-      GNUNET_TUN_calculate_udp4_checksum (pkt4,
-                                          pkt4_udp,
-                                          payload,
-                                          payload_length);
-      GNUNET_memcpy (&pkt4_udp[1],
-                     payload,
-                     payload_length);
-    }
-    break;
-  case IPPROTO_TCP:
-    {
-      struct GNUNET_TUN_TcpHeader *pkt4_tcp = (struct GNUNET_TUN_TcpHeader *) &pkt4[1];
-
-      *pkt4_tcp = *tcp_header;
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Sending TCP packet from port %u to port %u\n",
-                  src_address->port,
-                  dst_address->port);
-      pkt4_tcp->source_port = htons (src_address->port);
-      pkt4_tcp->destination_port = htons (dst_address->port);
-      GNUNET_TUN_calculate_tcp4_checksum (pkt4,
-                                          pkt4_tcp,
-                                          payload,
-                                          payload_length);
-      GNUNET_memcpy (&pkt4_tcp[1],
-                     payload,
-                     payload_length);
-    }
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-}
-
-
-/**
- * Prepare an IPv6 packet for transmission via the TUN interface.
- * Initializes the IP header and calculates checksums (IP+UDP/TCP).
- * For UDP, the UDP header will be fully created, whereas for TCP
- * only the ports and checksum will be filled in.  So for TCP,
- * a skeleton TCP header must be part of the provided payload.
- *
- * @param payload payload of the packet (starting with UDP payload or
- *                TCP header, depending on protocol)
- * @param payload_length number of bytes in @a payload
- * @param protocol IPPROTO_UDP or IPPROTO_TCP
- * @param tcp_header skeleton TCP header data to send, NULL for UDP
- * @param src_address source address to use (IP and port)
- * @param dst_address destination address to use (IP and port)
- * @param pkt6 where to write the assembled packet; must
- *        contain enough space for the IP header, UDP/TCP header
- *        AND the payload
- */
-static void
-prepare_ipv6_packet (const void *payload,
-                     size_t payload_length,
-                     int protocol,
-                     const struct GNUNET_TUN_TcpHeader *tcp_header,
-                     const struct SocketAddress *src_address,
-                     const struct SocketAddress *dst_address,
-                     struct GNUNET_TUN_IPv6Header *pkt6)
-{
-  size_t len;
-
-  len = payload_length;
-  switch (protocol)
-  {
-  case IPPROTO_UDP:
-    len += sizeof (struct GNUNET_TUN_UdpHeader);
-    break;
-  case IPPROTO_TCP:
-    len += sizeof (struct GNUNET_TUN_TcpHeader);
-    break;
-  default:
-    GNUNET_break (0);
-    return;
-  }
-  if (len > UINT16_MAX)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  GNUNET_TUN_initialize_ipv6_header (pkt6,
-                                     protocol,
-                                     len,
-                                     &src_address->address.ipv6,
-                                     &dst_address->address.ipv6);
-
-  switch (protocol)
-  {
-  case IPPROTO_UDP:
-    {
-      struct GNUNET_TUN_UdpHeader *pkt6_udp = (struct GNUNET_TUN_UdpHeader *) &pkt6[1];
-
-      pkt6_udp->source_port = htons (src_address->port);
-      pkt6_udp->destination_port = htons (dst_address->port);
-      pkt6_udp->len = htons ((uint16_t) payload_length);
-      GNUNET_TUN_calculate_udp6_checksum (pkt6,
-                                          pkt6_udp,
-                                          payload,
-                                          payload_length);
-      GNUNET_memcpy (&pkt6_udp[1],
-                     payload,
-                     payload_length);
-    }
-    break;
-  case IPPROTO_TCP:
-    {
-      struct GNUNET_TUN_TcpHeader *pkt6_tcp = (struct GNUNET_TUN_TcpHeader *) &pkt6[1];
-
-      /* GNUNET_memcpy first here as some TCP header fields are initialized this way! */
-      *pkt6_tcp = *tcp_header;
-      pkt6_tcp->source_port = htons (src_address->port);
-      pkt6_tcp->destination_port = htons (dst_address->port);
-      GNUNET_TUN_calculate_tcp6_checksum (pkt6,
-                                          pkt6_tcp,
-                                          payload,
-                                          payload_length);
-      GNUNET_memcpy (&pkt6_tcp[1],
-                     payload,
-                     payload_length);
-    }
-    break;
-  default:
-    GNUNET_assert (0);
-    break;
-  }
-}
-
-
-/**
- * Send a TCP packet via the TUN interface.
+ * Send a UDP packet via the TUN interface.
  *
- * @param destination_address IP and port to use for the TCP packet's destination
- * @param source_address IP and port to use for the TCP packet's source
- * @param tcp_header header template to use
- * @param payload payload of the TCP packet
- * @param payload_length number of bytes in @a payload
+ * @param destination_address IP and port to use for the UDP packet's destination
+ * @param source_address IP and port to use for the UDP packet's source
+ * @param payload payload of the UDP packet (does NOT include UDP header)
+ * @param payload_length number of bytes of data in @a payload
  */
 static void
-send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
+send_udp_packet_via_tun (const struct SocketAddress *destination_address,
                          const struct SocketAddress *source_address,
-                         const struct GNUNET_TUN_TcpHeader *tcp_header,
-                         const void *payload,
-                         size_t payload_length)
+                         const void *payload, size_t payload_length)
 {
   size_t len;
 
   GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# TCP packets sent via TUN"),
-                            1,
-                            GNUNET_NO);
+                            gettext_noop ("# UDP packets sent via TUN"),
+                            1, GNUNET_NO);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending packet with %u bytes TCP payload via TUN\n",
+              "Sending packet with %u bytes UDP payload via TUN\n",
               (unsigned int) payload_length);
   len = sizeof (struct GNUNET_MessageHeader) + sizeof (struct GNUNET_TUN_Layer2PacketHeader);
   switch (source_address->af)
@@ -1868,9 +1356,9 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
     GNUNET_break (0);
     return;
   }
-  len += sizeof (struct GNUNET_TUN_TcpHeader);
+  len += sizeof (struct GNUNET_TUN_UdpHeader);
   len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1880,7 +1368,7 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
     struct GNUNET_MessageHeader *hdr;
     struct GNUNET_TUN_Layer2PacketHeader *tun;
 
-    hdr = (struct GNUNET_MessageHeader *) buf;
+    hdr= (struct GNUNET_MessageHeader *) buf;
     hdr->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
     hdr->size = htons (len);
     tun = (struct GNUNET_TUN_Layer2PacketHeader*) &hdr[1];
@@ -1889,14 +1377,13 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
     {
     case AF_INET:
       {
-        struct GNUNET_TUN_IPv4Header *ipv4
-          = (struct GNUNET_TUN_IPv4Header*) &tun[1];
+        struct GNUNET_TUN_IPv4Header * ipv4 = (struct GNUNET_TUN_IPv4Header*) &tun[1];
 
         tun->proto = htons (ETH_P_IPV4);
         prepare_ipv4_packet (payload,
                              payload_length,
-                             IPPROTO_TCP,
-                             tcp_header,
+                             IPPROTO_UDP,
+                             NULL,
                              source_address,
                              destination_address,
                              ipv4);
@@ -1904,14 +1391,13 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
       break;
     case AF_INET6:
       {
-        struct GNUNET_TUN_IPv6Header *ipv6
-          = (struct GNUNET_TUN_IPv6Header*) &tun[1];
+        struct GNUNET_TUN_IPv6Header * ipv6 = (struct GNUNET_TUN_IPv6Header*) &tun[1];
 
         tun->proto = htons (ETH_P_IPV6);
         prepare_ipv6_packet (payload,
                              payload_length,
-                             IPPROTO_TCP,
-                             tcp_header,
+                             IPPROTO_UDP,
+                             NULL,
                              source_address,
                              destination_address,
                              ipv6);
@@ -1925,48 +1411,202 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
       (void) GNUNET_HELPER_send (helper_handle,
                                  (const struct GNUNET_MessageHeader*) buf,
                                  GNUNET_YES,
-                                 NULL,
-                                 NULL);
+                                 NULL, NULL);
   }
 }
 
 
 /**
- * Process a request via cadet to send a request to a TCP service
- * offered by this system.
+ * Check a request to forward UDP data to the Internet via this peer.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct ChannelState *`
- * @param message the actual message
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_tcp_service (void *cls,
-                     struct GNUNET_CADET_Channel *channel,
-                     void **channel_ctx,
-                     const struct GNUNET_MessageHeader *message)
+check_udp_remote (void *cls,
+                  const struct GNUNET_EXIT_UdpInternetMessage *msg)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_TcpServiceStartMessage *start;
-  uint16_t pkt_len = ntohs (message->size);
+  struct ChannelState *state = cls;
 
-  if (NULL == state)
+  if (GNUNET_YES == state->is_dns)
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
-  if (GNUNET_YES == state->is_dns)
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request to forward UDP data to the Internet via this peer.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ */
+static void
+handle_udp_remote (void *cls,
+                   const struct GNUNET_EXIT_UdpInternetMessage *msg)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (msg->header.size) - sizeof (struct GNUNET_EXIT_UdpInternetMessage);
+  const struct in_addr *v4;
+  const struct in6_addr *v6;
+  const void *payload;
+  int af;
+
+  if (GNUNET_SYSERR == state->is_dns)
+  {
+    /* channel is UDP/TCP from now on */
+    state->is_dns = GNUNET_NO;
+  }
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Bytes received from CADET"),
+                            pkt_len, GNUNET_NO);
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# UDP IP-exit requests received via cadet"),
+                            1, GNUNET_NO);
+  af = (int) ntohl (msg->af);
+  state->specifics.tcp_udp.ri.remote_address.af = af;
+  switch (af)
   {
+  case AF_INET:
+    if (pkt_len < sizeof (struct in_addr))
+      {
+      GNUNET_break_op (0);
+      return;
+    }
+    if (! ipv4_exit)
+    {
+      GNUNET_break_op (0);
+      return;
+    }
+    v4 = (const struct in_addr*) &msg[1];
+    payload = &v4[1];
+    pkt_len -= sizeof (struct in_addr);
+    state->specifics.tcp_udp.ri.remote_address.address.ipv4 = *v4;
+    break;
+  case AF_INET6:
+    if (pkt_len < sizeof (struct in6_addr))
+    {
+      GNUNET_break_op (0);
+      return;
+    }
+    if (! ipv6_exit)
+    {
+      GNUNET_break_op (0);
+      return;
+    }
+    v6 = (const struct in6_addr*) &msg[1];
+    payload = &v6[1];
+    pkt_len -= sizeof (struct in6_addr);
+    state->specifics.tcp_udp.ri.remote_address.address.ipv6 = *v6;
+    break;
+  default:
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    return;
+  }
+  {
+    char buf[INET6_ADDRSTRLEN];
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received data from %s for forwarding to UDP %s:%u\n",
+                GNUNET_i2s (&state->peer),
+                inet_ntop (af,
+                           &state->specifics.tcp_udp.ri.remote_address.address,
+                           buf, sizeof (buf)),
+                (unsigned int) ntohs (msg->destination_port));
   }
+  state->specifics.tcp_udp.ri.remote_address.proto = IPPROTO_UDP;
+  state->specifics.tcp_udp.ri.remote_address.port = msg->destination_port;
+  if (NULL == state->specifics.tcp_udp.heap_node)
+    setup_state_record (state);
+  if (0 != ntohs (msg->source_port))
+    state->specifics.tcp_udp.ri.local_address.port = msg->source_port;
+  send_udp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
+                           &state->specifics.tcp_udp.ri.local_address,
+                           payload,
+                           pkt_len);
+  GNUNET_CADET_receive_done (state->channel);
+}
+
+
+/**
+ * Check a request via cadet to send a request to a UDP service
+ * offered by this system.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_udp_service (void *cls,
+                   const struct GNUNET_EXIT_UdpServiceMessage *msg)
+{
+  struct ChannelState *state = cls;
+
   if (NULL == state->specifics.tcp_udp.serv)
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request via cadet to send a request to a UDP service
+ * offered by this system.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ */
+static void
+handle_udp_service (void *cls,
+                    const struct GNUNET_EXIT_UdpServiceMessage *msg)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (msg->header.size) - sizeof (struct GNUNET_EXIT_UdpServiceMessage);
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Bytes received from CADET"),
+                            pkt_len, GNUNET_NO);
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# UDP service requests received via cadet"),
+                            1, GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received data from %s for forwarding to UDP service %s on port %u\n",
+       GNUNET_i2s (&state->peer),
+       GNUNET_h2s (&state->specifics.tcp_udp.serv->descriptor),
+       (unsigned int) ntohs (msg->destination_port));
+  setup_state_record (state);
+  if (0 != ntohs (msg->source_port))
+    state->specifics.tcp_udp.ri.local_address.port = msg->source_port;
+  send_udp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
+                           &state->specifics.tcp_udp.ri.local_address,
+                           &msg[1],
+                           pkt_len);
+  GNUNET_CADET_receive_done (state->channel);
+}
+
+
+/**
+ * Process a request via cadet to send a request to a TCP service
+ * offered by this system.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param start the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static void
+handle_tcp_service (void *cls,
+                    const struct GNUNET_EXIT_TcpServiceStartMessage *start)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (start->header.size) - sizeof (struct GNUNET_EXIT_TcpServiceStartMessage);
+
   if (GNUNET_SYSERR == state->is_dns)
   {
     /* channel is UDP/TCP from now on */
@@ -1980,24 +1620,6 @@ receive_tcp_service (void *cls,
                             gettext_noop ("# Bytes received from CADET"),
                             pkt_len,
                             GNUNET_NO);
-  /* check that we got at least a valid header */
-  if (pkt_len < sizeof (struct GNUNET_EXIT_TcpServiceStartMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  start = (const struct GNUNET_EXIT_TcpServiceStartMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_TcpServiceStartMessage);
-  if (NULL != state->specifics.tcp_udp.heap_node)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (start->tcp_header.off * 4 < sizeof (struct GNUNET_TUN_TcpHeader))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
   GNUNET_break_op (ntohl (start->reserved) == 0);
   /* setup fresh connection */
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -2011,34 +1633,23 @@ receive_tcp_service (void *cls,
                            &start->tcp_header,
                            &start[1],
                            pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+  GNUNET_CADET_receive_done (state->channel);
 }
 
 
 /**
- * Process a request to forward TCP data to the Internet via this peer.
+ * Check a request to forward TCP data to the Internet via this peer.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct ChannelState *`
- * @param message the actual message
+ * @param cls our `struct ChannelState *`
+ * @param start the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_tcp_remote (void *cls GNUNET_UNUSED,
-                    struct GNUNET_CADET_Channel *channel,
-                    void **channel_ctx GNUNET_UNUSED,
-                    const struct GNUNET_MessageHeader *message)
+check_tcp_remote (void *cls,
+                    const struct GNUNET_EXIT_TcpInternetStartMessage *start)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_TcpInternetStartMessage *start;
-  uint16_t pkt_len = ntohs (message->size);
-  const struct in_addr *v4;
-  const struct in6_addr *v6;
-  const void *payload;
-  int af;
+  struct ChannelState *state = cls;
 
   if (NULL == state)
   {
@@ -2050,24 +1661,6 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
-  if (GNUNET_SYSERR == state->is_dns)
-  {
-    /* channel is UDP/TCP from now on */
-    state->is_dns = GNUNET_NO;
-  }
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes received from CADET"),
-                            pkt_len, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# TCP IP-exit creation requests received via cadet"),
-                            1, GNUNET_NO);
-  if (pkt_len < sizeof (struct GNUNET_EXIT_TcpInternetStartMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  start = (const struct GNUNET_EXIT_TcpInternetStartMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_TcpInternetStartMessage);
   if ( (NULL != state->specifics.tcp_udp.serv) ||
        (NULL != state->specifics.tcp_udp.heap_node) )
   {
@@ -2079,6 +1672,38 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request to forward TCP data to the Internet via this peer.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param start the actual message
+ */
+static void
+handle_tcp_remote (void *cls,
+                   const struct GNUNET_EXIT_TcpInternetStartMessage *start)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (start->header.size) - sizeof (struct GNUNET_EXIT_TcpInternetStartMessage);
+  const struct in_addr *v4;
+  const struct in6_addr *v6;
+  const void *payload;
+  int af;
+
+  if (GNUNET_SYSERR == state->is_dns)
+  {
+    /* channel is UDP/TCP from now on */
+    state->is_dns = GNUNET_NO;
+  }
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Bytes received from CADET"),
+                            pkt_len, GNUNET_NO);
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# TCP IP-exit creation requests received via cadet"),
+                            1, GNUNET_NO);
   af = (int) ntohl (start->af);
   state->specifics.tcp_udp.ri.remote_address.af = af;
   switch (af)
@@ -2087,12 +1712,12 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
     if (pkt_len < sizeof (struct in_addr))
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     if (! ipv4_exit)
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     v4 = (const struct in_addr*) &start[1];
     payload = &v4[1];
@@ -2103,12 +1728,12 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
     if (pkt_len < sizeof (struct in6_addr))
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     if (! ipv6_exit)
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     v6 = (const struct in6_addr*) &start[1];
     payload = &v6[1];
@@ -2117,7 +1742,7 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
     break;
   default:
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    return;
   }
   {
     char buf[INET6_ADDRSTRLEN];
@@ -2135,46 +1760,27 @@ receive_tcp_remote (void *cls GNUNET_UNUSED,
   send_tcp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
                            &state->specifics.tcp_udp.ri.local_address,
                            &start->tcp_header,
-                           payload, pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+                           payload,
+                           pkt_len);
+  GNUNET_CADET_receive_done (state->channel);
 }
 
 
 /**
- * Process a request to forward TCP data on an established
+ * Check a request to forward TCP data on an established
  * connection via this peer.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct ChannelState *`
+ * @param cls our `struct ChannelState *`
  * @param message the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_tcp_data (void *cls GNUNET_UNUSED,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx GNUNET_UNUSED,
-                  const struct GNUNET_MessageHeader *message)
+check_tcp_data (void *cls,
+                const struct GNUNET_EXIT_TcpDataMessage *data)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_TcpDataMessage *data;
-  uint16_t pkt_len = ntohs (message->size);
+  struct ChannelState *state = cls;
 
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes received from CADET"),
-                            pkt_len, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# TCP data requests received via cadet"),
-                            1, GNUNET_NO);
-  if (pkt_len < sizeof (struct GNUNET_EXIT_TcpDataMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  data = (const struct GNUNET_EXIT_TcpDataMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_TcpDataMessage);
   if ( (NULL == state) ||
        (NULL == state->specifics.tcp_udp.heap_node) )
   {
@@ -2195,6 +1801,30 @@ receive_tcp_data (void *cls GNUNET_UNUSED,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request to forward TCP data on an established
+ * connection via this peer.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param message the actual message
+ */
+static void
+handle_tcp_data (void *cls,
+                 const struct GNUNET_EXIT_TcpDataMessage *data)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (data->header.size) - sizeof (struct GNUNET_EXIT_TcpDataMessage);
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Bytes received from CADET"),
+                            pkt_len, GNUNET_NO);
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# TCP data requests received via cadet"),
+                            1, GNUNET_NO);
   if (GNUNET_SYSERR == state->is_dns)
   {
     /* channel is UDP/TCP from now on */
@@ -2218,110 +1848,7 @@ receive_tcp_data (void *cls GNUNET_UNUSED,
                            &state->specifics.tcp_udp.ri.local_address,
                            &data->tcp_header,
                            &data[1], pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
-}
-
-
-/**
- * Send an ICMP packet via the TUN interface.
- *
- * @param destination_address IP to use for the ICMP packet's destination
- * @param source_address IP to use for the ICMP packet's source
- * @param icmp_header ICMP header to send
- * @param payload payload of the ICMP packet (does NOT include ICMP header)
- * @param payload_length number of bytes of data in @a payload
- */
-static void
-send_icmp_packet_via_tun (const struct SocketAddress *destination_address,
-                          const struct SocketAddress *source_address,
-                          const struct GNUNET_TUN_IcmpHeader *icmp_header,
-                          const void *payload, size_t payload_length)
-{
-  size_t len;
-  struct GNUNET_TUN_IcmpHeader *icmp;
-
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# ICMP packets sent via TUN"),
-                            1, GNUNET_NO);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending packet with %u bytes ICMP payload via TUN\n",
-              (unsigned int) payload_length);
-  len = sizeof (struct GNUNET_MessageHeader) + sizeof (struct GNUNET_TUN_Layer2PacketHeader);
-  switch (destination_address->af)
-  {
-  case AF_INET:
-    len += sizeof (struct GNUNET_TUN_IPv4Header);
-    break;
-  case AF_INET6:
-    len += sizeof (struct GNUNET_TUN_IPv6Header);
-    break;
-  default:
-    GNUNET_break (0);
-    return;
-  }
-  len += sizeof (struct GNUNET_TUN_IcmpHeader);
-  len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  {
-    char buf[len] GNUNET_ALIGN;
-    struct GNUNET_MessageHeader *hdr;
-    struct GNUNET_TUN_Layer2PacketHeader *tun;
-
-    hdr= (struct GNUNET_MessageHeader *) buf;
-    hdr->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-    hdr->size = htons (len);
-    tun = (struct GNUNET_TUN_Layer2PacketHeader*) &hdr[1];
-    tun->flags = htons (0);
-    switch (source_address->af)
-    {
-    case AF_INET:
-      {
-        struct GNUNET_TUN_IPv4Header * ipv4 = (struct GNUNET_TUN_IPv4Header*) &tun[1];
-
-        tun->proto = htons (ETH_P_IPV4);
-        GNUNET_TUN_initialize_ipv4_header (ipv4,
-                                           IPPROTO_ICMP,
-                                           sizeof (struct GNUNET_TUN_IcmpHeader) + payload_length,
-                                           &source_address->address.ipv4,
-                                           &destination_address->address.ipv4);
-        icmp = (struct GNUNET_TUN_IcmpHeader*) &ipv4[1];
-      }
-      break;
-    case AF_INET6:
-      {
-        struct GNUNET_TUN_IPv6Header * ipv6 = (struct GNUNET_TUN_IPv6Header*) &tun[1];
-
-        tun->proto = htons (ETH_P_IPV6);
-        GNUNET_TUN_initialize_ipv6_header (ipv6,
-                                           IPPROTO_ICMPV6,
-                                           sizeof (struct GNUNET_TUN_IcmpHeader) + payload_length,
-                                           &source_address->address.ipv6,
-                                           &destination_address->address.ipv6);
-        icmp = (struct GNUNET_TUN_IcmpHeader*) &ipv6[1];
-      }
-      break;
-    default:
-      GNUNET_assert (0);
-      break;
-    }
-    *icmp = *icmp_header;
-    GNUNET_memcpy (&icmp[1],
-            payload,
-            payload_length);
-    GNUNET_TUN_calculate_icmp_checksum (icmp,
-                                        payload,
-                                        payload_length);
-    if (NULL != helper_handle)
-      (void) GNUNET_HELPER_send (helper_handle,
-                                 (const struct GNUNET_MessageHeader*) buf,
-                                 GNUNET_YES,
-                                 NULL, NULL);
-  }
+  GNUNET_CADET_receive_done (state->channel);
 }
 
 
@@ -2378,35 +1905,46 @@ make_up_icmpv6_payload (struct ChannelState *state,
 
 
 /**
- * Process a request to forward ICMP data to the Internet via this peer.
+ * Check a request to forward ICMP data to the Internet via this peer.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_icmp_remote (void *cls,
-                     struct GNUNET_CADET_Channel *channel,
-                     void **channel_ctx,
-                     const struct GNUNET_MessageHeader *message)
+check_icmp_remote (void *cls,
+                   const struct GNUNET_EXIT_IcmpInternetMessage *msg)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_IcmpInternetMessage *msg;
-  uint16_t pkt_len = ntohs (message->size);
-  const struct in_addr *v4;
-  const struct in6_addr *v6;
-  const void *payload;
-  char buf[sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
-  int af;
+  struct ChannelState *state = cls;
 
   if (GNUNET_YES == state->is_dns)
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request to forward ICMP data to the Internet via this peer.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ */
+static void
+handle_icmp_remote (void *cls,
+                    const struct GNUNET_EXIT_IcmpInternetMessage *msg)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (msg->header.size) - sizeof (struct GNUNET_EXIT_IcmpInternetMessage);
+  const struct in_addr *v4;
+  const struct in6_addr *v6;
+  const void *payload;
+  char buf[sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
+  int af;
+
   if (GNUNET_SYSERR == state->is_dns)
   {
     /* channel is UDP/TCP from now on */
@@ -2418,13 +1956,6 @@ receive_icmp_remote (void *cls,
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# ICMP IP-exit requests received via cadet"),
                             1, GNUNET_NO);
-  if (pkt_len < sizeof (struct GNUNET_EXIT_IcmpInternetMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct GNUNET_EXIT_IcmpInternetMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_IcmpInternetMessage);
 
   af = (int) ntohl (msg->af);
   if ( (NULL != state->specifics.tcp_udp.heap_node) &&
@@ -2432,7 +1963,7 @@ receive_icmp_remote (void *cls,
   {
     /* other peer switched AF on this channel; not allowed */
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    return;
   }
 
   switch (af)
@@ -2441,12 +1972,12 @@ receive_icmp_remote (void *cls,
     if (pkt_len < sizeof (struct in_addr))
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     if (! ipv4_exit)
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     v4 = (const struct in_addr*) &msg[1];
     payload = &v4[1];
@@ -2471,7 +2002,7 @@ receive_icmp_remote (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       /* make up payload */
       {
@@ -2491,7 +2022,7 @@ receive_icmp_remote (void *cls,
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
                                 1, GNUNET_NO);
-      return GNUNET_SYSERR;
+      return;
     }
     /* end AF_INET */
     break;
@@ -2499,12 +2030,12 @@ receive_icmp_remote (void *cls,
     if (pkt_len < sizeof (struct in6_addr))
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     if (! ipv6_exit)
     {
       GNUNET_break_op (0);
-      return GNUNET_SYSERR;
+      return;
     }
     v6 = (const struct in6_addr*) &msg[1];
     payload = &v6[1];
@@ -2530,7 +2061,7 @@ receive_icmp_remote (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       /* make up payload */
       {
@@ -2550,14 +2081,14 @@ receive_icmp_remote (void *cls,
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
                                 1, GNUNET_NO);
-      return GNUNET_SYSERR;
+      return;
     }
     /* end AF_INET6 */
     break;
   default:
     /* bad AF */
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    return;
   }
 
   {
@@ -2573,8 +2104,7 @@ receive_icmp_remote (void *cls,
                             &state->specifics.tcp_udp.ri.local_address,
                             &msg->icmp_header,
                             payload, pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+  GNUNET_CADET_receive_done (state->channel);
 }
 
 
@@ -2629,28 +2159,19 @@ make_up_icmp_service_payload (struct ChannelState *state,
 
 
 /**
- * Process a request via cadet to send ICMP data to a service
+ * Check a request via cadet to send ICMP data to a service
  * offered by this system.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_icmp_service (void *cls,
-                      struct GNUNET_CADET_Channel *channel,
-                      void **channel_ctx,
-                      const struct GNUNET_MessageHeader *message)
+check_icmp_service (void *cls,
+                    const struct GNUNET_EXIT_IcmpServiceMessage *msg)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_IcmpServiceMessage *msg;
-  uint16_t pkt_len = ntohs (message->size);
-  struct GNUNET_TUN_IcmpHeader icmp;
-  char buf[sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
-  const void *payload;
+  struct ChannelState *state = cls;
 
   if (GNUNET_YES == state->is_dns)
   {
@@ -2662,6 +2183,27 @@ receive_icmp_service (void *cls,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a request via cadet to send ICMP data to a service
+ * offered by this system.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param msg the actual message
+ */
+static void
+handle_icmp_service (void *cls,
+                     const struct GNUNET_EXIT_IcmpServiceMessage *msg)
+{
+  struct ChannelState *state = cls;
+  uint16_t pkt_len = ntohs (msg->header.size) - sizeof (struct GNUNET_EXIT_IcmpServiceMessage);
+  struct GNUNET_TUN_IcmpHeader icmp;
+  char buf[sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
+  const void *payload;
+
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# Bytes received from CADET"),
                             pkt_len, GNUNET_NO);
@@ -2669,13 +2211,6 @@ receive_icmp_service (void *cls,
                             gettext_noop ("# ICMP service requests received via cadet"),
                             1, GNUNET_NO);
   /* check that we got at least a valid header */
-  if (pkt_len < sizeof (struct GNUNET_EXIT_IcmpServiceMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct GNUNET_EXIT_IcmpServiceMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_IcmpServiceMessage);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received data from %s for forwarding to ICMP service %s\n",
               GNUNET_i2s (&state->peer),
@@ -2707,7 +2242,7 @@ receive_icmp_service (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2718,7 +2253,7 @@ receive_icmp_service (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2729,12 +2264,12 @@ receive_icmp_service (void *cls,
         GNUNET_STATISTICS_update (stats,
                                   gettext_noop ("# ICMPv4 packets dropped (impossible PT to v6)"),
                                   1, GNUNET_NO);
-        return GNUNET_OK;
+        return;
       }
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2744,7 +2279,7 @@ receive_icmp_service (void *cls,
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
                                 1, GNUNET_NO);
-      return GNUNET_SYSERR;
+      return;
     }
     /* end of AF_INET */
     break;
@@ -2765,7 +2300,7 @@ receive_icmp_service (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2776,7 +2311,7 @@ receive_icmp_service (void *cls,
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2788,12 +2323,12 @@ receive_icmp_service (void *cls,
         GNUNET_STATISTICS_update (stats,
                                   gettext_noop ("# ICMPv6 packets dropped (impossible PT to v4)"),
                                   1, GNUNET_NO);
-        return GNUNET_OK;
+        return;
       }
       if (0 != pkt_len)
       {
         GNUNET_break_op (0);
-        return GNUNET_SYSERR;
+        return;
       }
       payload = buf;
       pkt_len = make_up_icmp_service_payload (state, buf);
@@ -2803,283 +2338,713 @@ receive_icmp_service (void *cls,
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
                                 1, GNUNET_NO);
-      return GNUNET_SYSERR;
+      return;
     }
     /* end of AF_INET6 */
     break;
   default:
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    return;
   }
 
   send_icmp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
                             &state->specifics.tcp_udp.ri.local_address,
                             &icmp,
-                            payload, pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+                            payload,
+                            pkt_len);
+  GNUNET_CADET_receive_done (state->channel);
 }
 
 
 /**
- * Send a UDP packet via the TUN interface.
+ * Free memory associated with a service record.
  *
- * @param destination_address IP and port to use for the UDP packet's destination
- * @param source_address IP and port to use for the UDP packet's source
- * @param payload payload of the UDP packet (does NOT include UDP header)
- * @param payload_length number of bytes of data in @a payload
+ * @param cls unused
+ * @param key service descriptor
+ * @param value service record to free
+ * @return #GNUNET_OK
  */
-static void
-send_udp_packet_via_tun (const struct SocketAddress *destination_address,
-                         const struct SocketAddress *source_address,
-                         const void *payload, size_t payload_length)
+static int
+free_service_record (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     void *value)
 {
-  size_t len;
+  struct LocalService *service = value;
+
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (services,
+                                                       key,
+                                                       service));
+  GNUNET_CADET_close_port (service->port);
+  GNUNET_free_non_null (service->name);
+  GNUNET_free (service);
+  return GNUNET_OK;
+}
+
 
+/**
+ * Callback from CADET for new channels.
+ *
+ * @param cls closure
+ * @param channel new handle to the channel
+ * @param initiator peer that started the channel
+ * @return initial channel context for the channel
+ */
+static void *
+new_service_channel (void *cls,
+                     struct GNUNET_CADET_Channel *channel,
+                     const struct GNUNET_PeerIdentity *initiator)
+{
+  struct LocalService *ls = cls;
+  struct ChannelState *s = GNUNET_new (struct ChannelState);
+
+  s->peer = *initiator;
   GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# UDP packets sent via TUN"),
-                            1, GNUNET_NO);
+                            gettext_noop ("# Inbound CADET channels created"),
+                            1,
+                            GNUNET_NO);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending packet with %u bytes UDP payload via TUN\n",
-              (unsigned int) payload_length);
-  len = sizeof (struct GNUNET_MessageHeader) + sizeof (struct GNUNET_TUN_Layer2PacketHeader);
-  switch (source_address->af)
+              "Received inbound channel from `%s'\n",
+              GNUNET_i2s (initiator));
+  s->channel = channel;
+  s->specifics.tcp_udp.serv = ls;
+  s->specifics.tcp_udp.ri.remote_address = ls->address;
+  return s;
+}
+
+
+/**
+ * Function called by cadet whenever an inbound channel is destroyed.
+ * Should clean up any associated state.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param channel connection to the other end (henceforth invalid)
+ */
+static void
+clean_channel (void *cls,
+               const struct GNUNET_CADET_Channel *channel)
+{
+  struct ChannelState *s = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Channel destroyed\n");
+  if (GNUNET_SYSERR == s->is_dns)
   {
-  case AF_INET:
-    len += sizeof (struct GNUNET_TUN_IPv4Header);
-    break;
-  case AF_INET6:
-    len += sizeof (struct GNUNET_TUN_IPv6Header);
-    break;
-  default:
-    GNUNET_break (0);
+    GNUNET_free (s);
     return;
   }
-  len += sizeof (struct GNUNET_TUN_UdpHeader);
-  len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (GNUNET_YES == s->is_dns)
   {
-    GNUNET_break (0);
-    return;
+    if (channels[s->specifics.dns.my_id] == s)
+      channels[s->specifics.dns.my_id] = NULL;
   }
+  else
   {
-    char buf[len] GNUNET_ALIGN;
-    struct GNUNET_MessageHeader *hdr;
-    struct GNUNET_TUN_Layer2PacketHeader *tun;
-
-    hdr= (struct GNUNET_MessageHeader *) buf;
-    hdr->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-    hdr->size = htons (len);
-    tun = (struct GNUNET_TUN_Layer2PacketHeader*) &hdr[1];
-    tun->flags = htons (0);
-    switch (source_address->af)
+    if (NULL != s->specifics.tcp_udp.heap_node)
     {
-    case AF_INET:
-      {
-        struct GNUNET_TUN_IPv4Header * ipv4 = (struct GNUNET_TUN_IPv4Header*) &tun[1];
-
-        tun->proto = htons (ETH_P_IPV4);
-        prepare_ipv4_packet (payload,
-                             payload_length,
-                             IPPROTO_UDP,
-                             NULL,
-                             source_address,
-                             destination_address,
-                             ipv4);
-      }
-      break;
-    case AF_INET6:
-      {
-        struct GNUNET_TUN_IPv6Header * ipv6 = (struct GNUNET_TUN_IPv6Header*) &tun[1];
-
-        tun->proto = htons (ETH_P_IPV6);
-        prepare_ipv6_packet (payload,
-                             payload_length,
-                             IPPROTO_UDP,
-                             NULL,
-                             source_address,
-                             destination_address,
-                             ipv6);
-      }
-      break;
-    default:
-      GNUNET_assert (0);
-      break;
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (connections_map,
+                                                           &s->specifics.tcp_udp.state_key,
+                                                           s));
+      GNUNET_CONTAINER_heap_remove_node (s->specifics.tcp_udp.heap_node);
+      s->specifics.tcp_udp.heap_node = NULL;
     }
-    if (NULL != helper_handle)
-      (void) GNUNET_HELPER_send (helper_handle,
-                                 (const struct GNUNET_MessageHeader*) buf,
-                                 GNUNET_YES,
-                                 NULL, NULL);
   }
+  GNUNET_free (s);
 }
 
 
 /**
- * Process a request to forward UDP data to the Internet via this peer.
+ * Given a service descriptor and a destination port, find the
+ * respective service entry.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
+ * @param proto IPPROTO_TCP or IPPROTO_UDP
+ * @param name name of the service
+ * @param destination_port destination port
+ * @param service service information record to store (service->name will be set).
  */
-static int
-receive_udp_remote (void *cls,
-                    struct GNUNET_CADET_Channel *channel,
-                    void **channel_ctx,
-                    const struct GNUNET_MessageHeader *message)
+static void
+store_service (int proto,
+               const char *name,
+               uint16_t destination_port,
+               struct LocalService *service)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_UdpInternetMessage *msg;
-  uint16_t pkt_len = ntohs (message->size);
-  const struct in_addr *v4;
-  const struct in6_addr *v6;
-  const void *payload;
-  int af;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (icmp_service,
+                           GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_SERVICE,
+                           struct GNUNET_EXIT_IcmpServiceMessage,
+                           service),
+    GNUNET_MQ_hd_var_size (udp_service,
+                           GNUNET_MESSAGE_TYPE_VPN_UDP_TO_SERVICE,
+                           struct GNUNET_EXIT_UdpServiceMessage,
+                           service),
+    GNUNET_MQ_hd_var_size (tcp_service,
+                           GNUNET_MESSAGE_TYPE_VPN_TCP_TO_SERVICE_START,
+                           struct GNUNET_EXIT_TcpServiceStartMessage,
+                           service),
+    GNUNET_MQ_hd_var_size (tcp_data,
+                           GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_EXIT,
+                           struct GNUNET_EXIT_TcpDataMessage,
+                           service),
+    GNUNET_MQ_handler_end ()
+  };
 
-  if (GNUNET_YES == state->is_dns)
+  struct GNUNET_HashCode cadet_port;
+
+  service->name = GNUNET_strdup (name);
+  GNUNET_TUN_service_name_to_hash (name,
+                                   &service->descriptor);
+  GNUNET_TUN_compute_service_cadet_port (&service->descriptor,
+                                         destination_port,
+                                         &cadet_port);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Opening CADET port %s for SERVICE exit %s on port %u\n",
+              GNUNET_h2s (&cadet_port),
+              name,
+              (unsigned int) destination_port);
+  service->port = GNUNET_CADET_open_port (cadet_handle,
+                                          &cadet_port,
+                                          &new_service_channel,
+                                          service,
+                                          NULL,
+                                          &clean_channel,
+                                          handlers);
+  service->is_udp = (IPPROTO_UDP == proto);
+  if (GNUNET_OK !=
+      GNUNET_CONTAINER_multihashmap_put (services,
+                                         &cadet_port,
+                                         service,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    GNUNET_CADET_close_port (service->port);
+    GNUNET_free_non_null (service->name);
+    GNUNET_free (service);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("Got duplicate service records for `%s:%u'\n"),
+                name,
+                (unsigned int) destination_port);
   }
-  if (GNUNET_SYSERR == state->is_dns)
+}
+
+
+/**
+ * Send the given packet via the cadet channel.
+ *
+ * @param s channel destination
+ * @param env message to queue
+ */
+static void
+send_packet_to_cadet_channel (struct ChannelState *s,
+                              struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_assert (NULL != s);
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Messages transmitted via cadet channels"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (s->channel),
+                  env);
+}
+
+
+/**
+ * @brief Handles an ICMP packet received from the helper.
+ *
+ * @param icmp A pointer to the Packet
+ * @param pktlen number of bytes in @a icmp
+ * @param af address family (AFINET or AF_INET6)
+ * @param destination_ip destination IP-address of the IP packet (should
+ *                       be our local address)
+ * @param source_ip original source IP-address of the IP packet (should
+ *                       be the original destination address)
+ */
+static void
+icmp_from_helper (const struct GNUNET_TUN_IcmpHeader *icmp,
+                  size_t pktlen,
+                  int af,
+                  const void *destination_ip,
+                  const void *source_ip)
+{
+  struct ChannelState *state;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_EXIT_IcmpToVPNMessage *i2v;
+  const struct GNUNET_TUN_IPv4Header *ipv4;
+  const struct GNUNET_TUN_IPv6Header *ipv6;
+  const struct GNUNET_TUN_UdpHeader *udp;
+  uint16_t source_port;
+  uint16_t destination_port;
+  uint8_t protocol;
+
   {
-    /* channel is UDP/TCP from now on */
-    state->is_dns = GNUNET_NO;
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received ICMP packet going from %s to %s\n",
+                inet_ntop (af,
+                           source_ip,
+                           sbuf, sizeof (sbuf)),
+                inet_ntop (af,
+                           destination_ip,
+                           dbuf, sizeof (dbuf)));
   }
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes received from CADET"),
-                            pkt_len, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# UDP IP-exit requests received via cadet"),
-                            1, GNUNET_NO);
-  if (pkt_len < sizeof (struct GNUNET_EXIT_UdpInternetMessage))
+  if (pktlen < sizeof (struct GNUNET_TUN_IcmpHeader))
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    /* blame kernel */
+    GNUNET_break (0);
+    return;
   }
-  msg = (const struct GNUNET_EXIT_UdpInternetMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_UdpInternetMessage);
-  af = (int) ntohl (msg->af);
-  state->specifics.tcp_udp.ri.remote_address.af = af;
+
+  /* Find out if this is an ICMP packet in response to an existing
+     TCP/UDP packet and if so, figure out ports / protocol of the
+     existing session from the IP data in the ICMP payload */
+  source_port = 0;
+  destination_port = 0;
   switch (af)
   {
   case AF_INET:
-    if (pkt_len < sizeof (struct in_addr))
-    {
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-    if (! ipv4_exit)
-    {
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-    v4 = (const struct in_addr*) &msg[1];
-    payload = &v4[1];
-    pkt_len -= sizeof (struct in_addr);
-    state->specifics.tcp_udp.ri.remote_address.address.ipv4 = *v4;
+    protocol = IPPROTO_ICMP;
+    switch (icmp->type)
+      {
+      case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
+      case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
+        break;
+      case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
+      case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
+      case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
+        if (pktlen <
+            sizeof (struct GNUNET_TUN_IcmpHeader) +
+            sizeof (struct GNUNET_TUN_IPv4Header) + 8)
+        {
+          /* blame kernel */
+          GNUNET_break (0);
+          return;
+        }
+        ipv4 = (const struct GNUNET_TUN_IPv4Header *) &icmp[1];
+        protocol = ipv4->protocol;
+        /* could be TCP or UDP, but both have the ports in the right
+           place, so that doesn't matter here */
+        udp = (const struct GNUNET_TUN_UdpHeader *) &ipv4[1];
+        /* swap ports, as they are from the original message */
+        destination_port = ntohs (udp->source_port);
+        source_port = ntohs (udp->destination_port);
+        /* throw away ICMP payload, won't be useful for the other side anyway */
+        pktlen = sizeof (struct GNUNET_TUN_IcmpHeader);
+        break;
+      default:
+        GNUNET_STATISTICS_update (stats,
+                                  gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
+                                  1, GNUNET_NO);
+        return;
+      }
     break;
   case AF_INET6:
-    if (pkt_len < sizeof (struct in6_addr))
-    {
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-    if (! ipv6_exit)
-    {
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-    v6 = (const struct in6_addr*) &msg[1];
-    payload = &v6[1];
-    pkt_len -= sizeof (struct in6_addr);
-    state->specifics.tcp_udp.ri.remote_address.address.ipv6 = *v6;
+    protocol = IPPROTO_ICMPV6;
+    switch (icmp->type)
+      {
+      case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
+      case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
+      case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
+      case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
+        if (pktlen <
+            sizeof (struct GNUNET_TUN_IcmpHeader) +
+            sizeof (struct GNUNET_TUN_IPv6Header) + 8)
+        {
+          /* blame kernel */
+          GNUNET_break (0);
+          return;
+        }
+        ipv6 = (const struct GNUNET_TUN_IPv6Header *) &icmp[1];
+        protocol = ipv6->next_header;
+        /* could be TCP or UDP, but both have the ports in the right
+           place, so that doesn't matter here */
+        udp = (const struct GNUNET_TUN_UdpHeader *) &ipv6[1];
+        /* swap ports, as they are from the original message */
+        destination_port = ntohs (udp->source_port);
+        source_port = ntohs (udp->destination_port);
+        /* throw away ICMP payload, won't be useful for the other side anyway */
+        pktlen = sizeof (struct GNUNET_TUN_IcmpHeader);
+        break;
+      case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
+      case GNUNET_TUN_ICMPTYPE6_ECHO_REPLY:
+        break;
+      default:
+        GNUNET_STATISTICS_update (stats,
+                                  gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
+                                  1, GNUNET_NO);
+        return;
+      }
     break;
   default:
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    GNUNET_assert (0);
   }
+  switch (protocol)
   {
-    char buf[INET6_ADDRSTRLEN];
+  case IPPROTO_ICMP:
+    state = get_redirect_state (af,
+                                IPPROTO_ICMP,
+                                source_ip,
+                                0,
+                                destination_ip,
+                                0,
+                                NULL);
+    break;
+  case IPPROTO_ICMPV6:
+    state = get_redirect_state (af,
+                                IPPROTO_ICMPV6,
+                                source_ip,
+                                0,
+                                destination_ip,
+                                0,
+                                NULL);
+    break;
+  case IPPROTO_UDP:
+    state = get_redirect_state (af,
+                                IPPROTO_UDP,
+                                source_ip,
+                                source_port,
+                                destination_ip,
+                                destination_port,
+                                NULL);
+    break;
+  case IPPROTO_TCP:
+    state = get_redirect_state (af,
+                                IPPROTO_TCP,
+                                source_ip,
+                                source_port,
+                                destination_ip,
+                                destination_port,
+                                NULL);
+    break;
+  default:
+    GNUNET_STATISTICS_update (stats,
+                              gettext_noop ("# ICMP packets dropped (not allowed)"),
+                              1,
+                              GNUNET_NO);
+    return;
+  }
+  if (NULL == state)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("ICMP Packet dropped, have no matching connection information\n"));
+    return;
+  }
+  env = GNUNET_MQ_msg_extra (i2v,
+                             pktlen - sizeof (struct GNUNET_TUN_IcmpHeader),
+                             GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_VPN);
+  i2v->af = htonl (af);
+  GNUNET_memcpy (&i2v->icmp_header,
+                 icmp,
+                 pktlen);
+  send_packet_to_cadet_channel (state,
+                                env);
+}
+
+
+/**
+ * @brief Handles an UDP packet received from the helper.
+ *
+ * @param udp A pointer to the Packet
+ * @param pktlen number of bytes in 'udp'
+ * @param af address family (AFINET or AF_INET6)
+ * @param destination_ip destination IP-address of the IP packet (should
+ *                       be our local address)
+ * @param source_ip original source IP-address of the IP packet (should
+ *                       be the original destination address)
+ */
+static void
+udp_from_helper (const struct GNUNET_TUN_UdpHeader *udp,
+                 size_t pktlen,
+                 int af,
+                 const void *destination_ip,
+                 const void *source_ip)
+{
+  struct ChannelState *state;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_EXIT_UdpReplyMessage *urm;
+
+  {
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received data from %s for forwarding to UDP %s:%u\n",
-                GNUNET_i2s (&state->peer),
+                "Received UDP packet going from %s:%u to %s:%u\n",
                 inet_ntop (af,
-                           &state->specifics.tcp_udp.ri.remote_address.address,
-                           buf, sizeof (buf)),
-                (unsigned int) ntohs (msg->destination_port));
+                           source_ip,
+                           sbuf, sizeof (sbuf)),
+                (unsigned int) ntohs (udp->source_port),
+                inet_ntop (af,
+                           destination_ip,
+                           dbuf, sizeof (dbuf)),
+                (unsigned int) ntohs (udp->destination_port));
   }
-  state->specifics.tcp_udp.ri.remote_address.proto = IPPROTO_UDP;
-  state->specifics.tcp_udp.ri.remote_address.port = msg->destination_port;
-  if (NULL == state->specifics.tcp_udp.heap_node)
-    setup_state_record (state);
-  if (0 != ntohs (msg->source_port))
-    state->specifics.tcp_udp.ri.local_address.port = msg->source_port;
-  send_udp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
-                           &state->specifics.tcp_udp.ri.local_address,
-                           payload, pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+  if (pktlen < sizeof (struct GNUNET_TUN_UdpHeader))
+  {
+    /* blame kernel */
+    GNUNET_break (0);
+    return;
+  }
+  if (pktlen != ntohs (udp->len))
+  {
+    /* blame kernel */
+    GNUNET_break (0);
+    return;
+  }
+  state = get_redirect_state (af,
+                              IPPROTO_UDP,
+                              source_ip,
+                              ntohs (udp->source_port),
+                              destination_ip,
+                              ntohs (udp->destination_port),
+                              NULL);
+  if (NULL == state)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("UDP Packet dropped, have no matching connection information\n"));
+    return;
+  }
+  env = GNUNET_MQ_msg_extra (urm,
+                             pktlen - sizeof (struct GNUNET_TUN_UdpHeader),
+                             GNUNET_MESSAGE_TYPE_VPN_UDP_REPLY);
+  urm->source_port = htons (0);
+  urm->destination_port = htons (0);
+  GNUNET_memcpy (&urm[1],
+                 &udp[1],
+                 pktlen - sizeof (struct GNUNET_TUN_UdpHeader));
+  send_packet_to_cadet_channel (state,
+                                env);
 }
 
 
 /**
- * Process a request via cadet to send a request to a UDP service
- * offered by this system.
+ * @brief Handles a TCP packet received from the helper.
  *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
+ * @param tcp A pointer to the Packet
+ * @param pktlen the length of the packet, including its TCP header
+ * @param af address family (AFINET or AF_INET6)
+ * @param destination_ip destination IP-address of the IP packet (should
+ *                       be our local address)
+ * @param source_ip original source IP-address of the IP packet (should
+ *                       be the original destination address)
  */
-static int
-receive_udp_service (void *cls,
-                     struct GNUNET_CADET_Channel *channel,
-                     void **channel_ctx,
-                     const struct GNUNET_MessageHeader *message)
+static void
+tcp_from_helper (const struct GNUNET_TUN_TcpHeader *tcp,
+                 size_t pktlen,
+                 int af,
+                 const void *destination_ip,
+                 const void *source_ip)
 {
-  struct ChannelState *state = *channel_ctx;
-  const struct GNUNET_EXIT_UdpServiceMessage *msg;
-  uint16_t pkt_len = ntohs (message->size);
+  struct ChannelState *state;
+  char buf[pktlen] GNUNET_ALIGN;
+  struct GNUNET_TUN_TcpHeader *mtcp;
+  struct GNUNET_EXIT_TcpDataMessage *tdm;
+  struct GNUNET_MQ_Envelope *env;
+  size_t mlen;
 
-  if (NULL == state->specifics.tcp_udp.serv)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received TCP packet with %u bytes going from %s:%u to %s:%u\n",
+                (unsigned int) (pktlen - sizeof (struct GNUNET_TUN_TcpHeader)),
+                inet_ntop (af,
+                           source_ip,
+                           sbuf, sizeof (sbuf)),
+                (unsigned int) ntohs (tcp->source_port),
+                inet_ntop (af,
+                           destination_ip,
+                           dbuf, sizeof (dbuf)),
+                (unsigned int) ntohs (tcp->destination_port));
   }
+  if (pktlen < sizeof (struct GNUNET_TUN_TcpHeader))
+  {
+    /* blame kernel */
+    GNUNET_break (0);
+    return;
+  }
+  state = get_redirect_state (af,
+                              IPPROTO_TCP,
+                              source_ip,
+                              ntohs (tcp->source_port),
+                              destination_ip,
+                              ntohs (tcp->destination_port),
+                              NULL);
+  if (NULL == state)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("TCP Packet dropped, have no matching connection information\n"));
+
+    return;
+  }
+  /* mug port numbers and crc to avoid information leakage;
+     sender will need to lookup the correct values anyway */
+  GNUNET_memcpy (buf, tcp, pktlen);
+  mtcp = (struct GNUNET_TUN_TcpHeader *) buf;
+  mtcp->source_port = 0;
+  mtcp->destination_port = 0;
+  mtcp->crc = 0;
+
+  mlen = sizeof (struct GNUNET_EXIT_TcpDataMessage) + (pktlen - sizeof (struct GNUNET_TUN_TcpHeader));
+  if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  env = GNUNET_MQ_msg_extra (tdm,
+                             pktlen - sizeof (struct GNUNET_TUN_TcpHeader),
+                             GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_VPN);
+  tdm->reserved = htonl (0);
+  GNUNET_memcpy (&tdm->tcp_header,
+                 buf,
+                 pktlen);
+  send_packet_to_cadet_channel (state,
+                                env);
+}
+
+
+/**
+ * Receive packets from the helper-process
+ *
+ * @param cls unused
+ * @param message message received from helper
+ */
+static int
+message_token (void *cls GNUNET_UNUSED,
+               const struct GNUNET_MessageHeader *message)
+{
+  const struct GNUNET_TUN_Layer2PacketHeader *pkt_tun;
+  size_t size;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Got %u-byte message of type %u from gnunet-helper-exit\n",
+              ntohs (message->size),
+              ntohs (message->type));
   GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes received from CADET"),
-                            pkt_len, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# UDP service requests received via cadet"),
+                            gettext_noop ("# Packets received from TUN"),
                             1, GNUNET_NO);
-  /* check that we got at least a valid header */
-  if (pkt_len < sizeof (struct GNUNET_EXIT_UdpServiceMessage))
+  if (ntohs (message->type) != GNUNET_MESSAGE_TYPE_VPN_HELPER)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    GNUNET_break (0);
+    return GNUNET_OK;
   }
-  msg = (const struct GNUNET_EXIT_UdpServiceMessage*) message;
-  pkt_len -= sizeof (struct GNUNET_EXIT_UdpServiceMessage);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received data from %s for forwarding to UDP service %s on port %u\n",
-       GNUNET_i2s (&state->peer),
-       GNUNET_h2s (&state->specifics.tcp_udp.serv->descriptor),
-       (unsigned int) ntohs (msg->destination_port));
-  setup_state_record (state);
-  if (0 != ntohs (msg->source_port))
-    state->specifics.tcp_udp.ri.local_address.port = msg->source_port;
-  send_udp_packet_via_tun (&state->specifics.tcp_udp.ri.remote_address,
-                           &state->specifics.tcp_udp.ri.local_address,
-                           &msg[1],
-                           pkt_len);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_YES;
+  size = ntohs (message->size);
+  if (size < sizeof (struct GNUNET_TUN_Layer2PacketHeader) + sizeof (struct GNUNET_MessageHeader))
+  {
+    GNUNET_break (0);
+    return GNUNET_OK;
+  }
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# Bytes received from TUN"),
+                            size, GNUNET_NO);
+  pkt_tun = (const struct GNUNET_TUN_Layer2PacketHeader *) &message[1];
+  size -= sizeof (struct GNUNET_TUN_Layer2PacketHeader) + sizeof (struct GNUNET_MessageHeader);
+  switch (ntohs (pkt_tun->proto))
+  {
+  case ETH_P_IPV4:
+    {
+      const struct GNUNET_TUN_IPv4Header *pkt4;
+
+      if (size < sizeof (struct GNUNET_TUN_IPv4Header))
+      {
+        /* Kernel to blame? */
+        GNUNET_break (0);
+        return GNUNET_OK;
+      }
+      pkt4 = (const struct GNUNET_TUN_IPv4Header *) &pkt_tun[1];
+      if (size != ntohs (pkt4->total_length))
+      {
+        /* Kernel to blame? */
+        GNUNET_break (0);
+        return GNUNET_OK;
+      }
+      if (pkt4->header_length * 4 != sizeof (struct GNUNET_TUN_IPv4Header))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _("IPv4 packet options received.  Ignored.\n"));
+        return GNUNET_OK;
+      }
+
+      size -= sizeof (struct GNUNET_TUN_IPv4Header);
+      switch (pkt4->protocol)
+      {
+      case IPPROTO_UDP:
+        udp_from_helper ((const struct GNUNET_TUN_UdpHeader *) &pkt4[1], size,
+                         AF_INET,
+                         &pkt4->destination_address,
+                         &pkt4->source_address);
+        break;
+      case IPPROTO_TCP:
+        tcp_from_helper ((const struct GNUNET_TUN_TcpHeader *) &pkt4[1], size,
+                         AF_INET,
+                         &pkt4->destination_address,
+                         &pkt4->source_address);
+        break;
+      case IPPROTO_ICMP:
+        icmp_from_helper ((const struct GNUNET_TUN_IcmpHeader *) &pkt4[1], size,
+                          AF_INET,
+                          &pkt4->destination_address,
+                          &pkt4->source_address);
+        break;
+      default:
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _("IPv4 packet with unsupported next header %u received.  Ignored.\n"),
+                    (int) pkt4->protocol);
+        return GNUNET_OK;
+      }
+    }
+    break;
+  case ETH_P_IPV6:
+    {
+      const struct GNUNET_TUN_IPv6Header *pkt6;
+
+      if (size < sizeof (struct GNUNET_TUN_IPv6Header))
+      {
+        /* Kernel to blame? */
+        GNUNET_break (0);
+        return GNUNET_OK;
+      }
+      pkt6 = (struct GNUNET_TUN_IPv6Header *) &pkt_tun[1];
+      if (size != ntohs (pkt6->payload_length) + sizeof (struct GNUNET_TUN_IPv6Header))
+      {
+        /* Kernel to blame? */
+        GNUNET_break (0);
+        return GNUNET_OK;
+      }
+      size -= sizeof (struct GNUNET_TUN_IPv6Header);
+      switch (pkt6->next_header)
+      {
+      case IPPROTO_UDP:
+        udp_from_helper ((const struct GNUNET_TUN_UdpHeader *) &pkt6[1], size,
+                         AF_INET6,
+                         &pkt6->destination_address,
+                         &pkt6->source_address);
+        break;
+      case IPPROTO_TCP:
+        tcp_from_helper ((const struct GNUNET_TUN_TcpHeader *) &pkt6[1], size,
+                         AF_INET6,
+                         &pkt6->destination_address,
+                         &pkt6->source_address);
+        break;
+      case IPPROTO_ICMPV6:
+        icmp_from_helper ((const struct GNUNET_TUN_IcmpHeader *) &pkt6[1], size,
+                          AF_INET6,
+                          &pkt6->destination_address,
+                          &pkt6->source_address);
+        break;
+      default:
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _("IPv6 packet with unsupported next header %d received.  Ignored.\n"),
+                    pkt6->next_header);
+        return GNUNET_OK;
+      }
+    }
+    break;
+  default:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("Packet from unknown protocol %u received.  Ignored.\n"),
+                ntohs (pkt_tun->proto));
+    break;
+  }
+  return GNUNET_OK;
 }
 
 
@@ -3089,16 +3054,12 @@ receive_udp_service (void *cls,
  * @param cls closure
  * @param channel new handle to the channel
  * @param initiator peer that started the channel
- * @param port destination port
- * @param options channel options flags
  * @return initial channel context for the channel
  */
 static void *
 new_channel (void *cls,
              struct GNUNET_CADET_Channel *channel,
-             const struct GNUNET_PeerIdentity *initiator,
-             const struct GNUNET_HashCode *port,
-             enum GNUNET_CADET_ChannelOption options)
+             const struct GNUNET_PeerIdentity *initiator)
 {
   struct ChannelState *s = GNUNET_new (struct ChannelState);
 
@@ -3106,7 +3067,8 @@ new_channel (void *cls,
   s->peer = *initiator;
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# Inbound CADET channels created"),
-                            1, GNUNET_NO);
+                            1,
+                            GNUNET_NO);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received inbound channel from `%s'\n",
               GNUNET_i2s (initiator));
@@ -3116,64 +3078,6 @@ new_channel (void *cls,
 
 
 /**
- * Function called by cadet whenever an inbound channel is destroyed.
- * Should clean up any associated state.
- *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
- */
-static void
-clean_channel (void *cls,
-               const struct GNUNET_CADET_Channel *channel,
-               void *channel_ctx)
-{
-  struct ChannelState *s = channel_ctx;
-  struct ChannelMessageQueue *tnq;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Channel destroyed\n");
-  if (GNUNET_SYSERR == s->is_dns)
-  {
-    GNUNET_free (s);
-    return;
-  }
-  if (GNUNET_YES == s->is_dns)
-  {
-    if (channels[s->specifics.dns.my_id] == s)
-      channels[s->specifics.dns.my_id] = NULL;
-    GNUNET_free_non_null (s->specifics.dns.reply);
-  }
-  else
-  {
-    while (NULL != (tnq = s->specifics.tcp_udp.head))
-    {
-      GNUNET_CONTAINER_DLL_remove (s->specifics.tcp_udp.head,
-                                   s->specifics.tcp_udp.tail,
-                                   tnq);
-      GNUNET_free (tnq);
-    }
-    if (NULL != s->specifics.tcp_udp.heap_node)
-    {
-      GNUNET_assert (GNUNET_YES ==
-                     GNUNET_CONTAINER_multihashmap_remove (connections_map,
-                                                           &s->specifics.tcp_udp.state_key,
-                                                           s));
-      GNUNET_CONTAINER_heap_remove_node (s->specifics.tcp_udp.heap_node);
-      s->specifics.tcp_udp.heap_node = NULL;
-    }
-  }
-  if (NULL != s->th)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (s->th);
-    s->th = NULL;
-  }
-  GNUNET_free (s);
-}
-
-
-/**
  * Function that frees everything from a hashmap
  *
  * @param cls unused
@@ -3241,6 +3145,21 @@ cleanup (void *cls)
                                            NULL);
     GNUNET_CONTAINER_multihashmap_destroy (services);
   }
+  if (NULL != dns_port)
+  {
+    GNUNET_CADET_close_port (dns_port);
+    dns_port = NULL;
+  }
+  if (NULL != cadet_port4)
+  {
+    GNUNET_CADET_close_port (cadet_port4);
+    cadet_port4 = NULL;
+  }
+  if (NULL != cadet_port6)
+  {
+    GNUNET_CADET_close_port (cadet_port6);
+    cadet_port6 = NULL;
+  }
   if (NULL != cadet_handle)
   {
     GNUNET_CADET_disconnect (cadet_handle);
@@ -3286,7 +3205,8 @@ cleanup (void *cls)
   }
   if (NULL != stats)
   {
-    GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
+    GNUNET_STATISTICS_destroy (stats,
+                               GNUNET_NO);
     stats = NULL;
   }
   for (i=0;i<8;i++)
@@ -3614,6 +3534,13 @@ parse_ip_options ()
 static void
 advertise_dns_exit ()
 {
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (dns_request,
+                           GNUNET_MESSAGE_TYPE_VPN_DNS_TO_INTERNET,
+                           struct DnsResponseMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
   char *dns_exit;
   struct GNUNET_HashCode port;
   struct in_addr dns_exit4;
@@ -3650,10 +3577,13 @@ advertise_dns_exit ()
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Opening CADET port %s for DNS exit service\n",
               GNUNET_h2s (&port));
-  GNUNET_CADET_open_port (cadet_handle,
-                          &port,
-                          &new_channel,
-                          NULL);
+  dns_port = GNUNET_CADET_open_port (cadet_handle,
+                                     &port,
+                                     &new_channel,
+                                     NULL,
+                                     NULL,
+                                     &clean_channel,
+                                     handlers);
   /* advertise exit */
   dht = GNUNET_DHT_connect (cfg,
                             1);
@@ -3833,16 +3763,24 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg_)
 {
-  static struct GNUNET_CADET_MessageHandler handlers[] = {
-    {&receive_icmp_service, GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_SERVICE, 0},
-    {&receive_icmp_remote, GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_INTERNET, 0},
-    {&receive_udp_service, GNUNET_MESSAGE_TYPE_VPN_UDP_TO_SERVICE, 0},
-    {&receive_udp_remote, GNUNET_MESSAGE_TYPE_VPN_UDP_TO_INTERNET, 0},
-    {&receive_tcp_service, GNUNET_MESSAGE_TYPE_VPN_TCP_TO_SERVICE_START, 0},
-    {&receive_tcp_remote, GNUNET_MESSAGE_TYPE_VPN_TCP_TO_INTERNET_START, 0},
-    {&receive_tcp_data, GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_EXIT, 0},
-    {&receive_dns_request, GNUNET_MESSAGE_TYPE_VPN_DNS_TO_INTERNET, 0},
-    {NULL, 0, 0}
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (icmp_remote,
+                           GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_INTERNET,
+                           struct GNUNET_EXIT_IcmpInternetMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (udp_remote,
+                           GNUNET_MESSAGE_TYPE_VPN_UDP_TO_INTERNET,
+                           struct GNUNET_EXIT_UdpInternetMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (tcp_remote,
+                           GNUNET_MESSAGE_TYPE_VPN_TCP_TO_INTERNET_START,
+                           struct GNUNET_EXIT_TcpInternetStartMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (tcp_data,
+                           GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_EXIT,
+                           struct GNUNET_EXIT_TcpDataMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
   };
   struct GNUNET_HashCode port;
   char *policy;
@@ -3889,10 +3827,7 @@ run (void *cls,
                                  NULL);
   stats = GNUNET_STATISTICS_create ("exit",
                                     cfg);
-  cadet_handle = GNUNET_CADET_connect (cfg,
-                                       NULL,
-                                       &clean_channel,
-                                       handlers);
+  cadet_handle = GNUNET_CADET_connect (cfg);
   if (NULL == cadet_handle)
   {
     GNUNET_SCHEDULER_shutdown ();
@@ -3925,10 +3860,13 @@ run (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Opening CADET port %s for IPv4 gateway service\n",
                 GNUNET_h2s (&port));
-    GNUNET_CADET_open_port (cadet_handle,
-                            &port,
-                            &new_channel,
-                            NULL);
+    cadet_port4 = GNUNET_CADET_open_port (cadet_handle,
+                                          &port,
+                                          &new_channel,
+                                          NULL,
+                                          NULL,
+                                          &clean_channel,
+                                          handlers);
     policy = NULL;
     if (GNUNET_OK !=
         GNUNET_CONFIGURATION_get_value_string (cfg,
@@ -3962,10 +3900,13 @@ run (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Opening CADET port %s for IPv6 gateway service\n",
                 GNUNET_h2s (&port));
-    GNUNET_CADET_open_port (cadet_handle,
-                            &port,
-                            &new_channel,
-                            NULL);
+    cadet_port6 = GNUNET_CADET_open_port (cadet_handle,
+                                          &port,
+                                          &new_channel,
+                                          NULL,
+                                          NULL,
+                                          &clean_channel,
+                                          handlers);
     policy = NULL;
     if (GNUNET_OK !=
         GNUNET_CONFIGURATION_get_value_string (cfg,
diff --git a/src/exit/gnunet-helper-exit-windows.c b/src/exit/gnunet-helper-exit-windows.c
index 6be65ccd5..aa7a7a4a3 100644
--- a/src/exit/gnunet-helper-exit-windows.c
+++ b/src/exit/gnunet-helper-exit-windows.c
@@ -77,7 +77,7 @@
 static boolean privilege_testing = FALSE;
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/exit/gnunet-helper-exit.c b/src/exit/gnunet-helper-exit.c
index e14c6ca43..3c6f97557 100644
--- a/src/exit/gnunet-helper-exit.c
+++ b/src/exit/gnunet-helper-exit.c
@@ -62,7 +62,7 @@
 #define DEBUG GNUNET_NO
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/fragmentation/Makefile.am b/src/fragmentation/Makefile.am
index 316f69158..85c4767c0 100644
--- a/src/fragmentation/Makefile.am
+++ b/src/fragmentation/Makefile.am
@@ -27,7 +27,7 @@ check_PROGRAMS = \
  test_fragmentation_parallel
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/fs/.gitignore b/src/fs/.gitignore
index cdfb27108..f0e2a4f7b 100644
--- a/src/fs/.gitignore
+++ b/src/fs/.gitignore
@@ -9,3 +9,35 @@ gnunet-helper-fs-publish
 gnunet-publish
 gnunet-search
 gnunet-service-fs
+test_fs_directory
+test_fs_download
+test_fs_download_cadet
+test_fs_download_indexed
+test_fs_download_persistence
+test_fs_file_information
+test_fs_getopt
+test_fs_list_indexed
+test_fs_namespace
+test_fs_namespace_list_updateable
+test_fs_publish
+test_fs_publish_persistence
+test_fs_search
+test_fs_search_persistence
+test_fs_search_probes
+test_fs_search_with_and
+test_fs_start_stop
+test_fs_test_lib
+test_fs_unindex
+test_fs_unindex_persistence
+test_fs_uri
+test_gnunet_fs_idx.py
+test_gnunet_fs_psd.py
+test_gnunet_fs_rec.py
+test_gnunet_service_fs_migration
+test_gnunet_service_fs_p2p
+test_gnunet_service_fs_p2p_cadet
+test_plugin_block_fs
+perf_gnunet_service_fs_p2p
+perf_gnunet_service_fs_p2p_index
+perf_gnunet_service_fs_p2p_respect
+rdir.gnd
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index 34f44f574..33260a794 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -219,6 +219,7 @@ gnunet_unindex_LDADD = \
 libgnunet_plugin_block_fs_la_SOURCES = \
   plugin_block_fs.c
 libgnunet_plugin_block_fs_la_LIBADD = \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
   $(top_builddir)/src/block/libgnunetblock.la \
   libgnunetfs.la \
   $(top_builddir)/src/util/libgnunetutil.la \
@@ -288,7 +289,7 @@ endif
 
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH; $(MONKEY)
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME; $(MONKEY)
 TESTS = \
  test_fs_directory \
  test_fs_download \
diff --git a/src/fs/fs.conf.in b/src/fs/fs.conf.in
index 3534378ae..d46de387f 100644
--- a/src/fs/fs.conf.in
+++ b/src/fs/fs.conf.in
@@ -36,7 +36,7 @@ MAX_PENDING_REQUESTS = 65536
 
 # How many requests do we have at most waiting in the queue towards
 # the datastore? (important for memory consumption)
-DATASTORE_QUEUE_SIZE = 1024
+DATASTORE_QUEUE_SIZE = 32
 
 # Maximum frequency we're allowed to poll the datastore
 # for content for migration (can be used to reduce
diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c
index 7e769483b..a5b82ec27 100644
--- a/src/fs/fs_api.c
+++ b/src/fs/fs_api.c
@@ -1647,7 +1647,9 @@ deserialize_publish_file (void *cls,
                 filename, emsg);
     GNUNET_free (emsg);
   }
-  pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc);
+  pc->top = GNUNET_FS_make_top (h,
+                                &GNUNET_FS_publish_signal_suspend_,
+                                pc);
   return GNUNET_OK;
 cleanup:
   GNUNET_free_non_null (pc->nid);
@@ -2278,7 +2280,9 @@ deserialize_unindex_file (void *cls,
     GNUNET_break (0);
     goto cleanup;
   }
-  uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc);
+  uc->top = GNUNET_FS_make_top (h,
+                                &GNUNET_FS_unindex_signal_suspend_,
+                                uc);
   pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME;
   pi.value.unindex.specifics.resume.message = uc->emsg;
   GNUNET_FS_unindex_make_status_ (&pi, uc,
@@ -2537,7 +2541,6 @@ signal_download_resume (struct GNUNET_FS_DownloadContext *dc)
     signal_download_resume (dcc);
     dcc = dcc->next;
   }
-  GNUNET_FS_download_start_downloading_ (dc);
 }
 
 
@@ -2806,7 +2809,8 @@ deserialize_download (struct GNUNET_FS_Handle *h,
       GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri));
   if (GNUNET_FS_uri_test_loc (dc->uri))
     GNUNET_assert (GNUNET_OK ==
-                   GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target));
+                   GNUNET_FS_uri_loc_get_peer_identity (dc->uri,
+                                                        &dc->target));
   if (NULL == dc->emsg)
   {
     dc->top_request = read_download_request (rh);
@@ -2816,10 +2820,14 @@ deserialize_download (struct GNUNET_FS_Handle *h,
       goto cleanup;
     }
   }
-  dn = get_download_sync_filename (dc, dc->serialization, ".dir");
+  dn = get_download_sync_filename (dc,
+                                   dc->serialization,
+                                   ".dir");
   if (NULL != dn)
   {
-    if (GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES))
+    if (GNUNET_YES ==
+        GNUNET_DISK_directory_test (dn,
+                                    GNUNET_YES))
       GNUNET_DISK_directory_scan (dn,
                                   &deserialize_subdownload,
                                   dc);
@@ -2836,15 +2844,17 @@ deserialize_download (struct GNUNET_FS_Handle *h,
     dc->search = search;
     search->download = dc;
   }
-  if ((NULL == parent) && (NULL == search))
+  if ( (NULL == parent) &&
+       (NULL == search) )
   {
-    dc->top =
-        GNUNET_FS_make_top (dc->h,
+    dc->top
+      = GNUNET_FS_make_top (dc->h,
                             &GNUNET_FS_download_signal_suspend_,
                             dc);
     signal_download_resume (dc);
   }
   GNUNET_free (uris);
+  GNUNET_assert (NULL == dc->job_queue);
   dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
                                        dc);
   return;
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h
index e85de94a7..be22ea73e 100644
--- a/src/fs/fs_api.h
+++ b/src/fs/fs_api.h
@@ -1464,21 +1464,11 @@ struct GNUNET_FS_UnindexContext
   struct GNUNET_CRYPTO_FileHashContext *fhc;
 
   /**
-   * Which values have we seen already?
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *seen_dh;
-
-  /**
    * Overall size of the file.
    */
   uint64_t file_size;
 
   /**
-   * Random offset given to #GNUNET_DATASTORE_get_key.
-   */
-  uint64_t roff;
-
-  /**
    * When did we start?
    */
   struct GNUNET_TIME_Absolute start_time;
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c
index a18a903be..514eb64b3 100644
--- a/src/fs/fs_directory.c
+++ b/src/fs/fs_directory.c
@@ -170,13 +170,14 @@ find_full_data (void *cls, const char *plugin_name,
  * @param data pointer to the beginning of the directory
  * @param offset offset of data in the directory
  * @param dep function to call on each entry
- * @param dep_cls closure for dep
- * @return GNUNET_OK if this could be a block in a directory,
- *         GNUNET_NO if this could be part of a directory (but not 100% OK)
- *         GNUNET_SYSERR if 'data' does not represent a directory
+ * @param dep_cls closure for @a dep
+ * @return #GNUNET_OK if this could be a block in a directory,
+ *         #GNUNET_NO if this could be part of a directory (but not 100% OK)
+ *         #GNUNET_SYSERR if @a data does not represent a directory
  */
 int
-GNUNET_FS_directory_list_contents (size_t size, const void *data,
+GNUNET_FS_directory_list_contents (size_t size,
+                                   const void *data,
                                    uint64_t offset,
                                    GNUNET_FS_DirectoryEntryProcessor dep,
                                    void *dep_cls)
@@ -194,12 +195,16 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
 
   if ((offset == 0) &&
       ((size < 8 + sizeof (uint32_t)) ||
-       (0 != memcmp (cdata, GNUNET_FS_DIRECTORY_MAGIC, 8))))
+       (0 != memcmp (cdata,
+                     GNUNET_FS_DIRECTORY_MAGIC,
+                     8))))
     return GNUNET_SYSERR;
   pos = offset;
   if (offset == 0)
   {
-    GNUNET_memcpy (&mdSize, &cdata[8], sizeof (uint32_t));
+    GNUNET_memcpy (&mdSize,
+                   &cdata[8],
+                   sizeof (uint32_t));
     mdSize = ntohl (mdSize);
     if (mdSize > size - 8 - sizeof (uint32_t))
     {
@@ -215,7 +220,12 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       GNUNET_break (0);
       return GNUNET_SYSERR;     /* malformed ! */
     }
-    dep (dep_cls, NULL, NULL, md, 0, NULL);
+    dep (dep_cls,
+         NULL,
+         NULL,
+         md,
+         0,
+         NULL);
     GNUNET_CONTAINER_meta_data_destroy (md);
     pos = 8 + sizeof (uint32_t) + mdSize;
   }
@@ -247,7 +257,7 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
 
     uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
     pos = epos + 1;
-    if (uri == NULL)
+    if (NULL == uri)
     {
       GNUNET_free (emsg);
       pos--;                    /* go back to '\0' to force going to next alignment */
@@ -260,7 +270,9 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       return GNUNET_NO;         /* illegal in directory! */
     }
 
-    GNUNET_memcpy (&mdSize, &cdata[pos], sizeof (uint32_t));
+    GNUNET_memcpy (&mdSize,
+                   &cdata[pos],
+                   sizeof (uint32_t));
     mdSize = ntohl (mdSize);
     pos += sizeof (uint32_t);
     if (pos + mdSize > size)
@@ -269,8 +281,9 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       return GNUNET_NO;         /* malformed - or partial download */
     }
 
-    md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos], mdSize);
-    if (md == NULL)
+    md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos],
+                                                 mdSize);
+    if (NULL == md)
     {
       GNUNET_FS_uri_destroy (uri);
       GNUNET_break (0);
@@ -282,10 +295,17 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
                                                 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
     full_data.size = 0;
     full_data.data = NULL;
-    GNUNET_CONTAINER_meta_data_iterate (md, &find_full_data, &full_data);
-    if (dep != NULL)
+    GNUNET_CONTAINER_meta_data_iterate (md,
+                                        &find_full_data,
+                                        &full_data);
+    if (NULL != dep)
     {
-      dep (dep_cls, filename, uri, md, full_data.size, full_data.data);
+      dep (dep_cls,
+           filename,
+           uri,
+           md,
+           full_data.size,
+           full_data.data);
     }
     GNUNET_free_non_null (full_data.data);
     GNUNET_free_non_null (filename);
@@ -548,11 +568,12 @@ block_align (size_t start, unsigned int count, const size_t * sizes,
  * @param bld directory to finish
  * @param rsize set to the number of bytes needed
  * @param rdata set to the encoded directory
- * @return GNUNET_OK on success
+ * @return #GNUNET_OK on success
  */
 int
 GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
-                                    size_t * rsize, void **rdata)
+                                    size_t * rsize,
+                                    void **rdata)
 {
   char *data;
   char *sptr;
@@ -575,9 +596,12 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
   bes = NULL;
   if (0 < bld->count)
   {
-    sizes = GNUNET_malloc (bld->count * sizeof (size_t));
-    perm = GNUNET_malloc (bld->count * sizeof (unsigned int));
-    bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *));
+    sizes = GNUNET_new_array (bld->count,
+                              size_t);
+    perm = GNUNET_new_array (bld->count,
+                             unsigned int);
+    bes = GNUNET_new_array (bld->count,
+                            struct BuilderEntry *);
     pos = bld->head;
     for (i = 0; i < bld->count; i++)
     {
@@ -599,7 +623,8 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
   data = GNUNET_malloc_large (size);
   if (data == NULL)
   {
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc");
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                         "malloc");
     *rsize = 0;
     *rdata = NULL;
     GNUNET_free_non_null (sizes);
@@ -608,17 +633,22 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
     return GNUNET_SYSERR;
   }
   *rdata = data;
-  GNUNET_memcpy (data, GNUNET_DIRECTORY_MAGIC, strlen (GNUNET_DIRECTORY_MAGIC));
+  GNUNET_memcpy (data,
+                 GNUNET_DIRECTORY_MAGIC,
+                 strlen (GNUNET_DIRECTORY_MAGIC));
   off = strlen (GNUNET_DIRECTORY_MAGIC);
 
   sptr = &data[off + sizeof (uint32_t)];
   ret =
-      GNUNET_CONTAINER_meta_data_serialize (bld->meta, &sptr,
+      GNUNET_CONTAINER_meta_data_serialize (bld->meta,
+                                            &sptr,
                                             size - off - sizeof (uint32_t),
                                             GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
   GNUNET_assert (ret != -1);
   big = htonl (ret);
-  GNUNET_memcpy (&data[off], &big, sizeof (uint32_t));
+  GNUNET_memcpy (&data[off],
+                 &big,
+                 sizeof (uint32_t));
   off += sizeof (uint32_t) + ret;
   for (j = 0; j < bld->count; j++)
   {
diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c
index 2f79c7c05..7b9f178fd 100644
--- a/src/fs/fs_dirmetascan.c
+++ b/src/fs/fs_dirmetascan.c
@@ -245,12 +245,10 @@ finish_scan (void *cls)
  * Calls the scanner progress handler.
  *
  * @param cls the closure (directory scanner object)
- * @param client always NULL
  * @param msg message from the helper process
  */
 static int
 process_helper_msgs (void *cls,
-                     void *client,
                      const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index 53b836f22..ce852f2d0 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -37,14 +37,14 @@ static int
 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 {
   return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
-      ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
-       ((NULL == dc->meta) &&
-        ((NULL == dc->filename) ||
-         ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
-          (NULL !=
-           strstr (dc->filename + strlen (dc->filename) -
-                   strlen (GNUNET_FS_DIRECTORY_EXT),
-                   GNUNET_FS_DIRECTORY_EXT))))));
+      ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
+        ( (NULL == dc->meta) &&
+          ( (NULL == dc->filename) ||
+            ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
+              (NULL !=
+               strstr (dc->filename + strlen (dc->filename) -
+                       strlen (GNUNET_FS_DIRECTORY_EXT),
+                       GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
 }
 
 
@@ -278,10 +278,12 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc);
  * @param data contents of the file (or NULL if they were not inlined)
  */
 static void
-trigger_recursive_download (void *cls, const char *filename,
+trigger_recursive_download (void *cls,
+                            const char *filename,
                             const struct GNUNET_FS_Uri *uri,
                             const struct GNUNET_CONTAINER_MetaData *meta,
-                            size_t length, const void *data);
+                            size_t length,
+                            const void *data);
 
 
 /**
@@ -304,24 +306,28 @@ full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
   if (size64 != (uint64_t) size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                _
-                ("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
+                _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
     return;
   }
   if (NULL != dc->filename)
   {
-    h = GNUNET_DISK_file_open (dc->filename, GNUNET_DISK_OPEN_READ,
+    h = GNUNET_DISK_file_open (dc->filename,
+                               GNUNET_DISK_OPEN_READ,
                                GNUNET_DISK_PERM_NONE);
   }
   else
   {
     GNUNET_assert (NULL != dc->temp_filename);
-    h = GNUNET_DISK_file_open (dc->temp_filename, GNUNET_DISK_OPEN_READ,
+    h = GNUNET_DISK_file_open (dc->temp_filename,
+                               GNUNET_DISK_OPEN_READ,
                                GNUNET_DISK_PERM_NONE);
   }
   if (NULL == h)
     return;                     /* oops */
-  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
+  data = GNUNET_DISK_file_map (h,
+                               &m,
+                               GNUNET_DISK_MAP_TYPE_READ,
+                               size);
   if (NULL == data)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -329,15 +335,25 @@ full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
   }
   else
   {
-    GNUNET_FS_directory_list_contents (size, data, 0,
-                                       &trigger_recursive_download, dc);
+    if (GNUNET_OK !=
+        GNUNET_FS_directory_list_contents (size,
+                                           data,
+                                           0,
+                                           &trigger_recursive_download,
+                                           dc))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("Failed to access full directroy contents of `%s' for recursive download\n"),
+                  dc->filename);
+    }
     GNUNET_DISK_file_unmap (m);
   }
   GNUNET_DISK_file_close (h);
   if (NULL == dc->filename)
   {
     if (0 != UNLINK (dc->temp_filename))
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
                                 dc->temp_filename);
     GNUNET_free (dc->temp_filename);
     dc->temp_filename = NULL;
@@ -362,14 +378,16 @@ check_completed (struct GNUNET_FS_DownloadContext *dc)
   struct GNUNET_FS_DownloadContext *pos;
 
   /* first, check if we need to download children */
-  if ((NULL == dc->child_head) && (is_recursive_download (dc)))
+  if (is_recursive_download (dc))
     full_recursive_download (dc);
   /* then, check if children are done already */
   for (pos = dc->child_head; NULL != pos; pos = pos->next)
   {
-    if ((pos->emsg == NULL) && (pos->completed < pos->length))
+    if ( (NULL == pos->emsg) &&
+         (pos->completed < pos->length) )
       return;                   /* not done yet */
-    if ((pos->child_head != NULL) && (pos->has_finished != GNUNET_YES))
+    if ( (NULL != pos->child_head) &&
+         (pos->has_finished != GNUNET_YES) )
       return;                   /* not transitively done yet */
   }
   /* All of our children are done, so mark this download done */
@@ -471,7 +489,11 @@ try_match_block (struct GNUNET_FS_DownloadContext *dc,
   }
   GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key);
   GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
-  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc))
+  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset],
+                                             dlen,
+                                             &sk,
+                                             &iv,
+                                             enc))
   {
     GNUNET_break (0);
     return;
@@ -484,7 +506,9 @@ try_match_block (struct GNUNET_FS_DownloadContext *dc,
     dr->state = BRS_RECONSTRUCT_META_UP;
     break;
   case BRS_CHK_SET:
-    if (0 != memcmp (&in_chk, &dr->chk, sizeof (struct ContentHashKey)))
+    if (0 != memcmp (&in_chk,
+                     &dr->chk,
+                     sizeof (struct ContentHashKey)))
     {
       /* other peer provided bogus meta data */
       GNUNET_break_op (0);
@@ -591,7 +615,10 @@ match_full_data (void *cls, const char *plugin_name,
     GNUNET_break_op (0);
     return 1;                   /* bogus meta data */
   }
-  try_match_block (dc, dc->top_request, data, data_len);
+  try_match_block (dc,
+                   dc->top_request,
+                   data,
+                   data_len);
   return 1;
 }
 
@@ -820,10 +847,12 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
  * @param data contents of the file (or NULL if they were not inlined)
  */
 static void
-trigger_recursive_download (void *cls, const char *filename,
+trigger_recursive_download (void *cls,
+                            const char *filename,
                             const struct GNUNET_FS_Uri *uri,
                             const struct GNUNET_CONTAINER_MetaData *meta,
-                            size_t length, const void *data)
+                            size_t length,
+                            const void *data)
 {
   struct GNUNET_FS_DownloadContext *dc = cls;
   struct GNUNET_FS_DownloadContext *cpos;
@@ -936,9 +965,17 @@ trigger_recursive_download (void *cls, const char *filename,
               (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
               (unsigned int)
               GNUNET_CONTAINER_meta_data_get_serialized_size (meta));
-  GNUNET_FS_download_start (dc->h, uri, meta, full_name, temp_name, 0,
+  GNUNET_FS_download_start (dc->h,
+                            uri,
+                            meta,
+                            full_name,
+                            temp_name,
+                            0,
                             GNUNET_FS_uri_chk_get_file_size (uri),
-                            dc->anonymity, dc->options, NULL, dc);
+                            dc->anonymity,
+                            dc->options,
+                            NULL,
+                            dc);
   GNUNET_free_non_null (full_name);
   GNUNET_free_non_null (temp_name);
   GNUNET_free_non_null (fn);
@@ -953,11 +990,9 @@ trigger_recursive_download (void *cls, const char *filename,
 void
 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
 {
-  unsigned int i;
-
   if (NULL == dr)
     return;
-  for (i = 0; i < dr->num_children; i++)
+  for (unsigned int i = 0; i < dr->num_children; i++)
     GNUNET_FS_free_download_request_ (dr->children[i]);
   GNUNET_free_non_null (dr->children);
   GNUNET_free (dr);
@@ -1363,7 +1398,6 @@ do_reconnect (void *cls)
 static void
 try_reconnect (struct GNUNET_FS_DownloadContext *dc)
 {
-
   if (NULL != dc->mq)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1380,6 +1414,7 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc)
               "Will try to reconnect in %s\n",
               GNUNET_STRINGS_relative_time_to_string (dc->reconnect_backoff,
                                                       GNUNET_YES));
+  GNUNET_break (NULL != dc->job_queue);
   dc->task =
     GNUNET_SCHEDULER_add_delayed (dc->reconnect_backoff,
                                   &do_reconnect,
@@ -1509,13 +1544,17 @@ create_download_request (struct DownloadRequest *parent,
   GNUNET_assert (dr->num_children > 0);
 
   dr->children =
-    GNUNET_malloc (dr->num_children * sizeof (struct DownloadRequest *));
+    GNUNET_new_array (dr->num_children,
+                      struct DownloadRequest *);
   for (i = 0; i < dr->num_children; i++)
   {
     dr->children[i] =
-      create_download_request (dr, i + head_skip, depth - 1,
+      create_download_request (dr,
+                               i + head_skip,
+                               depth - 1,
                                dr_offset + (i + head_skip) * child_block_size,
-                               file_start_offset, desired_length);
+                               file_start_offset,
+                               desired_length);
   }
   return dr;
 }
@@ -1533,7 +1572,7 @@ reconstruct_cont (void *cls)
   struct GNUNET_FS_DownloadContext *dc = cls;
 
   /* clean up state from tree encoder */
-  if (dc->task != NULL)
+  if (NULL != dc->task)
   {
     GNUNET_SCHEDULER_cancel (dc->task);
     dc->task = NULL;
@@ -1584,9 +1623,13 @@ get_next_block (void *cls)
  * @param block_size size of block (in bytes)
  */
 static void
-reconstruct_cb (void *cls, const struct ContentHashKey *chk, uint64_t offset,
-                unsigned int depth, enum GNUNET_BLOCK_Type type,
-                const void *block, uint16_t block_size)
+reconstruct_cb (void *cls,
+                const struct ContentHashKey *chk,
+                uint64_t offset,
+                unsigned int depth,
+                enum GNUNET_BLOCK_Type type,
+                const void *block,
+                uint16_t block_size)
 {
   struct GNUNET_FS_DownloadContext *dc = cls;
   struct GNUNET_FS_ProgressInfo pi;
@@ -1607,7 +1650,8 @@ reconstruct_cb (void *cls, const struct ContentHashKey *chk, uint64_t offset,
                   "Block %u < %u irrelevant for our range\n",
                   chld,
                   dr->children[0]->chk_idx);
-      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
+                                           dc);
       return; /* irrelevant block */
     }
     if (chld > dr->children[dr->num_children-1]->chk_idx)
@@ -1701,8 +1745,10 @@ reconstruct_cb (void *cls, const struct ContentHashKey *chk, uint64_t offset,
     GNUNET_assert (0);
     break;
   }
-  dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
-  if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP))
+  dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
+                                       dc);
+  if ( (dr == dc->top_request) &&
+       (dr->state == BRS_DOWNLOAD_UP) )
     check_completed (dc);
 }
 
@@ -1882,7 +1928,8 @@ GNUNET_FS_download_start_task_ (void *cls)
                                      &reconstruct_cb,
                                      NULL,
                                      &reconstruct_cont);
-    dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+    dc->task = GNUNET_SCHEDULER_add_now (&get_next_block,
+                                         dc);
   }
   else
   {
@@ -2037,6 +2084,7 @@ create_download_context (struct GNUNET_FS_Handle *h,
               filename,
               (unsigned long long) length,
               dc->treedepth);
+  GNUNET_assert (NULL == dc->job_queue);
   dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_,
                                        dc);
   return dc;
@@ -2199,9 +2247,10 @@ GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
   if (NULL != dc->mq)
     return; /* already running */
   GNUNET_assert (NULL == dc->job_queue);
+  GNUNET_assert (NULL == dc->task);
   GNUNET_assert (NULL != dc->active);
-  dc->job_queue =
-      GNUNET_FS_queue_ (dc->h,
+  dc->job_queue
+    = GNUNET_FS_queue_ (dc->h,
                         &activate_fs_download,
                         &deactivate_fs_download,
                         dc,
@@ -2240,14 +2289,15 @@ GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc)
   pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
   GNUNET_FS_download_make_status_ (&pi, dc);
 
-  dc->job_queue =
-    GNUNET_FS_queue_ (dc->h,
-                      &activate_fs_download,
-                      &deactivate_fs_download,
-                      dc, (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
-                      (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
-                      ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
-                      : GNUNET_FS_QUEUE_PRIORITY_PROBE);
+  GNUNET_assert (NULL == dc->task);
+  dc->job_queue
+    = GNUNET_FS_queue_ (dc->h,
+                        &activate_fs_download,
+                        &deactivate_fs_download,
+                        dc, (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
+                        (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
+                        ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
+                        : GNUNET_FS_QUEUE_PRIORITY_PROBE);
 
 }
 
diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c
index f78e311d3..bfe45957e 100644
--- a/src/fs/fs_getopt.c
+++ b/src/fs/fs_getopt.c
@@ -25,6 +25,7 @@
  */
 #include "platform.h"
 #include "gnunet_fs_service.h"
+#include "gnunet_getopt_lib.h"
 #include "fs_api.h"
 
 /* ******************** command-line option parsing API ******************** */
@@ -41,10 +42,10 @@
  * @param value command line argument given
  * @return GNUNET_OK on success
  */
-int
-GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value)
+static int
+getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
+                     *ctx, void *scls, const char *option,
+                      const char *value)
 {
   struct GNUNET_FS_Uri **uri = scls;
   struct GNUNET_FS_Uri *u = *uri;
@@ -107,6 +108,34 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
   return GNUNET_OK;
 }
 
+/**
+ * Allow user to specify keywords.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] topKeywords set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_KEYWORDS (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_Uri **topKeywords)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_keywords,
+    .scls = (void *) topKeywords  
+  };
+
+  return clo;
+}
 
 /**
  * Command-line option parser function that allows the user to specify
@@ -120,11 +149,11 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
  * @param value command line argument given
  * @return #GNUNET_OK on success
  */
-int
-GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                               void *scls,
-                               const char *option,
-                               const char *value)
+static int
+getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                     void *scls,
+                     const char *option,
+                     const char *value)
 {
   struct GNUNET_CONTAINER_MetaData **mm = scls;
 #if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR
@@ -200,4 +229,36 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext
   return GNUNET_OK;
 }
 
+/**
+ * Allow user to specify metadata.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] metadata set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_METADATA (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_CONTAINER_MetaData **meta)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_metadata,
+    .scls = (void *) meta
+  };
+
+  return clo;
+}
+
+
+
+
 /* end of fs_getopt.c */
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 86a58a58b..759467ba4 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -837,7 +837,7 @@ hash_for_index_cb (void *cls,
   GNUNET_assert (fn != NULL);
   slen = strlen (fn) + 1;
   if (slen >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage))
+      GNUNET_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _
@@ -1226,7 +1226,7 @@ fip_signal_start (void *cls,
   {
     kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
     pc->reserve_entries += kc;
-    pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
+    pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc;
   }
   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
   *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c
index 75ce4b54f..8c6f5edcf 100644
--- a/src/fs/fs_search.c
+++ b/src/fs/fs_search.c
@@ -1088,15 +1088,17 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
   unsigned int left;
   unsigned int todo;
   unsigned int fit;
-  int first_call;
   unsigned int search_request_map_offset;
   unsigned int keyword_offset;
+  int first_call;
 
   memset (&mbc, 0, sizeof (mbc));
   mbc.sc = sc;
   if (GNUNET_FS_uri_test_ksk (sc->uri))
   {
-    mbc.put_cnt = 0;
+    /* This will calculate the result set size ONLY for
+       "keyword_offset == 0", so we will have to recalculate
+       it for the other keywords later! */
     GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                            &find_result_set,
                                            &mbc);
@@ -1109,7 +1111,6 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
   }
   search_request_map_offset = 0;
   keyword_offset = 0;
-
   first_call = GNUNET_YES;
   while ( (0 != (left =
                  (total_seen_results - search_request_map_offset))) ||
@@ -1120,7 +1121,7 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
     if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY))
       options |= SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY;
 
-    fit = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*sm)) / sizeof (struct GNUNET_HashCode);
+    fit = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*sm)) / sizeof (struct GNUNET_HashCode);
     todo = GNUNET_MIN (fit,
                        left);
     env = GNUNET_MQ_msg_extra (sm,
@@ -1128,6 +1129,11 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
                                GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
     mbc.skip_cnt = search_request_map_offset;
     mbc.xoff = (struct GNUNET_HashCode *) &sm[1];
+    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
+    sm->anonymity_level = htonl (sc->anonymity);
+    memset (&sm->target,
+            0,
+            sizeof (struct GNUNET_PeerIdentity));
 
     if (GNUNET_FS_uri_test_ksk (sc->uri))
     {
@@ -1135,11 +1141,6 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
       /* calculate how many results we can send in this message */
       mbc.put_cnt = todo;
       /* now build message */
-      sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
-      sm->anonymity_level = htonl (sc->anonymity);
-      memset (&sm->target,
-              0,
-              sizeof (struct GNUNET_PeerIdentity));
       sm->query = sc->requests[keyword_offset].uquery;
       GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                              &build_result_set,
@@ -1156,11 +1157,17 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
       {
         sm->options = htonl (options);
         keyword_offset++;
-        search_request_map_offset = 0;
         if (sc->uri->data.ksk.keywordCount != keyword_offset)
         {
           /* more keywords => more requesting to be done... */
           first_call = GNUNET_YES;
+          search_request_map_offset = 0;
+          mbc.put_cnt = 0;
+          mbc.keyword_offset = keyword_offset;
+          GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                                 &find_result_set,
+                                                 &mbc);
+          total_seen_results = mbc.put_cnt;
         }
       }
     }
@@ -1168,11 +1175,6 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
     {
       GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri));
 
-      sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
-      sm->anonymity_level = htonl (sc->anonymity);
-      memset (&sm->target,
-              0,
-              sizeof (struct GNUNET_PeerIdentity));
       GNUNET_CRYPTO_ecdsa_public_key_derive (&sc->uri->data.sks.ns,
                                              sc->uri->data.sks.identifier,
                                              "fs-ublock",
@@ -1383,9 +1385,10 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc)
     GNUNET_assert (0 != sc->uri->data.ksk.keywordCount);
     anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
     GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub);
-    sc->requests =
-        GNUNET_malloc (sizeof (struct SearchRequestEntry) *
-                       sc->uri->data.ksk.keywordCount);
+    sc->requests
+      = GNUNET_new_array (sc->uri->data.ksk.keywordCount,
+                          struct SearchRequestEntry);
+
     for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
     {
       keyword = &sc->uri->data.ksk.keywords[i][1];
@@ -1666,8 +1669,8 @@ search_result_stop (void *cls,
   if (NULL != sr->download)
   {
     sr->download->search = NULL;
-    sr->download->top =
-        GNUNET_FS_make_top (sr->download->h,
+    sr->download->top
+      = GNUNET_FS_make_top (sr->download->h,
                             &GNUNET_FS_download_signal_suspend_,
                             sr->download);
     if (NULL != sr->download->serialization)
@@ -1679,7 +1682,8 @@ search_result_stop (void *cls,
       sr->download->serialization = NULL;
     }
     pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT;
-    GNUNET_FS_download_make_status_ (&pi, sr->download);
+    GNUNET_FS_download_make_status_ (&pi,
+                                     sr->download);
     GNUNET_FS_download_sync_ (sr->download);
     sr->download = NULL;
   }
@@ -1745,25 +1749,28 @@ GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc)
   if (NULL != sc->top)
     GNUNET_FS_end_top (sc->h, sc->top);
   GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
-                                         &search_result_stop, sc);
+                                         &search_result_stop,
+                                         sc);
   if (NULL != sc->psearch_result)
     sc->psearch_result->update_search = NULL;
   if (NULL != sc->serialization)
   {
     GNUNET_FS_remove_sync_file_ (sc->h,
-                                 (sc->psearch_result !=
-                                  NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH :
-                                 GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                 (NULL != sc->psearch_result)
+                                 ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                 : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
                                  sc->serialization);
     GNUNET_FS_remove_sync_dir_ (sc->h,
-                                (sc->psearch_result !=
-                                 NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH :
-                                GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                (NULL != sc->psearch_result)
+                                ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
                                 sc->serialization);
     GNUNET_free (sc->serialization);
   }
   pi.status = GNUNET_FS_STATUS_SEARCH_STOPPED;
-  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi,
+                                                   sc->h,
+                                                   sc);
   GNUNET_break (NULL == sc->client_info);
   if (NULL != sc->task)
   {
diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c
index b3c632203..e57e4e494 100644
--- a/src/fs/fs_tree.c
+++ b/src/fs/fs_tree.c
@@ -285,9 +285,9 @@ GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size,
   te->progress = progress;
   te->cont = cont;
   te->chk_tree_depth = GNUNET_FS_compute_depth (size);
-  te->chk_tree =
-      GNUNET_malloc (te->chk_tree_depth * CHK_PER_INODE *
-                     sizeof (struct ContentHashKey));
+  te->chk_tree
+    = GNUNET_new_array (te->chk_tree_depth * CHK_PER_INODE,
+                        struct ContentHashKey);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Created tree encoder for file with %llu bytes and depth %u\n",
               (unsigned long long) size,
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c
index a672b84d5..e1c7ea535 100644
--- a/src/fs/fs_unindex.c
+++ b/src/fs/fs_unindex.c
@@ -312,8 +312,6 @@ unindex_finish (struct GNUNET_FS_UnindexContext *uc)
   uc->fh = NULL;
   GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
   uc->dsh = NULL;
-  GNUNET_CONTAINER_multihashmap_destroy (uc->seen_dh);
-  uc->seen_dh = NULL;
   uc->state = UNINDEX_STATE_FS_NOTIFY;
   GNUNET_FS_unindex_sync_ (uc);
   uc->mq = GNUNET_CLIENT_connect (uc->h->cfg,
@@ -444,7 +442,6 @@ continue_after_remove (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to remove UBlock: %s\n"),
                 msg);
-  GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
   uc->ksk_offset++;
   GNUNET_FS_unindex_do_remove_kblocks_ (uc);
 }
@@ -454,7 +451,8 @@ continue_after_remove (void *cls,
  * Function called from datastore with result from us looking for
  * a UBlock.  There are four cases:
  * 1) no result, means we move on to the next keyword
- * 2) UID is the same as the first UID, means we move on to next keyword
+ * 2) data hash is the same as an already seen data hash, means we move on to
+ *    next keyword
  * 3) UBlock for a different CHK, means we keep looking for more
  * 4) UBlock is for our CHK, means we remove the block and then move
  *           on to the next keyword
@@ -485,34 +483,15 @@ process_kblock_for_unindex (void *cls,
   const struct UBlock *ub;
   struct GNUNET_FS_Uri *chk_uri;
   struct GNUNET_HashCode query;
-  struct GNUNET_HashCode dh;
 
   uc->dqe = NULL;
   if (NULL == data)
   {
     /* no result */
-    GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
     uc->ksk_offset++;
     GNUNET_FS_unindex_do_remove_kblocks_ (uc);
     return;
   }
-  GNUNET_CRYPTO_hash (data,
-                      size,
-                      &dh);
-  if (GNUNET_YES ==
-      GNUNET_CONTAINER_multihashmap_contains (uc->seen_dh,
-                                              &dh))
-  {
-    GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
-    uc->ksk_offset++;
-    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
-    return;
-  }
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (uc->seen_dh,
-                                                    &dh,
-                                                    uc,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
   if (size < sizeof (struct UBlock))
   {
@@ -565,23 +544,24 @@ process_kblock_for_unindex (void *cls,
   GNUNET_FS_uri_destroy (chk_uri);
   /* matches! */
   uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
-                                     key,
+                                     key,
                                      size,
                                      data,
-                                     0 /* priority */,
+                                     0 /* priority */,
                                      1 /* queue size */,
-                                     &continue_after_remove,
-                                     uc);
+                                     &continue_after_remove,
+                                     uc);
   return;
  get_next:
   uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
-                                      uc->roff++,
-                                      &uc->uquery,
-                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
-                                      0 /* priority */,
+                                      uid + 1 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
                                       1 /* queue size */,
-                                      &process_kblock_for_unindex,
-                                      uc);
+                                      &process_kblock_for_unindex,
+                                      uc);
 }
 
 
@@ -626,13 +606,14 @@ GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
                       sizeof (dpub),
                       &uc->uquery);
   uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
-                                      uc->roff++,
-                                      &uc->uquery,
-                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
-                                      0 /* priority */,
+                                      0 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
                                       1 /* queue size */,
-                                      &process_kblock_for_unindex,
-                                      uc);
+                                      &process_kblock_for_unindex,
+                                      uc);
 }
 
 
@@ -825,8 +806,6 @@ GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
   uc->start_time = GNUNET_TIME_absolute_get ();
   uc->file_size = size;
   uc->client_info = cctx;
-  uc->seen_dh = GNUNET_CONTAINER_multihashmap_create (4,
-                                                       GNUNET_NO);
   GNUNET_FS_unindex_sync_ (uc);
   pi.status = GNUNET_FS_STATUS_UNINDEX_START;
   pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c
index fa27e6e9b..11968b750 100644
--- a/src/fs/fs_uri.c
+++ b/src/fs/fs_uri.c
@@ -309,7 +309,8 @@ uri_ksk_parse (const char *s,
   }
   iret = max;
   dup = GNUNET_strdup (s);
-  keywords = GNUNET_malloc (max * sizeof (char *));
+  keywords = GNUNET_new_array (max,
+                               char *);
   for (i = slen - 1; i >= (int) pos; i--)
   {
     if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22")))
@@ -937,7 +938,8 @@ GNUNET_FS_uri_ksk_merge (const struct GNUNET_FS_Uri *u1,
     return NULL;
   }
   kc = u1->data.ksk.keywordCount;
-  kl = GNUNET_malloc ((kc + u2->data.ksk.keywordCount) * sizeof (char *));
+  kl = GNUNET_new_array (kc + u2->data.ksk.keywordCount,
+                         char *);
   for (i = 0; i < u1->data.ksk.keywordCount; i++)
     kl[i] = GNUNET_strdup (u1->data.ksk.keywords[i]);
   for (i = 0; i < u2->data.ksk.keywordCount; i++)
@@ -991,8 +993,9 @@ GNUNET_FS_uri_dup (const struct GNUNET_FS_Uri *uri)
     }
     if (ret->data.ksk.keywordCount > 0)
     {
-      ret->data.ksk.keywords =
-          GNUNET_malloc (ret->data.ksk.keywordCount * sizeof (char *));
+      ret->data.ksk.keywords
+        = GNUNET_new_array (ret->data.ksk.keywordCount,
+                            char *);
       for (i = 0; i < ret->data.ksk.keywordCount; i++)
         ret->data.ksk.keywords[i] = GNUNET_strdup (uri->data.ksk.keywords[i]);
     }
@@ -1078,7 +1081,8 @@ GNUNET_FS_uri_ksk_create (const char *keywords,
     *emsg = GNUNET_strdup (_("Number of double-quotes not balanced!\n"));
     return NULL;
   }
-  keywordarr = GNUNET_malloc (num_Words * sizeof (char *));
+  keywordarr = GNUNET_new_array (num_Words,
+                                 char *);
   num_Words = 0;
   inWord = 0;
   pos = searchString;
@@ -1151,7 +1155,8 @@ GNUNET_FS_uri_ksk_create_from_args (unsigned int argc,
   uri = GNUNET_new (struct GNUNET_FS_Uri);
   uri->type = GNUNET_FS_URI_KSK;
   uri->data.ksk.keywordCount = argc;
-  uri->data.ksk.keywords = GNUNET_malloc (argc * sizeof (char *));
+  uri->data.ksk.keywords = GNUNET_new_array (argc,
+                                             char *);
   for (i = 0; i < argc; i++)
   {
     keyword = argv[i];
@@ -1766,8 +1771,9 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
     }
     /* x3 because there might be a normalized variant of every keyword,
        plus theoretically one more for mime... */
-    ret->data.ksk.keywords = GNUNET_malloc
-      (sizeof (char *) * (ent + tok_keywords + paren_keywords) * 3);
+    ret->data.ksk.keywords
+      = GNUNET_new_array ((ent + tok_keywords + paren_keywords) * 3,
+                          char *);
     GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret);
   }
   if (tok_keywords > 0)
diff --git a/src/fs/gnunet-auto-share.c b/src/fs/gnunet-auto-share.c
index cc0111111..0976b37e4 100644
--- a/src/fs/gnunet-auto-share.c
+++ b/src/fs/gnunet-auto-share.c
@@ -72,7 +72,7 @@ static int ret;
 /**
  * Are we running 'verbosely'?
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Configuration to use.
@@ -759,26 +759,38 @@ free_item (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of sender-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity_level},
-    {'d', "disable-creation-time", NULL,
-     gettext_noop
-     ("disable adding the creation time to the metadata of the uploaded file"),
-     0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
-    {'D', "disable-extractor", NULL,
-     gettext_noop ("do not use libextractor to add keywords or metadata"),
-     0, &GNUNET_GETOPT_set_one, &disable_extractor},
-    {'p', "priority", "PRIORITY",
-     gettext_noop ("specify the priority of the content"),
-     1, &GNUNET_GETOPT_set_uint, &content_priority},
-    {'r', "replication", "LEVEL",
-     gettext_noop ("set the desired replication LEVEL"),
-     1, &GNUNET_GETOPT_set_uint, &replication_level},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of sender-anonymity"),
+                                   &anonymity_level),
+
+    GNUNET_GETOPT_option_flag ('d',
+                                  "disable-creation-time",
+                                  gettext_noop ("disable adding the creation time to the metadata of the uploaded file"),
+                                  &do_disable_creation_time),
+
+    GNUNET_GETOPT_option_flag ('D',
+                                  "disable-extractor",
+                                  gettext_noop ("do not use libextractor to add keywords or metadata"),
+                                  &disable_extractor),
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "priority",
+                                   "PRIORITY",
+                                   gettext_noop ("specify the priority of the content"),
+                                   &content_priority),
+
+    GNUNET_GETOPT_option_uint ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired replication LEVEL"),
+                                   &replication_level),
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
   struct WorkItem *wi;
diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c
index 6d9adb8ab..83c1eb505 100644
--- a/src/fs/gnunet-download.c
+++ b/src/fs/gnunet-download.c
@@ -30,7 +30,7 @@
 
 static int ret;
 
-static int verbose;
+static unsigned int verbose;
 
 static int delete_incomplete;
 
@@ -299,33 +299,51 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of receiver-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity},
-    {'D', "delete-incomplete", NULL,
-     gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"),
-     0, &GNUNET_GETOPT_set_one, &delete_incomplete},
-    {'n', "no-network", NULL,
-     gettext_noop ("only search the local peer (no P2P network search)"),
-     0, &GNUNET_GETOPT_set_one, &local_only},
-    {'o', "output", "FILENAME",
-     gettext_noop ("write the file to FILENAME"),
-     1, &GNUNET_GETOPT_set_string, &filename},
-    {'p', "parallelism", "DOWNLOADS",
-     gettext_noop
-     ("set the maximum number of parallel downloads that is allowed"),
-     1, &GNUNET_GETOPT_set_uint, &parallelism},
-    {'r', "request-parallelism", "REQUESTS",
-     gettext_noop
-     ("set the maximum number of parallel requests for blocks that is allowed"),
-     1, &GNUNET_GETOPT_set_uint, &request_parallelism},
-    {'R', "recursive", NULL,
-     gettext_noop ("download a GNUnet directory recursively"),
-     0, &GNUNET_GETOPT_set_one, &do_recursive},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_increment_value, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_uint ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of receiver-anonymity"),
+                                   &anonymity),
+
+    GNUNET_GETOPT_option_flag ('D',
+                                  "delete-incomplete",
+                                  gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"),
+                                  &delete_incomplete),
+
+    GNUNET_GETOPT_option_flag ('n',
+                                  "no-network",
+                                  gettext_noop ("only search the local peer (no P2P network search)"),
+                                  &local_only), 
+
+    GNUNET_GETOPT_option_string ('o',
+                                 "output",
+                                 "FILENAME",
+                                 gettext_noop ("write the file to FILENAME"),
+                                 &filename),
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "parallelism",
+                                   "DOWNLOADS",
+                                   gettext_noop ("set the maximum number of parallel downloads that is allowed"),
+                                   &parallelism),
+
+    GNUNET_GETOPT_option_uint ('r',
+                                   "request-parallelism",
+                                   "REQUESTS",
+                                   gettext_noop ("set the maximum number of parallel requests for blocks that is allowed"),
+                                   &request_parallelism), 
+
+    GNUNET_GETOPT_option_flag ('R',
+                                  "recursive",
+                                  gettext_noop ("download a GNUnet directory recursively"),
+                                  &do_recursive),
+
+    GNUNET_GETOPT_option_increment_uint ('V',
+                                          "verbose",
+                                          gettext_noop ("be verbose (print progress information)"),
+                                          &verbose), 
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/gnunet-fs-profiler.c b/src/fs/gnunet-fs-profiler.c
index cfbe57bbd..ac9f6777f 100644
--- a/src/fs/gnunet-fs-profiler.c
+++ b/src/fs/gnunet-fs-profiler.c
@@ -203,16 +203,26 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "num-peers", "COUNT",
-     gettext_noop ("run the experiment with COUNT peers"),
-     1, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'H', "hosts", "HOSTFILE",
-     gettext_noop ("specifies name of a file with the HOSTS the testbed should use"),
-     1, &GNUNET_GETOPT_set_string, &host_filename},
-    {'t', "timeout", "DELAY",
-     gettext_noop ("automatically terminate experiment after DELAY"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('n',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("run the experiment with COUNT peers"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_option_string ('H',
+                                 "hosts",
+                                 "HOSTFILE",
+                                 gettext_noop ("specifies name of a file with the HOSTS the testbed should use"),
+                                 &host_filename),
+
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("automatically terminate experiment after DELAY"),
+                                            &timeout),
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/fs/gnunet-fs.c b/src/fs/gnunet-fs.c
index 7c20e025f..ad8f223ff 100644
--- a/src/fs/gnunet-fs.c
+++ b/src/fs/gnunet-fs.c
@@ -43,7 +43,7 @@ static int list_indexed_files;
 /**
  * Option -v given?
  */
-static int verbose;
+static unsigned int verbose;
 
 
 /**
@@ -112,11 +112,14 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "list-indexed", NULL,
-     gettext_noop ("print a list of all indexed files"), 0,
-     &GNUNET_GETOPT_set_one, &list_indexed_files},
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_flag ('i',
+                                  "list-indexed",
+                                  gettext_noop ("print a list of all indexed files"),
+                                  &list_indexed_files),
+
+    GNUNET_GETOPT_option_verbose (&verbose),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c
index 55836e772..72a656de1 100644
--- a/src/fs/gnunet-helper-fs-publish.c
+++ b/src/fs/gnunet-helper-fs-publish.c
@@ -480,7 +480,7 @@ make_dev_zero (int fd,
   GNUNET_assert (-1 != z);
   if (z == fd)
     return;
-  dup2 (z, fd);
+  GNUNET_break (fd == dup2 (z, fd));
   GNUNET_assert (0 == close (z));
 }
 
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c
index a563d7b7a..395aad7db 100644
--- a/src/fs/gnunet-publish.c
+++ b/src/fs/gnunet-publish.c
@@ -37,7 +37,7 @@ static int ret;
 /**
  * Command line option 'verbose' set
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Handle to our configuration.
@@ -893,63 +893,98 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of sender-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level},
-    {'d', "disable-creation-time", NULL,
-     gettext_noop
-     ("disable adding the creation time to the metadata of the uploaded file"),
-     0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
-    {'D', "disable-extractor", NULL,
-     gettext_noop ("do not use libextractor to add keywords or metadata"),
-     0, &GNUNET_GETOPT_set_one, &disable_extractor},
-    {'e', "extract", NULL,
-     gettext_noop
-     ("print list of extracted keywords that would be used, but do not perform upload"),
-     0, &GNUNET_GETOPT_set_one, &extract_only},
-    {'k', "key", "KEYWORD",
-     gettext_noop
-     ("add an additional keyword for the top-level file or directory"
-      " (this option can be specified multiple times)"),
-     1, &GNUNET_FS_getopt_set_keywords, &topKeywords},
-    {'m', "meta", "TYPE:VALUE",
-     gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
-     1, &GNUNET_FS_getopt_set_metadata, &meta},
-    {'n', "noindex", NULL,
-     gettext_noop ("do not index, perform full insertion (stores entire "
-                   "file in encrypted form in GNUnet database)"),
-     0, &GNUNET_GETOPT_set_one, &do_insert},
-    {'N', "next", "ID",
-     gettext_noop
-     ("specify ID of an updated version to be published in the future"
-      " (for namespace insertions only)"),
-     1, &GNUNET_GETOPT_set_string, &next_id},
-    {'p', "priority", "PRIORITY",
-     gettext_noop ("specify the priority of the content"),
-     1, &GNUNET_GETOPT_set_uint, &bo.content_priority},
-    {'P', "pseudonym", "NAME",
-     gettext_noop
-     ("publish the files under the pseudonym NAME (place file into namespace)"),
-     1, &GNUNET_GETOPT_set_string, &pseudonym},
-    {'r', "replication", "LEVEL",
-     gettext_noop ("set the desired replication LEVEL"),
-     1, &GNUNET_GETOPT_set_uint, &bo.replication_level},
-    {'s', "simulate-only", NULL,
-     gettext_noop ("only simulate the process but do not do any "
-                   "actual publishing (useful to compute URIs)"),
-     0, &GNUNET_GETOPT_set_one, &do_simulate},
-    {'t', "this", "ID",
-     gettext_noop ("set the ID of this version of the publication"
-                   " (for namespace insertions only)"),
-     1, &GNUNET_GETOPT_set_string, &this_id},
-    {'u', "uri", "URI",
-     gettext_noop ("URI to be published (can be used instead of passing a "
-                   "file to add keywords to the file with the respective URI)"),
-     1, &GNUNET_GETOPT_set_string, &uri_string},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_uint ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of sender-anonymity"),
+                                   &bo.anonymity_level),
+
+    GNUNET_GETOPT_option_flag ('d',
+                                  "disable-creation-time",
+                                  gettext_noop ("disable adding the creation time to the "
+                                                "metadata of the uploaded file"),
+                                  &do_disable_creation_time),
+
+    GNUNET_GETOPT_option_flag ('D',
+                                  "disable-extractor",
+                                  gettext_noop ("do not use libextractor to add keywords or metadata"),
+                                  &disable_extractor),
+
+    GNUNET_GETOPT_option_flag ('e',
+                                  "extract",
+                                  gettext_noop ("print list of extracted keywords that would "
+                                                "be used, but do not perform upload"),
+                                  &extract_only),
+
+    GNUNET_FS_GETOPT_KEYWORDS ('k',
+                               "key",
+                               "KEYWORD",
+                               gettext_noop ("add an additional keyword for the top-level "
+                                             "file or directory (this option can be specified multiple times)"),
+                               &topKeywords),
+
+    GNUNET_FS_GETOPT_METADATA ('m',
+                               "meta",
+                               "TYPE:VALUE",
+                               gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
+                               &meta),
+
+    GNUNET_GETOPT_option_flag ('n',
+                                  "noindex",
+                                  gettext_noop ("do not index, perform full insertion (stores "
+                                                "entire file in encrypted form in GNUnet database)"),
+                                  &do_insert),
+
+    GNUNET_GETOPT_option_string ('N',
+                                 "next",
+                                 "ID",
+                                 gettext_noop ("specify ID of an updated version to be "
+                                               "published in the future (for namespace insertions only)"),
+                                 &next_id),
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "priority",
+                                   "PRIORITY",
+                                   gettext_noop ("specify the priority of the content"),
+                                   &bo.content_priority),
+
+    GNUNET_GETOPT_option_string ('P',
+                                 "pseudonym",
+                                 "NAME",
+                                 gettext_noop ("publish the files under the pseudonym "
+                                               "NAME (place file into namespace)"),
+                                 &pseudonym),
+
+    GNUNET_GETOPT_option_uint ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired replication LEVEL"),
+                                   &bo.replication_level),
+
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "simulate-only",
+                                  gettext_noop ("only simulate the process but do not do "
+                                                "any actual publishing (useful to compute URIs)"),
+                                  &do_simulate),
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "this",
+                                 "ID",
+                                 gettext_noop ("set the ID of this version of the publication "
+                                               "(for namespace insertions only)"),
+                                 &this_id),
+
+    GNUNET_GETOPT_option_string ('u',
+                                 "uri",
+                                 "URI",
+                                 gettext_noop ("URI to be published (can be used instead of passing a "
+                                               "file to add keywords to the file with the respective URI)"),
+                                 &uri_string), 
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
   bo.expiration_time =
diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c
index dfe6d0e75..ef1743436 100644
--- a/src/fs/gnunet-search.c
+++ b/src/fs/gnunet-search.c
@@ -51,7 +51,7 @@ static unsigned int results_limit;
 
 static unsigned int results;
 
-static int verbose;
+static unsigned int verbose;
 
 static int local_only;
 
@@ -305,26 +305,42 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of receiver-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity},
-    {'n', "no-network", NULL,
-     gettext_noop ("only search the local peer (no P2P network search)"),
-     0, &GNUNET_GETOPT_set_one, &local_only},
-    {'o', "output", "PREFIX",
-     gettext_noop ("write search results to file starting with PREFIX"),
-     1, &GNUNET_GETOPT_set_string, &output_filename},
-    {'t', "timeout", "DELAY",
-     gettext_noop ("automatically terminate search after DELAY"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
-    {'N', "results", "VALUE",
-     gettext_noop
-     ("automatically terminate search after VALUE results are found"),
-     1, &GNUNET_GETOPT_set_uint, &results_limit},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of receiver-anonymity"),
+                                   &anonymity),
+
+
+    GNUNET_GETOPT_option_flag ('n',
+                                  "no-network",
+                                  gettext_noop ("only search the local peer (no P2P network search)"),
+                                  &local_only),
+                                  
+    GNUNET_GETOPT_option_string ('o',
+                                 "output",
+                                 "PREFIX",
+                                 gettext_noop ("write search results to file starting with PREFIX"),
+                                 &output_filename),                              
+
+    GNUNET_GETOPT_option_relative_time ('t', 
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("automatically terminate search after DELAY"),
+                                            &timeout),
+
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
+    GNUNET_GETOPT_option_uint ('N',
+                                   "results",
+                                   "VALUE",
+                                   gettext_noop ("automatically terminate search "
+                                                 "after VALUE results are found"),
+                                   &results_limit),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index e38fdb032..09b1e05d8 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -38,7 +38,6 @@
 #include "gnunet_util_lib.h"
 #include "gnunet-service-fs_cp.h"
 #include "gnunet-service-fs_indexing.h"
-#include "gnunet-service-fs_lc.h"
 #include "gnunet-service-fs_pe.h"
 #include "gnunet-service-fs_pr.h"
 #include "gnunet-service-fs_push.h"
@@ -1177,7 +1176,6 @@ handle_client_unindex (void *cls,
 static void
 shutdown_task (void *cls)
 {
-  GSF_cadet_stop_client ();
   GSF_cadet_stop_server ();
   if (NULL != GSF_core)
   {
@@ -1320,7 +1318,6 @@ main_init (const struct GNUNET_CONFIGURATION_Handle *c)
                                     NULL);
   datastore_get_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE);
   GSF_cadet_start_server ();
-  GSF_cadet_start_client ();
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   return GNUNET_OK;
@@ -1351,7 +1348,7 @@ run (void *cls,
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO,
                                "fs",
                                "DATASTORE_QUEUE_SIZE");
-    dqs = 1024;
+    dqs = 32;
   }
   GSF_datastore_queue_size = (unsigned int) dqs;
   GSF_enable_randomized_delays =
diff --git a/src/fs/gnunet-service-fs_cadet.h b/src/fs/gnunet-service-fs_cadet.h
index 060a3993c..1fbd3a406 100644
--- a/src/fs/gnunet-service-fs_cadet.h
+++ b/src/fs/gnunet-service-fs_cadet.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -38,14 +38,15 @@ struct GSF_CadetRequest;
  * @param cls closure
  * @param type type of the block, ANY on error
  * @param expiration expiration time for the block
- * @param data_size number of bytes in 'data', 0 on error
+ * @param data_size number of bytes in @a data, 0 on error
  * @param data reply block data, NULL on error
  */
-typedef void (*GSF_CadetReplyProcessor)(void *cls,
-                                         enum GNUNET_BLOCK_Type type,
-                                         struct GNUNET_TIME_Absolute expiration,
-                                         size_t data_size,
-                                         const void *data);
+typedef void
+(*GSF_CadetReplyProcessor)(void *cls,
+                           enum GNUNET_BLOCK_Type type,
+                           struct GNUNET_TIME_Absolute expiration,
+                           size_t data_size,
+                           const void *data);
 
 
 /**
@@ -55,14 +56,28 @@ typedef void (*GSF_CadetReplyProcessor)(void *cls,
  * @param query hash to query for the block
  * @param type desired type for the block
  * @param proc function to call with result
- * @param proc_cls closure for 'proc'
+ * @param proc_cls closure for @a proc
  * @return handle to cancel the operation
  */
 struct GSF_CadetRequest *
 GSF_cadet_query (const struct GNUNET_PeerIdentity *target,
-                  const struct GNUNET_HashCode *query,
-                  enum GNUNET_BLOCK_Type type,
-                  GSF_CadetReplyProcessor proc, void *proc_cls);
+                 const struct GNUNET_HashCode *query,
+                 enum GNUNET_BLOCK_Type type,
+                 GSF_CadetReplyProcessor proc,
+                 void *proc_cls);
+
+/**
+ * Function called on each active cadets to shut them down.
+ *
+ * @param cls NULL
+ * @param key target peer, unused
+ * @param value the `struct CadetHandle` to destroy
+ * @return #GNUNET_YES (continue to iterate)
+ */
+int
+GSF_cadet_release_clients (void *cls,
+                           const struct GNUNET_PeerIdentity *key,
+                           void *value);
 
 
 /**
@@ -89,17 +104,15 @@ void
 GSF_cadet_stop_server (void);
 
 /**
- * Initialize subsystem for non-anonymous file-sharing.
+ * Cadet channel for creating outbound channels.
  */
-void
-GSF_cadet_start_client (void);
-
+extern struct GNUNET_CADET_Handle *cadet_handle;
 
 /**
- * Shutdown subsystem for non-anonymous file-sharing.
+ * Map from peer identities to 'struct CadetHandles' with cadet
+ * channels to those peers.
  */
-void
-GSF_cadet_stop_client (void);
+extern struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
 
 
 GNUNET_NETWORK_STRUCT_BEGIN
diff --git a/src/fs/gnunet-service-fs_cadet_client.c b/src/fs/gnunet-service-fs_cadet_client.c
index 4e268b93c..c729ebe41 100644
--- a/src/fs/gnunet-service-fs_cadet_client.c
+++ b/src/fs/gnunet-service-fs_cadet_client.c
@@ -77,7 +77,7 @@ struct GSF_CadetRequest
   GSF_CadetReplyProcessor proc;
 
   /**
-   * Closure for 'proc'
+   * Closure for @e proc
    */
   void *proc_cls;
 
@@ -126,11 +126,6 @@ struct CadetHandle
   struct GNUNET_CADET_Channel *channel;
 
   /**
-   * Handle for active write operation, or NULL.
-   */
-  struct GNUNET_CADET_TransmitHandle *wh;
-
-  /**
    * Which peer does this cadet go to?
    */
   struct GNUNET_PeerIdentity target;
@@ -140,14 +135,14 @@ struct CadetHandle
    * a few seconds to give the application a chance to give
    * us another query).
    */
-  struct GNUNET_SCHEDULER_Task * timeout_task;
+  struct GNUNET_SCHEDULER_Task *timeout_task;
 
   /**
    * Task to reset cadets that had errors (asynchronously,
    * as we may not be able to do it immediately during a
    * callback from the cadet API).
    */
-  struct GNUNET_SCHEDULER_Task * reset_task;
+  struct GNUNET_SCHEDULER_Task *reset_task;
 
 };
 
@@ -155,13 +150,13 @@ struct CadetHandle
 /**
  * Cadet channel for creating outbound channels.
  */
-static struct GNUNET_CADET_Handle *cadet_handle;
+struct GNUNET_CADET_Handle *cadet_handle;
 
 /**
  * Map from peer identities to 'struct CadetHandles' with cadet
  * channels to those peers.
  */
-static struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
+struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
 
 
 /* ********************* client-side code ************************* */
@@ -170,10 +165,10 @@ static struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
 /**
  * Transmit pending requests via the cadet.
  *
- * @param mh cadet to process
+ * @param cls `struct CadetHandle` to process
  */
 static void
-transmit_pending (struct CadetHandle *mh);
+transmit_pending (void *cls);
 
 
 /**
@@ -206,65 +201,19 @@ move_to_pending (void *cls,
 
 
 /**
- * We had a serious error, tear down and re-create cadet from scratch.
- *
- * @param mh cadet to reset
- */
-static void
-reset_cadet (struct CadetHandle *mh)
-{
-  struct GNUNET_CADET_Channel *channel = mh->channel;
-  struct GNUNET_HashCode port;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Resetting cadet channel to %s\n",
-              GNUNET_i2s (&mh->target));
-  mh->channel = NULL;
-
-  if (NULL != channel)
-  {
-    /* Avoid loop */
-    if (NULL != mh->wh)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (mh->wh);
-      mh->wh = NULL;
-    }
-    GNUNET_CADET_channel_destroy (channel);
-  }
-  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map,
-                                         &move_to_pending,
-                                         mh);
-  GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
-                      strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
-                      &port);
-  mh->channel = GNUNET_CADET_channel_create (cadet_handle,
-                                             mh,
-                                             &mh->target,
-                                             &port,
-                                             GNUNET_CADET_OPTION_RELIABLE);
-  transmit_pending (mh);
-}
-
-
-/**
- * Task called when it is time to destroy an inactive cadet channel.
+ * Functions with this signature are called whenever a complete reply
+ * is received.
  *
- * @param cls the `struct CadetHandle` to tear down
+ * @param cls closure with the `struct CadetHandle`
+ * @param srm the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
  */
-static void
-cadet_timeout (void *cls)
+static int
+check_reply (void *cls,
+             const struct CadetReplyMessage *srm)
 {
-  struct CadetHandle *mh = cls;
-  struct GNUNET_CADET_Channel *tun;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Timeout on cadet channel to %s\n",
-              GNUNET_i2s (&mh->target));
-  mh->timeout_task = NULL;
-  tun = mh->channel;
-  mh->channel = NULL;
-  if(NULL != tun)
-        GNUNET_CADET_channel_destroy (tun);
+  /* We check later... */
+  return GNUNET_OK;
 }
 
 
@@ -274,13 +223,7 @@ cadet_timeout (void *cls)
  * @param cls the `struct CadetHandle` to tear down
  */
 static void
-reset_cadet_task (void *cls)
-{
-  struct CadetHandle *mh = cls;
-
-  mh->reset_task = NULL;
-  reset_cadet (mh);
-}
+reset_cadet_task (void *cls);
 
 
 /**
@@ -300,83 +243,6 @@ reset_cadet_async (struct CadetHandle *mh)
 
 
 /**
- * Functions of this signature are called whenever we are ready to transmit
- * query via a cadet.
- *
- * @param cls the struct CadetHandle for which we did the write call
- * @param size the number of bytes that can be written to @a buf
- * @param buf where to write the message
- * @return number of bytes written to @a buf
- */
-static size_t
-transmit_sqm (void *cls,
-              size_t size,
-              void *buf)
-{
-  struct CadetHandle *mh = cls;
-  struct CadetQueryMessage sqm;
-  struct GSF_CadetRequest *sr;
-
-  mh->wh = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Cadet channel to %s failed during transmission attempt, rebuilding\n",
-                GNUNET_i2s (&mh->target));
-    reset_cadet_async (mh);
-    return 0;
-  }
-  sr = mh->pending_head;
-  if (NULL == sr)
-    return 0;
-  GNUNET_assert (size >= sizeof (struct CadetQueryMessage));
-  GNUNET_CONTAINER_DLL_remove (mh->pending_head,
-                               mh->pending_tail,
-                               sr);
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (mh->waiting_map,
-                                                    &sr->query,
-                                                    sr,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
-  sr->was_transmitted = GNUNET_YES;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending query for %s via cadet to %s\n",
-              GNUNET_h2s (&sr->query),
-              GNUNET_i2s (&mh->target));
-  sqm.header.size = htons (sizeof (sqm));
-  sqm.header.type = htons (GNUNET_MESSAGE_TYPE_FS_CADET_QUERY);
-  sqm.type = htonl (sr->type);
-  sqm.query = sr->query;
-  GNUNET_memcpy (buf, &sqm, sizeof (sqm));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Successfully transmitted %u bytes via cadet to %s\n",
-              (unsigned int) size,
-              GNUNET_i2s (&mh->target));
-  transmit_pending (mh);
-  return sizeof (sqm);
-}
-
-
-/**
- * Transmit pending requests via the cadet.
- *
- * @param mh cadet to process
- */
-static void
-transmit_pending (struct CadetHandle *mh)
-{
-  if (NULL == mh->channel)
-    return;
-  if (NULL != mh->wh)
-    return;
-  mh->wh = GNUNET_CADET_notify_transmit_ready (mh->channel, GNUNET_YES /* allow cork */,
-                                              GNUNET_TIME_UNIT_FOREVER_REL,
-                                              sizeof (struct CadetQueryMessage),
-                                              &transmit_sqm, mh);
-}
-
-
-/**
  * Closure for handle_reply().
  */
 struct HandleReplyClosure
@@ -393,7 +259,7 @@ struct HandleReplyClosure
   struct GNUNET_TIME_Absolute expiration;
 
   /**
-   * Number of bytes in 'data'.
+   * Number of bytes in @e data.
    */
   size_t data_size;
 
@@ -419,9 +285,9 @@ struct HandleReplyClosure
  * @return #GNUNET_YES (continue to iterate)
  */
 static int
-handle_reply (void *cls,
-              const struct GNUNET_HashCode *key,
-              void *value)
+process_reply (void *cls,
+               const struct GNUNET_HashCode *key,
+               void *value)
 {
   struct HandleReplyClosure *hrc = cls;
   struct GSF_CadetRequest *sr = value;
@@ -439,42 +305,52 @@ handle_reply (void *cls,
 
 
 /**
+ * Iterator called on each entry in a waiting map to
+ * call the 'proc' continuation and release associated
+ * resources.
+ *
+ * @param cls the `struct CadetHandle`
+ * @param key the key of the entry in the map (the query)
+ * @param value the `struct GSF_CadetRequest` to clean up
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+free_waiting_entry (void *cls,
+                    const struct GNUNET_HashCode *key,
+                    void *value)
+{
+  struct GSF_CadetRequest *sr = value;
+
+  GSF_cadet_query_cancel (sr);
+  return GNUNET_YES;
+}
+
+
+/**
  * Functions with this signature are called whenever a complete reply
  * is received.
  *
  * @param cls closure with the `struct CadetHandle`
- * @param channel channel handle
- * @param channel_ctx channel context
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ * @param srm the actual message
  */
-static int
-reply_cb (void *cls,
-          struct GNUNET_CADET_Channel *channel,
-          void **channel_ctx,
-          const struct GNUNET_MessageHeader *message)
+static void
+handle_reply (void *cls,
+              const struct CadetReplyMessage *srm)
 {
-  struct CadetHandle *mh = *channel_ctx;
-  const struct CadetReplyMessage *srm;
+  struct CadetHandle *mh = cls;
   struct HandleReplyClosure hrc;
   uint16_t msize;
   enum GNUNET_BLOCK_Type type;
   struct GNUNET_HashCode query;
 
-  msize = ntohs (message->size);
-  if (sizeof (struct CadetReplyMessage) > msize)
-  {
-    GNUNET_break_op (0);
-    reset_cadet_async (mh);
-    return GNUNET_SYSERR;
-  }
-  srm = (const struct CadetReplyMessage *) message;
-  msize -= sizeof (struct CadetReplyMessage);
+  msize = ntohs (srm->header.size) - sizeof (struct CadetReplyMessage);
   type = (enum GNUNET_BLOCK_Type) ntohl (srm->type);
   if (GNUNET_YES !=
       GNUNET_BLOCK_get_key (GSF_block_ctx,
                             type,
-                            &srm[1], msize, &query))
+                            &srm[1],
+                            msize,
+                            &query))
   {
     GNUNET_break_op (0);
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -483,13 +359,13 @@ reply_cb (void *cls,
                 msize,
                 GNUNET_i2s (&mh->target));
     reset_cadet_async (mh);
-    return GNUNET_SYSERR;
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received reply `%s' via cadet from peer %s\n",
               GNUNET_h2s (&query),
               GNUNET_i2s (&mh->target));
-  GNUNET_CADET_receive_done (channel);
+  GNUNET_CADET_receive_done (mh->channel);
   GNUNET_STATISTICS_update (GSF_stats,
                             gettext_noop ("# replies received via cadet"), 1,
                             GNUNET_NO);
@@ -500,16 +376,206 @@ reply_cb (void *cls,
   hrc.found = GNUNET_NO;
   GNUNET_CONTAINER_multihashmap_get_multiple (mh->waiting_map,
                                               &query,
-                                              &handle_reply,
+                                              &process_reply,
                                               &hrc);
   if (GNUNET_NO == hrc.found)
   {
     GNUNET_STATISTICS_update (GSF_stats,
                               gettext_noop ("# replies received via cadet dropped"), 1,
                               GNUNET_NO);
-    return GNUNET_OK;
   }
-  return GNUNET_OK;
+}
+
+
+/**
+ * Function called by cadet when a client disconnects.
+ * Cleans up our `struct CadetClient` of that channel.
+ *
+ * @param cls our `struct CadetClient`
+ * @param channel channel of the disconnecting client
+ */
+static void
+disconnect_cb (void *cls,
+               const struct GNUNET_CADET_Channel *channel)
+{
+  struct CadetHandle *mh = cls;
+  struct GSF_CadetRequest *sr;
+
+  if (NULL == mh->channel)
+    return; /* being destroyed elsewhere */
+  GNUNET_assert (channel == mh->channel);
+  mh->channel = NULL;
+  while (NULL != (sr = mh->pending_head))
+    GSF_cadet_query_cancel (sr);
+  /* first remove `mh` from the `cadet_map`, so that if the
+     callback from `free_waiting_entry()` happens to re-issue
+     the request, we don't immediately have it back in the
+     `waiting_map`. */
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multipeermap_remove (cadet_map,
+                                                       &mh->target,
+                                                       mh));
+  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map,
+                                         &free_waiting_entry,
+                                         mh);
+  if (NULL != mh->timeout_task)
+    GNUNET_SCHEDULER_cancel (mh->timeout_task);
+  if (NULL != mh->reset_task)
+    GNUNET_SCHEDULER_cancel (mh->reset_task);
+  GNUNET_assert (0 ==
+                 GNUNET_CONTAINER_multihashmap_size (mh->waiting_map));
+  GNUNET_CONTAINER_multihashmap_destroy (mh->waiting_map);
+  GNUNET_free (mh);
+}
+
+
+/**
+ * Function called whenever an MQ-channel's transmission window size changes.
+ *
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative..
+ */
+static void
+window_change_cb (void *cls,
+                  const struct GNUNET_CADET_Channel *channel,
+                  int window_size)
+{
+  /* FIXME: for flow control, implement? */
+#if 0
+  /* Something like this instead of the GNUNET_MQ_notify_sent() in
+     transmit_pending() might be good (once the window change CB works...) */
+  if (0 < window_size) /* test needed? */
+    transmit_pending (mh);
+#endif
+}
+
+
+/**
+ * We had a serious error, tear down and re-create cadet from scratch.
+ *
+ * @param mh cadet to reset
+ */
+static void
+reset_cadet (struct CadetHandle *mh)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Resetting cadet channel to %s\n",
+              GNUNET_i2s (&mh->target));
+  GNUNET_CADET_channel_destroy (mh->channel);
+  mh->channel = NULL;
+  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map,
+                                         &move_to_pending,
+                                         mh);
+  {
+    struct GNUNET_MQ_MessageHandler handlers[] = {
+      GNUNET_MQ_hd_var_size (reply,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY,
+                             struct CadetReplyMessage,
+                             mh),
+      GNUNET_MQ_handler_end ()
+    };
+    struct GNUNET_HashCode port;
+
+    GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
+                        strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
+                        &port);
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
+                                               mh,
+                                               &mh->target,
+                                               &port,
+                                               GNUNET_CADET_OPTION_RELIABLE,
+                                               &window_change_cb,
+                                               &disconnect_cb,
+                                               handlers);
+  }
+  transmit_pending (mh);
+}
+
+
+/**
+ * Task called when it is time to destroy an inactive cadet channel.
+ *
+ * @param cls the `struct CadetHandle` to tear down
+ */
+static void
+cadet_timeout (void *cls)
+{
+  struct CadetHandle *mh = cls;
+  struct GNUNET_CADET_Channel *tun;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Timeout on cadet channel to %s\n",
+              GNUNET_i2s (&mh->target));
+  mh->timeout_task = NULL;
+  tun = mh->channel;
+  mh->channel = NULL;
+  if (NULL != tun)
+    GNUNET_CADET_channel_destroy (tun);
+}
+
+
+/**
+ * Task called when it is time to reset an cadet.
+ *
+ * @param cls the `struct CadetHandle` to tear down
+ */
+static void
+reset_cadet_task (void *cls)
+{
+  struct CadetHandle *mh = cls;
+
+  mh->reset_task = NULL;
+  reset_cadet (mh);
+}
+
+
+/**
+ * Transmit pending requests via the cadet.
+ *
+ * @param cls `struct CadetHandle` to process
+ */
+static void
+transmit_pending (void *cls)
+{
+  struct CadetHandle *mh = cls;
+  struct GNUNET_MQ_Handle *mq = GNUNET_CADET_get_mq (mh->channel);
+  struct GSF_CadetRequest *sr;
+  struct GNUNET_MQ_Envelope *env;
+  struct CadetQueryMessage *sqm;
+
+  if ( (0 != GNUNET_MQ_get_length (mq)) ||
+       (NULL == (sr = mh->pending_head)) )
+    return;
+  GNUNET_CONTAINER_DLL_remove (mh->pending_head,
+                               mh->pending_tail,
+                               sr);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (mh->waiting_map,
+                                                    &sr->query,
+                                                    sr,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+  sr->was_transmitted = GNUNET_YES;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending query for %s via cadet to %s\n",
+              GNUNET_h2s (&sr->query),
+              GNUNET_i2s (&mh->target));
+  env = GNUNET_MQ_msg (sqm,
+                       GNUNET_MESSAGE_TYPE_FS_CADET_QUERY);
+  sqm->type = htonl (sr->type);
+  sqm->query = sr->query;
+  GNUNET_MQ_notify_sent (env,
+                         &transmit_pending,
+                         mh);
+  GNUNET_MQ_send (mq,
+                  env);
 }
 
 
@@ -522,7 +588,6 @@ static struct CadetHandle *
 get_cadet (const struct GNUNET_PeerIdentity *target)
 {
   struct CadetHandle *mh;
-  struct GNUNET_HashCode port;
 
   mh = GNUNET_CONTAINER_multipeermap_get (cadet_map,
                                           target);
@@ -549,17 +614,28 @@ get_cadet (const struct GNUNET_PeerIdentity *target)
                                                     &mh->target,
                                                     mh,
                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
-                      strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
-                      &port);
-  mh->channel = GNUNET_CADET_channel_create (cadet_handle,
-                                             mh,
-                                             &mh->target,
-                                             &port,
-                                             GNUNET_CADET_OPTION_RELIABLE);
-  GNUNET_assert (mh ==
-                 GNUNET_CONTAINER_multipeermap_get (cadet_map,
-                                                    target));
+  {
+    struct GNUNET_MQ_MessageHandler handlers[] = {
+      GNUNET_MQ_hd_var_size (reply,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY,
+                             struct CadetReplyMessage,
+                             mh),
+      GNUNET_MQ_handler_end ()
+    };
+    struct GNUNET_HashCode port;
+
+    GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
+                        strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
+                        &port);
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
+                                               mh,
+                                               &mh->target,
+                                               &port,
+                                               GNUNET_CADET_OPTION_RELIABLE,
+                                               &window_change_cb,
+                                               &disconnect_cb,
+                                               handlers);
+  }
   return mh;
 }
 
@@ -576,9 +652,10 @@ get_cadet (const struct GNUNET_PeerIdentity *target)
  */
 struct GSF_CadetRequest *
 GSF_cadet_query (const struct GNUNET_PeerIdentity *target,
-                const struct GNUNET_HashCode *query,
-                enum GNUNET_BLOCK_Type type,
-                GSF_CadetReplyProcessor proc, void *proc_cls)
+                 const struct GNUNET_HashCode *query,
+                 enum GNUNET_BLOCK_Type type,
+                 GSF_CadetReplyProcessor proc,
+                 void *proc_cls)
 {
   struct CadetHandle *mh;
   struct GSF_CadetRequest *sr;
@@ -646,93 +723,6 @@ GSF_cadet_query_cancel (struct GSF_CadetRequest *sr)
 
 
 /**
- * Iterator called on each entry in a waiting map to
- * call the 'proc' continuation and release associated
- * resources.
- *
- * @param cls the `struct CadetHandle`
- * @param key the key of the entry in the map (the query)
- * @param value the `struct GSF_CadetRequest` to clean up
- * @return #GNUNET_YES (continue to iterate)
- */
-static int
-free_waiting_entry (void *cls,
-                    const struct GNUNET_HashCode *key,
-                    void *value)
-{
-  struct GSF_CadetRequest *sr = value;
-
-  GSF_cadet_query_cancel (sr);
-  return GNUNET_YES;
-}
-
-
-/**
- * Function called by cadet when a client disconnects.
- * Cleans up our `struct CadetClient` of that channel.
- *
- * @param cls NULL
- * @param channel channel of the disconnecting client
- * @param channel_ctx our `struct CadetClient`
- */
-static void
-cleaner_cb (void *cls,
-            const struct GNUNET_CADET_Channel *channel,
-            void *channel_ctx)
-{
-  struct CadetHandle *mh = channel_ctx;
-  struct GSF_CadetRequest *sr;
-
-  if (NULL == mh->channel)
-    return; /* being destroyed elsewhere */
-  GNUNET_assert (channel == mh->channel);
-  mh->channel = NULL;
-  while (NULL != (sr = mh->pending_head))
-    GSF_cadet_query_cancel (sr);
-  /* first remove `mh` from the `cadet_map`, so that if the
-     callback from `free_waiting_entry()` happens to re-issue
-     the request, we don't immediately have it back in the
-     `waiting_map`. */
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multipeermap_remove (cadet_map,
-                                                       &mh->target,
-                                                       mh));
-  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map,
-                                         &free_waiting_entry,
-                                         mh);
-  if (NULL != mh->wh)
-    GNUNET_CADET_notify_transmit_ready_cancel (mh->wh);
-  if (NULL != mh->timeout_task)
-    GNUNET_SCHEDULER_cancel (mh->timeout_task);
-  if (NULL != mh->reset_task)
-    GNUNET_SCHEDULER_cancel (mh->reset_task);
-  GNUNET_assert (0 ==
-                 GNUNET_CONTAINER_multihashmap_size (mh->waiting_map));
-  GNUNET_CONTAINER_multihashmap_destroy (mh->waiting_map);
-  GNUNET_free (mh);
-}
-
-
-/**
- * Initialize subsystem for non-anonymous file-sharing.
- */
-void
-GSF_cadet_start_client ()
-{
-  static const struct GNUNET_CADET_MessageHandler handlers[] = {
-    { &reply_cb, GNUNET_MESSAGE_TYPE_FS_CADET_REPLY, 0 },
-    { NULL, 0, 0 }
-  };
-
-  cadet_map = GNUNET_CONTAINER_multipeermap_create (16, GNUNET_YES);
-  cadet_handle = GNUNET_CADET_connect (GSF_cfg,
-                                     NULL,
-                                     &cleaner_cb,
-                                     handlers);
-}
-
-
-/**
  * Function called on each active cadets to shut them down.
  *
  * @param cls NULL
@@ -740,10 +730,10 @@ GSF_cadet_start_client ()
  * @param value the `struct CadetHandle` to destroy
  * @return #GNUNET_YES (continue to iterate)
  */
-static int
-release_cadets (void *cls,
-               const struct GNUNET_PeerIdentity *key,
-               void *value)
+int
+GSF_cadet_release_clients (void *cls,
+                           const struct GNUNET_PeerIdentity *key,
+                           void *value)
 {
   struct CadetHandle *mh = value;
 
@@ -756,23 +746,5 @@ release_cadets (void *cls,
 }
 
 
-/**
- * Shutdown subsystem for non-anonymous file-sharing.
- */
-void
-GSF_cadet_stop_client ()
-{
-  GNUNET_CONTAINER_multipeermap_iterate (cadet_map,
-                                         &release_cadets,
-                                         NULL);
-  GNUNET_CONTAINER_multipeermap_destroy (cadet_map);
-  cadet_map = NULL;
-  if (NULL != cadet_handle)
-  {
-    GNUNET_CADET_disconnect (cadet_handle);
-    cadet_handle = NULL;
-  }
-}
-
 
 /* end of gnunet-service-fs_cadet_client.c */
diff --git a/src/fs/gnunet-service-fs_cadet_server.c b/src/fs/gnunet-service-fs_cadet_server.c
index ac86537c3..02ebb2ffe 100644
--- a/src/fs/gnunet-service-fs_cadet_server.c
+++ b/src/fs/gnunet-service-fs_cadet_server.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012, 2013 GNUnet e.V.
+     Copyright (C) 2012, 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -86,11 +86,6 @@ struct CadetClient
   struct GNUNET_CADET_Channel *channel;
 
   /**
-   * Handle for active write operation, or NULL.
-   */
-  struct GNUNET_CADET_TransmitHandle *wh;
-
-  /**
    * Head of write queue.
    */
   struct WriteQueueItem *wqi_head;
@@ -124,9 +119,9 @@ struct CadetClient
 
 
 /**
- * Listen channel for incoming requests.
+ * Listen port for incoming requests.
  */
-static struct GNUNET_CADET_Handle *listen_channel;
+static struct GNUNET_CADET_Port *cadet_port;
 
 /**
  * Head of DLL of cadet clients.
@@ -188,121 +183,29 @@ refresh_timeout_task (struct CadetClient *sc)
 
 
 /**
- * We're done handling a request from a client, read the next one.
+ * Check if we are done with the write queue, and if so tell CADET
+ * that we are ready to read more.
  *
- * @param sc client to continue reading requests from
+ * @param cls where to process the write queue
  */
 static void
-continue_reading (struct CadetClient *sc)
-{
-  refresh_timeout_task (sc);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Finished processing cadet request from client %p, ready to receive the next one\n",
-              sc);
-  GNUNET_CADET_receive_done (sc->channel);
-}
-
-
-/**
- * Transmit the next entry from the write queue.
- *
- * @param sc where to process the write queue
- */
-static void
-continue_writing (struct CadetClient *sc);
-
-
-/**
- * Send a reply now, cadet is ready.
- *
- * @param cls closure with the `struct CadetClient` which sent the query
- * @param size number of bytes available in @a buf
- * @param buf where to write the message
- * @return number of bytes written to @a buf
- */
-static size_t
-write_continuation (void *cls,
-                    size_t size,
-                    void *buf)
+continue_writing (void *cls)
 {
   struct CadetClient *sc = cls;
-  struct GNUNET_CADET_Channel *tun;
-  struct WriteQueueItem *wqi;
-  size_t ret;
+  struct GNUNET_MQ_Handle *mq;
 
-  sc->wh = NULL;
-  if (NULL == (wqi = sc->wqi_head))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Write queue empty, reading more requests\n");
-    return 0;
-  }
-  if ( (0 == size) ||
-       (size < wqi->msize) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Transmission of reply failed, terminating cadet\n");
-    tun = sc->channel;
-    sc->channel = NULL;
-    GNUNET_CADET_channel_destroy (tun);
-    return 0;
-  }
-  GNUNET_CONTAINER_DLL_remove (sc->wqi_head,
-                               sc->wqi_tail,
-                               wqi);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Transmitted %u byte reply via cadet to %p\n",
-              (unsigned int) size,
-              sc);
-  GNUNET_STATISTICS_update (GSF_stats,
-                            gettext_noop ("# Blocks transferred via cadet"), 1,
-                            GNUNET_NO);
-  ret = wqi->msize;
-  GNUNET_memcpy (buf, &wqi[1], ret);
-  GNUNET_free (wqi);
-  continue_writing (sc);
-  return ret;
-}
-
-
-/**
- * Transmit the next entry from the write queue.
- *
- * @param sc where to process the write queue
- */
-static void
-continue_writing (struct CadetClient *sc)
-{
-  struct WriteQueueItem *wqi;
-  struct GNUNET_CADET_Channel *tun;
-
-  if (NULL != sc->wh)
+  mq = GNUNET_CADET_get_mq (sc->channel);
+  if (0 != GNUNET_MQ_get_length (mq))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Write pending, waiting for it to complete\n");
-    return; /* write already pending */
-  }
-  if (NULL == (wqi = sc->wqi_head))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Write queue empty, reading more requests\n");
-    continue_reading (sc);
-    return;
-  }
-  sc->wh = GNUNET_CADET_notify_transmit_ready (sc->channel, GNUNET_NO,
-                                              GNUNET_TIME_UNIT_FOREVER_REL,
-                                              wqi->msize,
-                                              &write_continuation,
-                                              sc);
-  if (NULL == sc->wh)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Write failed; terminating cadet\n");
-    tun = sc->channel;
-    sc->channel = NULL;
-    GNUNET_CADET_channel_destroy (tun);
     return;
   }
+  refresh_timeout_task (sc);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished processing cadet request from client %p, ready to receive the next one\n",
+              sc);
+  GNUNET_CADET_receive_done (sc->channel);
 }
 
 
@@ -316,24 +219,26 @@ continue_writing (struct CadetClient *sc)
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
  */
 static void
 handle_datastore_reply (void *cls,
-                        const struct GNUNET_HashCode *key,
-                        size_t size,
-                        const void *data,
-                        enum GNUNET_BLOCK_Type type,
-                        uint32_t priority,
-                        uint32_t anonymity,
-                        struct GNUNET_TIME_Absolute expiration,
+                        const struct GNUNET_HashCode *key,
+                        size_t size,
+                        const void *data,
+                        enum GNUNET_BLOCK_Type type,
+                        uint32_t priority,
+                        uint32_t anonymity,
+                        uint32_t replication,
+                        struct GNUNET_TIME_Absolute expiration,
                         uint64_t uid)
 {
   struct CadetClient *sc = cls;
   size_t msize = size + sizeof (struct CadetReplyMessage);
-  struct WriteQueueItem *wqi;
+  struct GNUNET_MQ_Envelope *env;
   struct CadetReplyMessage *srm;
 
   sc->qe = NULL;
@@ -357,7 +262,8 @@ handle_datastore_reply (void *cls,
                   GNUNET_h2s (key));
     }
     GNUNET_STATISTICS_update (GSF_stats,
-                              gettext_noop ("# queries received via CADET not answered"), 1,
+                              gettext_noop ("# queries received via CADET not answered"),
+                              1,
                               GNUNET_NO);
     continue_writing (sc);
     return;
@@ -369,11 +275,16 @@ handle_datastore_reply (void *cls,
                 GNUNET_h2s (key));
     if (GNUNET_OK !=
         GNUNET_FS_handle_on_demand_block (key,
-                                          size, data, type,
-                                          priority, anonymity,
-                                          expiration, uid,
-                                          &handle_datastore_reply,
-                                          sc))
+                                    size,
+                                    data,
+                                    type,
+                                    priority,
+                                    anonymity,
+                                    replication,
+                                    expiration,
+                                    uid,
+                                    &handle_datastore_reply,
+                                    sc))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "On-demand encoding request failed\n");
@@ -381,7 +292,7 @@ handle_datastore_reply (void *cls,
     }
     return;
   }
-  if (msize > GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize > GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     continue_writing (sc);
@@ -394,19 +305,23 @@ handle_datastore_reply (void *cls,
               (unsigned int) type,
               GNUNET_h2s (key),
               sc);
-  wqi = GNUNET_malloc (sizeof (struct WriteQueueItem) + msize);
-  wqi->msize = msize;
-  srm = (struct CadetReplyMessage *) &wqi[1];
-  srm->header.size = htons ((uint16_t) msize);
-  srm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_CADET_REPLY);
+  env = GNUNET_MQ_msg_extra (srm,
+                             size,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY);
   srm->type = htonl (type);
   srm->expiration = GNUNET_TIME_absolute_hton (expiration);
-  GNUNET_memcpy (&srm[1], data, size);
-  sc->reply_size = msize;
-  GNUNET_CONTAINER_DLL_insert (sc->wqi_head,
-                               sc->wqi_tail,
-                               wqi);
-  continue_writing (sc);
+  GNUNET_memcpy (&srm[1],
+                 data,
+                 size);
+  GNUNET_MQ_notify_sent (env,
+                         &continue_writing,
+                         sc);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Blocks transferred via cadet"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (sc->channel),
+                  env);
 }
 
 
@@ -414,46 +329,39 @@ handle_datastore_reply (void *cls,
  * Functions with this signature are called whenever a
  * complete query message is received.
  *
- * Do not call #GNUNET_SERVER_mst_destroy() in callback
- *
  * @param cls closure with the `struct CadetClient`
- * @param channel channel handle
- * @param channel_ctx channel context
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ * @param sqm the actual message
  */
-static int
-request_cb (void *cls,
-            struct GNUNET_CADET_Channel *channel,
-            void **channel_ctx,
-            const struct GNUNET_MessageHeader *message)
+static void
+handle_request (void *cls,
+                const struct CadetQueryMessage *sqm)
 {
-  struct CadetClient *sc = *channel_ctx;
-  const struct CadetQueryMessage *sqm;
+  struct CadetClient *sc = cls;
 
-  sqm = (const struct CadetQueryMessage *) message;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received query for `%s' via cadet from client %p\n",
               GNUNET_h2s (&sqm->query),
               sc);
   GNUNET_STATISTICS_update (GSF_stats,
-                            gettext_noop ("# queries received via cadet"), 1,
+                            gettext_noop ("# queries received via cadet"),
+                            1,
                             GNUNET_NO);
   refresh_timeout_task (sc);
   sc->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
-                                     0,
-                                     &sqm->query,
-                                     ntohl (sqm->type),
-                                     0 /* priority */,
-                                     GSF_datastore_queue_size,
-                                     &handle_datastore_reply, sc);
+                                     0 /* next_uid */,
+                                     false /* random */,
+                                     &sqm->query,
+                                     ntohl (sqm->type),
+                                     0 /* priority */,
+                                     GSF_datastore_queue_size,
+                                     &handle_datastore_reply,
+                                     sc);
   if (NULL == sc->qe)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Queueing request with datastore failed (queue full?)\n");
     continue_writing (sc);
   }
-  return GNUNET_OK;
 }
 
 
@@ -464,16 +372,12 @@ request_cb (void *cls,
  * @param channel the channel representing the cadet
  * @param initiator the identity of the peer who wants to establish a cadet
  *            with us; NULL on binding error
- * @param port cadet port used for the incoming connection
- * @param options channel option flags
- * @return initial channel context (our 'struct CadetClient')
+ * @return initial channel context (our `struct CadetClient`)
  */
 static void *
-accept_cb (void *cls,
-           struct GNUNET_CADET_Channel *channel,
-           const struct GNUNET_PeerIdentity *initiator,
-           const struct GNUNET_HashCode *port,
-           enum GNUNET_CADET_ChannelOption options)
+connect_cb (void *cls,
+            struct GNUNET_CADET_Channel *channel,
+            const struct GNUNET_PeerIdentity *initiator)
 {
   struct CadetClient *sc;
 
@@ -481,13 +385,15 @@ accept_cb (void *cls,
   if (sc_count >= sc_count_max)
   {
     GNUNET_STATISTICS_update (GSF_stats,
-                              gettext_noop ("# cadet client connections rejected"), 1,
+                              gettext_noop ("# cadet client connections rejected"),
+                              1,
                               GNUNET_NO);
     GNUNET_CADET_channel_destroy (channel);
     return NULL;
   }
   GNUNET_STATISTICS_update (GSF_stats,
-                            gettext_noop ("# cadet connections active"), 1,
+                            gettext_noop ("# cadet connections active"),
+                            1,
                             GNUNET_NO);
   sc = GNUNET_new (struct CadetClient);
   sc->channel = channel;
@@ -506,18 +412,17 @@ accept_cb (void *cls,
 
 /**
  * Function called by cadet when a client disconnects.
- * Cleans up our 'struct CadetClient' of that channel.
+ * Cleans up our `struct CadetClient` of that channel.
  *
- * @param cls NULL
+ * @param cls  our `struct CadetClient`
  * @param channel channel of the disconnecting client
- * @param channel_ctx our 'struct CadetClient'
+ * @param channel_ctx
  */
 static void
-cleaner_cb (void *cls,
-            const struct GNUNET_CADET_Channel *channel,
-            void *channel_ctx)
+disconnect_cb (void *cls,
+               const struct GNUNET_CADET_Channel *channel)
 {
-  struct CadetClient *sc = channel_ctx;
+  struct CadetClient *sc = cls;
   struct WriteQueueItem *wqi;
 
   if (NULL == sc)
@@ -533,8 +438,6 @@ cleaner_cb (void *cls,
     GNUNET_SCHEDULER_cancel (sc->terminate_task);
   if (NULL != sc->timeout_task)
     GNUNET_SCHEDULER_cancel (sc->timeout_task);
-  if (NULL != sc->wh)
-    GNUNET_CADET_notify_transmit_ready_cancel (sc->wh);
   if (NULL != sc->qe)
     GNUNET_DATASTORE_cancel (sc->qe);
   while (NULL != (wqi = sc->wqi_head))
@@ -553,14 +456,40 @@ cleaner_cb (void *cls,
 
 
 /**
+ * Function called whenever an MQ-channel's transmission window size changes.
+ *
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative..
+ */
+static void
+window_change_cb (void *cls,
+                  const struct GNUNET_CADET_Channel *channel,
+                  int window_size)
+{
+  /* FIXME: could do flow control here... */
+}
+
+
+/**
  * Initialize subsystem for non-anonymous file-sharing.
  */
 void
 GSF_cadet_start_server ()
 {
-  static const struct GNUNET_CADET_MessageHandler handlers[] = {
-    { &request_cb, GNUNET_MESSAGE_TYPE_FS_CADET_QUERY, sizeof (struct CadetQueryMessage)},
-    { NULL, 0, 0 }
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (request,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_QUERY,
+                             struct CadetQueryMessage,
+                             NULL),
+    GNUNET_MQ_handler_end ()
   };
   struct GNUNET_HashCode port;
 
@@ -573,18 +502,19 @@ GSF_cadet_start_server ()
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Initializing cadet FS server with a limit of %llu connections\n",
               sc_count_max);
-  listen_channel = GNUNET_CADET_connect (GSF_cfg,
-                                         NULL,
-                                         &cleaner_cb,
-                                         handlers);
-  GNUNET_assert (NULL != listen_channel);
+  cadet_map = GNUNET_CONTAINER_multipeermap_create (16, GNUNET_YES);
+  cadet_handle = GNUNET_CADET_connect (GSF_cfg);
+  GNUNET_assert (NULL != cadet_handle);
   GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
                       strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
                       &port);
-  GNUNET_CADET_open_port (listen_channel,
-                          &port,
-                          &accept_cb,
-                          NULL);
+  cadet_port = GNUNET_CADET_open_port (cadet_handle,
+                                       &port,
+                                       &connect_cb,
+                                       NULL,
+                                       &window_change_cb,
+                                       &disconnect_cb,
+                                       handlers);
 }
 
 
@@ -594,10 +524,20 @@ GSF_cadet_start_server ()
 void
 GSF_cadet_stop_server ()
 {
-  if (NULL != listen_channel)
+  GNUNET_CONTAINER_multipeermap_iterate (cadet_map,
+                                         &GSF_cadet_release_clients,
+                                         NULL);
+  GNUNET_CONTAINER_multipeermap_destroy (cadet_map);
+  cadet_map = NULL;
+  if (NULL != cadet_port)
+  {
+    GNUNET_CADET_close_port (cadet_port);
+    cadet_port = NULL;
+  }
+  if (NULL != cadet_handle)
   {
-    GNUNET_CADET_disconnect (listen_channel);
-    listen_channel = NULL;
+    GNUNET_CADET_disconnect (cadet_handle);
+    cadet_handle = NULL;
   }
   GNUNET_assert (NULL == sc_head);
   GNUNET_assert (0 == sc_count);
diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c
index 3f7783ded..817aed257 100644
--- a/src/fs/gnunet-service-fs_cp.c
+++ b/src/fs/gnunet-service-fs_cp.c
@@ -859,7 +859,7 @@ handle_p2p_reply (void *cls,
   size_t msize;
 
   GNUNET_assert (data_len + sizeof (struct PutMessage) <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   GNUNET_assert (peerreq->pr == pr);
   prd = GSF_pending_request_get_data_ (pr);
   if (NULL == data)
@@ -883,7 +883,7 @@ handle_p2p_reply (void *cls,
                             gettext_noop ("# replies received for other peers"),
                             1, GNUNET_NO);
   msize = sizeof (struct PutMessage) + data_len;
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
index 3ce68f487..b5e8af95f 100644
--- a/src/fs/gnunet-service-fs_indexing.c
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -266,6 +266,7 @@ remove_cont (void *cls, int success,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -280,6 +281,7 @@ GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode * key,
                                   enum GNUNET_BLOCK_Type type,
                                   uint32_t priority,
                                   uint32_t anonymity,
+                                  uint32_t replication,
                                   struct GNUNET_TIME_Absolute expiration,
                                   uint64_t uid,
                                   GNUNET_DATASTORE_DatumProcessor cont,
@@ -412,6 +414,7 @@ GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode * key,
         GNUNET_BLOCK_TYPE_FS_DBLOCK,
         priority,
         anonymity,
+        replication,
         expiration,
         uid);
   return GNUNET_OK;
@@ -438,7 +441,7 @@ GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq)
     fn = pos->filename;
     slen = strlen (fn) + 1;
     if (slen + sizeof (struct IndexInfoMessage) >=
-        GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        GNUNET_MAX_MESSAGE_SIZE)
     {
       GNUNET_break (0);
       break;
diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h
index 8b861e3f7..331c51105 100644
--- a/src/fs/gnunet-service-fs_indexing.h
+++ b/src/fs/gnunet-service-fs_indexing.h
@@ -47,6 +47,7 @@
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -55,9 +56,13 @@
  * @return #GNUNET_OK on success
  */
 int
-GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode * key, uint32_t size,
-                                  const void *data, enum GNUNET_BLOCK_Type type,
-                                  uint32_t priority, uint32_t anonymity,
+GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key,
+                                  uint32_t size,
+                                  const void *data,
+                                  enum GNUNET_BLOCK_Type type,
+                                  uint32_t priority,
+                                  uint32_t anonymity,
+                                  uint32_t replication,
                                   struct GNUNET_TIME_Absolute expiration,
                                   uint64_t uid,
                                   GNUNET_DATASTORE_DatumProcessor cont,
diff --git a/src/fs/gnunet-service-fs_lc.c b/src/fs/gnunet-service-fs_lc.c
deleted file mode 100644
index 9ffd6cadd..000000000
--- a/src/fs/gnunet-service-fs_lc.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file fs/gnunet-service-fs_lc.c
- * @brief API to handle 'local clients'
- * @author Christian Grothoff
- */
-#include "platform.h"
-#include "gnunet-service-fs.h"
-#include "gnunet-service-fs_lc.h"
-#include "gnunet-service-fs_cp.h"
-#include "gnunet-service-fs_pr.h"
-
-
-
-/* end of gnunet-service-fs_lc.c */
diff --git a/src/fs/gnunet-service-fs_lc.h b/src/fs/gnunet-service-fs_lc.h
deleted file mode 100644
index 6671ed33c..000000000
--- a/src/fs/gnunet-service-fs_lc.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file fs/gnunet-service-fs_lc.h
- * @brief API to handle 'local clients'
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_FS_LC_H
-#define GNUNET_SERVICE_FS_LC_H
-
-#include "gnunet-service-fs.h"
-
-
-#endif
-/* end of gnunet-service-fs_lc.h */
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index 63462f7dc..7c64ab98b 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -97,9 +97,9 @@ struct GSF_PendingRequest
   struct GNUNET_HashCode *replies_seen;
 
   /**
-   * Bloomfilter masking replies we've already seen.
+   * Block group for filtering replies we've already seen.
    */
-  struct GNUNET_CONTAINER_BloomFilter *bf;
+  struct GNUNET_BLOCK_Group *bg;
 
   /**
    * Entry for this pending request in the expiration heap, or NULL.
@@ -160,20 +160,27 @@ struct GSF_PendingRequest
   struct GNUNET_SCHEDULER_Task * warn_task;
 
   /**
-   * Current offset for querying our local datastore for results.
-   * Starts at a random value, incremented until we get the same
-   * UID again (detected using 'first_uid'), which is then used
-   * to termiante the iteration.
+   * Do we have a first UID yet?
+   */
+  bool have_first_uid;
+
+  /**
+   * Have we seen a NULL result yet?
    */
-  uint64_t local_result_offset;
+  bool seen_null;
 
   /**
    * Unique ID of the first result from the local datastore;
-   * used to detect wrap-around of the offset.
+   * used to terminate the loop.
    */
   uint64_t first_uid;
 
   /**
+   * Result count.
+   */
+  size_t result_count;
+
+  /**
    * How often have we retried this request via 'cadet'?
    * (used to bound overall retries).
    */
@@ -189,16 +196,6 @@ struct GSF_PendingRequest
    */
   unsigned int replies_seen_size;
 
-  /**
-   * Mingle value we currently use for the bf.
-   */
-  uint32_t mingle;
-
-  /**
-   * Do we have a first UID yet?
-   */
-  unsigned int have_first_uid;
-
 };
 
 
@@ -248,18 +245,36 @@ static unsigned long long max_pending_requests = (32 * 1024);
  * fresh one of minimal size without problems) OR if our peer is the
  * initiator (in which case we may resize to larger than mimimum size).
  *
+ * @param type type of the request
  * @param pr request for which the BF is to be recomputed
  */
 static void
-refresh_bloomfilter (struct GSF_PendingRequest *pr)
+refresh_bloomfilter (enum GNUNET_BLOCK_Type type,
+                     struct GSF_PendingRequest *pr)
 {
-  if (pr->bf != NULL)
-    GNUNET_CONTAINER_bloomfilter_free (pr->bf);
-  pr->mingle =
-      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX);
-  pr->bf =
-      GNUNET_BLOCK_construct_bloomfilter (pr->mingle, pr->replies_seen,
-                                          pr->replies_seen_count);
+  if (NULL != pr->bg)
+  {
+    GNUNET_BLOCK_group_destroy (pr->bg);
+    pr->bg = NULL;
+  }
+  if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type)
+    return; /* no need */
+  pr->bg
+    = GNUNET_BLOCK_group_create (GSF_block_ctx,
+                                 type,
+                                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                                           UINT32_MAX),
+                                 NULL,
+                                 0,
+                                 "seen-set-size",
+                                 pr->replies_seen_count,
+                                 NULL);
+  if (NULL == pr->bg)
+    return;
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_BLOCK_group_set_seen (pr->bg,
+                                             pr->replies_seen,
+                                             pr->replies_seen_count));
 }
 
 
@@ -319,8 +334,6 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
   if (NULL != target)
     extra += sizeof (struct GNUNET_PeerIdentity);
   pr = GNUNET_malloc (sizeof (struct GSF_PendingRequest) + extra);
-  pr->local_result_offset =
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
   pr->public_data.query = *query;
   eptr = (struct GNUNET_HashCode *) &pr[1];
   if (NULL != target)
@@ -355,25 +368,31 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
   if (replies_seen_count > 0)
   {
     pr->replies_seen_size = replies_seen_count;
-    pr->replies_seen =
-        GNUNET_malloc (sizeof (struct GNUNET_HashCode) * pr->replies_seen_size);
+    pr->replies_seen = GNUNET_new_array (pr->replies_seen_size,
+                                         struct GNUNET_HashCode);
     GNUNET_memcpy (pr->replies_seen,
-            replies_seen,
-            replies_seen_count * sizeof (struct GNUNET_HashCode));
+                   replies_seen,
+                   replies_seen_count * sizeof (struct GNUNET_HashCode));
     pr->replies_seen_count = replies_seen_count;
   }
-  if (NULL != bf_data)
+  if ( (NULL != bf_data) &&
+       (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type) )
   {
-    pr->bf =
-        GNUNET_CONTAINER_bloomfilter_init (bf_data,
-                                           bf_size,
-                                           GNUNET_CONSTANTS_BLOOMFILTER_K);
-    pr->mingle = mingle;
+    pr->bg
+      =  GNUNET_BLOCK_group_create (GSF_block_ctx,
+                                    pr->public_data.type,
+                                    mingle,
+                                    bf_data,
+                                    bf_size,
+                                    "seen-set-size",
+                                    0,
+                                    NULL);
   }
   else if ((replies_seen_count > 0) &&
            (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)))
   {
-    refresh_bloomfilter (pr);
+    refresh_bloomfilter (pr->public_data.type,
+                         pr);
   }
   GNUNET_CONTAINER_multihashmap_put (pr_map,
                                      &pr->public_data.query,
@@ -461,46 +480,37 @@ GSF_pending_request_update_ (struct GSF_PendingRequest *pr,
                              const struct GNUNET_HashCode * replies_seen,
                              unsigned int replies_seen_count)
 {
-  unsigned int i;
-  struct GNUNET_HashCode mhash;
-
   if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count)
     return;                     /* integer overflow */
   if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))
   {
     /* we're responsible for the BF, full refresh */
     if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size)
-      GNUNET_array_grow (pr->replies_seen, pr->replies_seen_size,
+      GNUNET_array_grow (pr->replies_seen,
+                         pr->replies_seen_size,
                          replies_seen_count + pr->replies_seen_count);
-    GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count], replies_seen,
-            sizeof (struct GNUNET_HashCode) * replies_seen_count);
+    GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count],
+                   replies_seen,
+                   sizeof (struct GNUNET_HashCode) * replies_seen_count);
     pr->replies_seen_count += replies_seen_count;
-    refresh_bloomfilter (pr);
+    refresh_bloomfilter (pr->public_data.type,
+                         pr);
   }
   else
   {
-    if (NULL == pr->bf)
+    if (NULL == pr->bg)
     {
       /* we're not the initiator, but the initiator did not give us
        * any bloom-filter, so we need to create one on-the-fly */
-      pr->mingle =
-          GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                    UINT32_MAX);
-      pr->bf =
-          GNUNET_BLOCK_construct_bloomfilter (pr->mingle,
-                                              replies_seen,
-                                              replies_seen_count);
+      refresh_bloomfilter (pr->public_data.type,
+                           pr);
     }
     else
     {
-      for (i = 0; i < pr->replies_seen_count; i++)
-      {
-        GNUNET_BLOCK_mingle_hash (&replies_seen[i],
-                                  pr->mingle,
-                                  &mhash);
-        GNUNET_CONTAINER_bloomfilter_add (pr->bf,
-                                          &mhash);
-      }
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_BLOCK_group_set_seen (pr->bg,
+                                                 replies_seen,
+                                                 pr->replies_seen_count));
     }
   }
   if (NULL != pr->gh)
@@ -530,6 +540,8 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
   struct GNUNET_TIME_Absolute now;
   int64_t ttl;
   int do_route;
+  void *bf_data;
+  uint32_t bf_nonce;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Building request message for `%s' of type %d\n",
@@ -553,7 +565,15 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
     bm |= GET_MESSAGE_BIT_TRANSMIT_TO;
     k++;
   }
-  bf_size = GNUNET_CONTAINER_bloomfilter_get_size (pr->bf);
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_group_serialize (pr->bg,
+                                    &bf_nonce,
+                                    &bf_data,
+                                    &bf_size))
+  {
+    bf_size = 0;
+    bf_data = NULL;
+  }
   env = GNUNET_MQ_msg_extra (gm,
                              bf_size + k * sizeof (struct GNUNET_PeerIdentity),
                              GNUNET_MESSAGE_TYPE_FS_GET);
@@ -571,7 +591,7 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
   now = GNUNET_TIME_absolute_get ();
   ttl = (int64_t) (pr->public_data.ttl.abs_value_us - now.abs_value_us);
   gm->ttl = htonl (ttl / 1000LL / 1000LL);
-  gm->filter_mutator = htonl (pr->mingle);
+  gm->filter_mutator = htonl (bf_nonce);
   gm->hash_bitmap = htonl (bm);
   gm->query = pr->public_data.query;
   ext = (struct GNUNET_PeerIdentity *) &gm[1];
@@ -581,11 +601,10 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
                          &ext[k++]);
   if (NULL != pr->public_data.target)
     ext[k++] = *pr->public_data.target;
-  if (NULL != pr->bf)
-    GNUNET_assert (GNUNET_SYSERR !=
-                   GNUNET_CONTAINER_bloomfilter_get_raw_data (pr->bf,
-                                                              (char *) &ext[k],
-                                                              bf_size));
+  GNUNET_memcpy (&ext[k],
+                 bf_data,
+                 bf_size);
+  GNUNET_free_non_null (bf_data);
   return env;
 }
 
@@ -624,11 +643,8 @@ clean_request (void *cls,
   }
   GSF_plan_notify_request_done_ (pr);
   GNUNET_free_non_null (pr->replies_seen);
-  if (NULL != pr->bf)
-  {
-    GNUNET_CONTAINER_bloomfilter_free (pr->bf);
-    pr->bf = NULL;
-  }
+  GNUNET_BLOCK_group_destroy (pr->bg);
+  pr->bg = NULL;
   GNUNET_PEER_change_rc (pr->sender_pid, -1);
   pr->sender_pid = 0;
   GNUNET_PEER_change_rc (pr->origin_pid, -1);
@@ -844,10 +860,9 @@ process_reply (void *cls,
   prq->eval =
       GNUNET_BLOCK_evaluate (GSF_block_ctx,
                              prq->type,
+                             pr->bg,
                              prq->eo,
                              key,
-                             &pr->bf,
-                             pr->mingle,
                              NULL,
                              0,
                              prq->data,
@@ -1325,6 +1340,124 @@ odc_warn_delay_task (void *cls)
 }
 
 
+/* Call our continuation (if we have any) */
+static void
+call_continuation (struct GSF_PendingRequest *pr)
+{
+  GSF_LocalLookupContinuation cont = pr->llc_cont;
+
+  GNUNET_assert (NULL == pr->qe);
+  if (NULL != pr->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (pr->warn_task);
+    pr->warn_task = NULL;
+  }
+  if (NULL == cont)
+    return;                     /* no continuation */
+  pr->llc_cont = NULL;
+  if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options))
+  {
+    if (GNUNET_BLOCK_EVALUATION_OK_LAST != pr->local_result)
+    {
+      /* Signal that we are done and that there won't be any
+         additional results to allow client to clean up state. */
+      pr->rh (pr->rh_cls,
+               GNUNET_BLOCK_EVALUATION_OK_LAST,
+               pr,
+               UINT32_MAX,
+               GNUNET_TIME_UNIT_ZERO_ABS,
+               GNUNET_TIME_UNIT_FOREVER_ABS,
+               GNUNET_BLOCK_TYPE_ANY,
+               NULL,
+               0);
+    }
+    /* Finally, call our continuation to signal that we are
+       done with local processing of this request; i.e. to
+       start reading again from the client. */
+    cont (pr->llc_cont_cls, NULL, GNUNET_BLOCK_EVALUATION_OK_LAST);
+    return;
+  }
+
+  cont (pr->llc_cont_cls, pr, pr->local_result);
+}
+
+
+/* Update stats and call continuation */
+static void
+no_more_local_results (struct GSF_PendingRequest *pr)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
+              "No further local responses available.\n");
+#if INSANE_STATISTICS
+  if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) ||
+       (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type) )
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# requested DBLOCK or IBLOCK not found"),
+                              1,
+                              GNUNET_NO);
+#endif
+  call_continuation (pr);
+}
+
+
+/* forward declaration */
+static void
+process_local_reply (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     size_t size,
+                     const void *data,
+                     enum GNUNET_BLOCK_Type type,
+                     uint32_t priority,
+                     uint32_t anonymity,
+                     uint32_t replication,
+                     struct GNUNET_TIME_Absolute expiration,
+                     uint64_t uid);
+
+
+/* Start a local query */
+static void
+start_local_query (struct GSF_PendingRequest *pr,
+                   uint64_t next_uid,
+                   bool random)
+{
+  pr->qe_start = GNUNET_TIME_absolute_get ();
+  pr->warn_task =
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_delay_task,
+                                    pr);
+  pr->qe =
+      GNUNET_DATASTORE_get_key (GSF_dsh,
+                                next_uid,
+                                random,
+                                &pr->public_data.query,
+                                pr->public_data.type ==
+                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
+                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
+                                (0 !=
+                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
+                                  public_data.options)) ? UINT_MAX : 1
+                                /* queue priority */ ,
+                                (0 !=
+                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
+                                  public_data.options)) ? UINT_MAX :
+                                GSF_datastore_queue_size
+                                /* max queue size */ ,
+                                &process_local_reply, pr);
+  if (NULL != pr->qe)
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n",
+              GNUNET_h2s (&pr->public_data.query),
+              pr->public_data.type,
+              (unsigned long long) next_uid);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Datastore lookups concluded (error queueing)"),
+                            1,
+                            GNUNET_NO);
+  call_continuation (pr);
+}
+
+
 /**
  * We're processing (local) results for a search request
  * from another peer.  Pass applicable results to the
@@ -1338,6 +1471,7 @@ odc_warn_delay_task (void *cls)
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -1350,73 +1484,76 @@ process_local_reply (void *cls,
                      enum GNUNET_BLOCK_Type type,
                      uint32_t priority,
                      uint32_t anonymity,
+                     uint32_t replication,
                      struct GNUNET_TIME_Absolute expiration,
                      uint64_t uid)
 {
   struct GSF_PendingRequest *pr = cls;
-  GSF_LocalLookupContinuation cont;
   struct ProcessReplyClosure prq;
   struct GNUNET_HashCode query;
   unsigned int old_rf;
 
   GNUNET_SCHEDULER_cancel (pr->warn_task);
   pr->warn_task = NULL;
-  if (NULL != pr->qe)
+  if (NULL == pr->qe)
+    goto called_from_on_demand;
+  pr->qe = NULL;
+  if ( (NULL == key) &&
+       pr->seen_null &&
+       !pr->have_first_uid) /* We have hit the end for the 2nd time with no results */
   {
-    pr->qe = NULL;
-    if (NULL == key)
-    {
+    /* No results */
 #if INSANE_STATISTICS
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# Datastore lookups concluded (no results)"),
-                                1, GNUNET_NO);
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups concluded (no results)"),
+                              1, GNUNET_NO);
 #endif
-    }
-    if (GNUNET_NO == pr->have_first_uid)
-    {
-      pr->first_uid = uid;
-      pr->have_first_uid = 1;
-    }
-    else
-    {
-      if ((uid == pr->first_uid) && (key != NULL))
-      {
-        GNUNET_STATISTICS_update (GSF_stats,
-                                  gettext_noop
-                                  ("# Datastore lookups concluded (seen all)"),
-                                  1, GNUNET_NO);
-        key = NULL;             /* all replies seen! */
-      }
-      pr->have_first_uid++;
-      if ((pr->have_first_uid > MAX_RESULTS) && (key != NULL))
-      {
-        GNUNET_STATISTICS_update (GSF_stats,
-                                  gettext_noop
-                                  ("# Datastore lookups aborted (more than MAX_RESULTS)"),
-                                  1, GNUNET_NO);
-        key = NULL;             /* all replies seen! */
-      }
-    }
+    no_more_local_results (pr);
+    return;
+  }
+  if ( ( (NULL == key) &&
+         pr->seen_null ) || /* We have hit the end for the 2nd time OR */
+       ( pr->seen_null &&
+         pr->have_first_uid &&
+         (uid >= pr->first_uid) ) ) /* We have hit the end and past first UID */
+  {
+    /* Seen all results */
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups concluded (seen all)"),
+                              1, GNUNET_NO);
+    no_more_local_results (pr);
+    return;
   }
   if (NULL == key)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
-                "No further local responses available.\n");
-#if INSANE_STATISTICS
-    if ((pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK) ||
-        (pr->public_data.type == GNUNET_BLOCK_TYPE_FS_IBLOCK))
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# requested DBLOCK or IBLOCK not found"), 1,
-                                GNUNET_NO);
-#endif
-    goto check_error_and_continue;
+    GNUNET_assert (!pr->seen_null);
+    pr->seen_null = true;
+    start_local_query (pr,
+                       0 /* next_uid */,
+                       false /* random */);
+    return;
+  }
+  if (!pr->have_first_uid)
+  {
+    pr->first_uid = uid;
+    pr->have_first_uid = true;
+  }
+  pr->result_count++;
+  if (pr->result_count > MAX_RESULTS)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups aborted (more than MAX_RESULTS)"),
+                              1, GNUNET_NO);
+    no_more_local_results (pr);
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received reply for `%s' of type %d with UID %llu from datastore.\n",
               GNUNET_h2s (key), type, (unsigned long long) uid);
-  if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND)
+  if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Found ONDEMAND block, performing on-demand encoding\n");
@@ -1429,9 +1566,17 @@ process_local_reply (void *cls,
         GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
                                       &odc_warn_delay_task, pr);
     if (GNUNET_OK ==
-        GNUNET_FS_handle_on_demand_block (key, size, data, type, priority,
-                                          anonymity, expiration, uid,
-                                          &process_local_reply, pr))
+        GNUNET_FS_handle_on_demand_block (key,
+                                          size,
+                                          data,
+                                          type,
+                                          priority,
+                                          anonymity,
+                                          replication,
+                                          expiration,
+                                          uid,
+                                          &process_local_reply,
+                                          pr))
     {
       GNUNET_STATISTICS_update (GSF_stats,
                                 gettext_noop
@@ -1443,33 +1588,12 @@ process_local_reply (void *cls,
                               gettext_noop ("# on-demand lookups failed"), 1,
                               GNUNET_NO);
     GNUNET_SCHEDULER_cancel (pr->warn_task);
-    pr->warn_task =
-        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                      &warn_delay_task, pr);
-    pr->qe =
-        GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1,
-                                  &pr->public_data.query,
-                                  pr->public_data.type ==
-                                  GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                  GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX : 1
-                                  /* queue priority */ ,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX :
-                                  GSF_datastore_queue_size
-                                  /* max queue size */ ,
-                                  &process_local_reply, pr);
-    if (NULL != pr->qe)
-      return;                   /* we're done */
-    GNUNET_STATISTICS_update (GSF_stats,
-                              gettext_noop
-                              ("# Datastore lookups concluded (error queueing)"),
-                              1, GNUNET_NO);
-    goto check_error_and_continue;
+    start_local_query (pr,
+                       uid + 1 /* next_uid */,
+                       false /* random */);
+    return;
   }
+called_from_on_demand:
   old_rf = pr->public_data.results_found;
   memset (&prq, 0, sizeof (prq));
   prq.data = data;
@@ -1481,34 +1605,9 @@ process_local_reply (void *cls,
     GNUNET_break (0);
     GNUNET_DATASTORE_remove (GSF_dsh, key, size, data, -1, -1,
                              NULL, NULL);
-    pr->qe_start = GNUNET_TIME_absolute_get ();
-    pr->warn_task =
-        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                      &warn_delay_task, pr);
-    pr->qe =
-        GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1,
-                                  &pr->public_data.query,
-                                  pr->public_data.type ==
-                                  GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                  GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX : 1
-                                  /* queue priority */ ,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX :
-                                  GSF_datastore_queue_size
-                                  /* max queue size */ ,
-                                  &process_local_reply, pr);
-    if (NULL == pr->qe)
-    {
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# Datastore lookups concluded (error queueing)"),
-                                1, GNUNET_NO);
-      goto check_error_and_continue;
-    }
+    start_local_query (pr,
+                       uid + 1 /* next_uid */,
+                       false /* random */);
     return;
   }
   prq.type = type;
@@ -1520,14 +1619,15 @@ process_local_reply (void *cls,
   prq.eo = GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO;
   process_reply (&prq, key, pr);
   pr->local_result = prq.eval;
-  if (prq.eval == GNUNET_BLOCK_EVALUATION_OK_LAST)
+  if (GNUNET_BLOCK_EVALUATION_OK_LAST == prq.eval)
   {
     GNUNET_STATISTICS_update (GSF_stats,
                               gettext_noop
                               ("# Datastore lookups concluded (found last result)"),
                               1,
                               GNUNET_NO);
-    goto check_error_and_continue;
+    call_continuation (pr);
+    return;
   }
   if ((0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) &&
       ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) ||
@@ -1539,66 +1639,12 @@ process_local_reply (void *cls,
                               gettext_noop ("# Datastore lookups concluded (load too high)"),
                               1,
                               GNUNET_NO);
-    goto check_error_and_continue;
-  }
-  pr->qe_start = GNUNET_TIME_absolute_get ();
-  pr->warn_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_delay_task,
-                                    pr);
-  pr->qe =
-      GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++,
-                                &pr->public_data.query,
-                                pr->public_data.type ==
-                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX : 1
-                                /* queue priority */ ,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX :
-                                GSF_datastore_queue_size
-                                /* max queue size */ ,
-                                &process_local_reply, pr);
-  /* check if we successfully queued another datastore request;
-   * if so, return, otherwise call our continuation (if we have
-   * any) */
-check_error_and_continue:
-  if (NULL != pr->qe)
-    return;
-  if (NULL != pr->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (pr->warn_task);
-    pr->warn_task = NULL;
-  }
-  if (NULL == (cont = pr->llc_cont))
-    return;                     /* no continuation */
-  pr->llc_cont = NULL;
-  if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options))
-  {
-    if (GNUNET_BLOCK_EVALUATION_OK_LAST != pr->local_result)
-    {
-      /* Signal that we are done and that there won't be any
-         additional results to allow client to clean up state. */
-      pr->rh (pr->rh_cls,
-               GNUNET_BLOCK_EVALUATION_OK_LAST,
-               pr,
-               UINT32_MAX,
-               GNUNET_TIME_UNIT_ZERO_ABS,
-               GNUNET_TIME_UNIT_FOREVER_ABS,
-               GNUNET_BLOCK_TYPE_ANY,
-               NULL, 0);
-    }
-    /* Finally, call our continuation to signal that we are
-       done with local processing of this request; i.e. to
-       start reading again from the client. */
-    cont (pr->llc_cont_cls, NULL, GNUNET_BLOCK_EVALUATION_OK_LAST);
+    call_continuation (pr);
     return;
   }
-
-  cont (pr->llc_cont_cls, pr, pr->local_result);
+  start_local_query (pr,
+                     uid + 1 /* next_uid */,
+                     false /* random */);
 }
 
 
@@ -1642,43 +1688,14 @@ GSF_local_lookup_ (struct GSF_PendingRequest *pr,
   GNUNET_assert (NULL == pr->llc_cont);
   pr->llc_cont = cont;
   pr->llc_cont_cls = cont_cls;
-  pr->qe_start = GNUNET_TIME_absolute_get ();
-  pr->warn_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_delay_task,
-                                    pr);
 #if INSANE_STATISTICS
   GNUNET_STATISTICS_update (GSF_stats,
                             gettext_noop ("# Datastore lookups initiated"), 1,
                             GNUNET_NO);
 #endif
-  pr->qe =
-      GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++,
-                                &pr->public_data.query,
-                                pr->public_data.type ==
-                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX : 1
-                                /* queue priority */ ,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX :
-                                GSF_datastore_queue_size
-                                /* max queue size */ ,
-                                &process_local_reply, pr);
-  if (NULL != pr->qe)
-    return;
-  GNUNET_STATISTICS_update (GSF_stats,
-                            gettext_noop
-                            ("# Datastore lookups concluded (error queueing)"),
-                            1, GNUNET_NO);
-  GNUNET_SCHEDULER_cancel (pr->warn_task);
-  pr->warn_task = NULL;
-  pr->llc_cont = NULL;
-  if (NULL != cont)
-    cont (cont_cls, pr, pr->local_result);
+  start_local_query(pr,
+                    0 /* next_uid */,
+                    true /* random */);
 }
 
 
diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c
index 361d30755..21c598a72 100644
--- a/src/fs/gnunet-service-fs_push.c
+++ b/src/fs/gnunet-service-fs_push.c
@@ -448,6 +448,7 @@ consider_gathering ()
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -460,6 +461,7 @@ process_migration_content (void *cls,
                            enum GNUNET_BLOCK_Type type,
                            uint32_t priority,
                            uint32_t anonymity,
+                           uint32_t replication,
                            struct GNUNET_TIME_Absolute expiration,
                            uint64_t uid)
 {
@@ -491,9 +493,11 @@ process_migration_content (void *cls,
                                           type,
                                           priority,
                                           anonymity,
+                                          replication,
                                           expiration,
                                           uid,
-                                          &process_migration_content, NULL))
+                                          &process_migration_content,
+                                          NULL))
       consider_gathering ();
     return;
   }
diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c
index bb4cb4ecb..e8c7f586d 100644
--- a/src/fs/gnunet-service-fs_put.c
+++ b/src/fs/gnunet-service-fs_put.c
@@ -72,9 +72,14 @@ struct PutOperator
   uint64_t zero_anonymity_count_estimate;
 
   /**
-   * Current offset when iterating the database.
+   * Count of results received from the database.
    */
-  uint64_t current_offset;
+  uint64_t result_count;
+
+  /**
+   * Next UID to request when iterating the database.
+   */
+  uint64_t next_uid;
 };
 
 
@@ -171,43 +176,51 @@ delay_dht_put_task (void *cls)
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
  */
 static void
 process_dht_put_content (void *cls,
-                         const struct GNUNET_HashCode * key,
-                         size_t size,
+                         const struct GNUNET_HashCode * key,
+                         size_t size,
                          const void *data,
-                         enum GNUNET_BLOCK_Type type,
-                         uint32_t priority, uint32_t anonymity,
-                         struct GNUNET_TIME_Absolute expiration, uint64_t uid)
+                         enum GNUNET_BLOCK_Type type,
+                         uint32_t priority,
+                         uint32_t anonymity,
+                         uint32_t replication,
+                         struct GNUNET_TIME_Absolute expiration,
+                         uint64_t uid)
 {
   struct PutOperator *po = cls;
 
   po->dht_qe = NULL;
   if (key == NULL)
   {
-    po->zero_anonymity_count_estimate = po->current_offset - 1;
-    po->current_offset = 0;
+    po->zero_anonymity_count_estimate = po->result_count;
+    po->result_count = 0;
+    po->next_uid = 0;
     po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
     return;
   }
+  po->result_count++;
+  po->next_uid = uid + 1;
   po->zero_anonymity_count_estimate =
-      GNUNET_MAX (po->current_offset, po->zero_anonymity_count_estimate);
+    GNUNET_MAX (po->result_count, po->zero_anonymity_count_estimate);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Retrieved block `%s' of type %u for DHT PUT\n", GNUNET_h2s (key),
               type);
   po->dht_put = GNUNET_DHT_put (GSF_dht,
                                 key,
                                 DEFAULT_PUT_REPLICATION,
-                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
                                 type,
                                 size,
                                 data,
-                                expiration,
-                                &delay_dht_put_blocks, po);
+                                expiration,
+                                &delay_dht_put_blocks,
+                                po);
 }
 
 
@@ -223,10 +236,13 @@ gather_dht_put_blocks (void *cls)
 
   po->dht_task = NULL;
   po->dht_qe =
-      GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh, po->current_offset++, 0,
+      GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh,
+                                           po->next_uid,
+                                           0,
                                            UINT_MAX,
                                            po->dht_put_type,
-                                           &process_dht_put_content, po);
+                                           &process_dht_put_content,
+                                           po);
   if (NULL == po->dht_qe)
     po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
 }
diff --git a/src/fs/gnunet-unindex.c b/src/fs/gnunet-unindex.c
index 40fa13b62..213d2b332 100644
--- a/src/fs/gnunet-unindex.c
+++ b/src/fs/gnunet-unindex.c
@@ -30,7 +30,7 @@
 
 static int ret;
 
-static int verbose;
+static unsigned int verbose;
 
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
@@ -162,10 +162,10 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c
index 415a2e3ed..902519f15 100644
--- a/src/fs/plugin_block_fs.c
+++ b/src/fs/plugin_block_fs.c
@@ -23,12 +23,12 @@
  * @brief blocks used for file-sharing
  * @author Christian Grothoff
  */
-
 #include "platform.h"
 #include "gnunet_block_plugin.h"
 #include "gnunet_fs_service.h"
 #include "block_fs.h"
 #include "gnunet_signatures.h"
+#include "gnunet_block_group_lib.h"
 
 
 /**
@@ -37,6 +37,70 @@
  */
 #define BLOOMFILTER_K 16
 
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_fs_create_group (void *cls,
+                              enum GNUNET_BLOCK_Type type,
+                              uint32_t nonce,
+                              const void *raw_data,
+                              size_t raw_data_size,
+                              va_list va)
+{
+  unsigned int size;
+  const char *guard;
+
+  switch (type)
+  {
+  case GNUNET_BLOCK_TYPE_FS_DBLOCK:
+    GNUNET_break (NULL == va_arg (va, const char *));
+    return NULL;
+  case GNUNET_BLOCK_TYPE_FS_IBLOCK:
+    GNUNET_break (NULL == va_arg (va, const char *));
+    return NULL;
+  case GNUNET_BLOCK_TYPE_FS_UBLOCK:
+    guard = va_arg (va, const char *);
+    if (0 != strcmp (guard,
+                     "seen-set-size"))
+    {
+      /* va-args invalid! bad bug, complain! */
+      GNUNET_break (0);
+      size = 8;
+    }
+    else
+    {
+      size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                          BLOOMFILTER_K);
+    }
+    if (0 == size)
+      size = raw_data_size; /* not for us to determine, use what we got! */
+    GNUNET_break (NULL == va_arg (va, const char *));
+    return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                         size,
+                                         BLOOMFILTER_K,
+                                         type,
+                                         nonce,
+                                         raw_data,
+                                         raw_data_size);
+  default:
+    GNUNET_break (NULL == va_arg (va, const char *));
+    GNUNET_break (0);
+    return NULL;
+  }
+}
+
+
 /**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the reply_block.
@@ -45,11 +109,11 @@
  * be done with the #GNUNET_BLOCK_get_key() function.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param bg group to use for evaluation
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -58,11 +122,11 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_fs_evaluate (void *cls,
+                          struct GNUNET_BLOCK_Context *ctx,
                           enum GNUNET_BLOCK_Type type,
+                          struct GNUNET_BLOCK_Group *bg,
                           enum GNUNET_BLOCK_EvaluationOptions eo,
                           const struct GNUNET_HashCode *query,
-                          struct GNUNET_CONTAINER_BloomFilter **bf,
-                          int32_t bf_mutator,
                           const void *xquery,
                           size_t xquery_size,
                           const void *reply_block,
@@ -71,7 +135,6 @@ block_plugin_fs_evaluate (void *cls,
   const struct UBlock *ub;
   struct GNUNET_HashCode hc;
   struct GNUNET_HashCode chash;
-  struct GNUNET_HashCode mhash;
 
   switch (type)
   {
@@ -125,26 +188,13 @@ block_plugin_fs_evaluate (void *cls,
       GNUNET_break_op (0);
       return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
     }
-    if (NULL != bf)
-    {
-      GNUNET_CRYPTO_hash (reply_block,
-                          reply_block_size,
-                          &chash);
-      GNUNET_BLOCK_mingle_hash (&chash,
-                                bf_mutator,
-                                &mhash);
-      if (NULL != *bf)
-      {
-        if (GNUNET_YES ==
-            GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-          return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-      }
-      else
-      {
-        *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K);
-      }
-      GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-    }
+    GNUNET_CRYPTO_hash (reply_block,
+                        reply_block_size,
+                        &chash);
+    if (GNUNET_YES ==
+        GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                            &chash))
+      return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
     return GNUNET_BLOCK_EVALUATION_OK_MORE;
   default:
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -214,6 +264,7 @@ libgnunet_plugin_block_fs_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_fs_evaluate;
   api->get_key = &block_plugin_fs_get_key;
+  api->create_group = &block_plugin_fs_create_group;
   api->types = types;
   return api;
 }
@@ -225,7 +276,7 @@ libgnunet_plugin_block_fs_init (void *cls)
 void *
 libgnunet_plugin_block_fs_done (void *cls)
 {
-  struct GNUNET_TRANSPORT_PluginFunctions *api = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
 
   GNUNET_free (api);
   return NULL;
diff --git a/src/fs/test_plugin_block_fs.c b/src/fs/test_plugin_block_fs.c
index 1fc9f2110..ba4f28bc5 100644
--- a/src/fs/test_plugin_block_fs.c
+++ b/src/fs/test_plugin_block_fs.c
@@ -40,28 +40,28 @@ test_fs (struct GNUNET_BLOCK_Context *ctx)
   if (GNUNET_BLOCK_EVALUATION_OK_LAST !=
       GNUNET_BLOCK_evaluate (ctx,
                              GNUNET_BLOCK_TYPE_FS_DBLOCK,
+                             NULL,
                              GNUNET_BLOCK_EO_NONE,
                              &key,
                              NULL, 0,
-                             NULL, 0,
                              block, sizeof (block)))
     return 2;
   if (GNUNET_BLOCK_EVALUATION_REQUEST_VALID !=
       GNUNET_BLOCK_evaluate (ctx,
                              GNUNET_BLOCK_TYPE_FS_DBLOCK,
+                             NULL,
                              GNUNET_BLOCK_EO_NONE,
                              &key,
                              NULL, 0,
-                             NULL, 0,
                              NULL, 0))
     return 4;
   GNUNET_log_skip (1, GNUNET_NO);
   if (GNUNET_BLOCK_EVALUATION_REQUEST_INVALID !=
       GNUNET_BLOCK_evaluate (ctx,
                              GNUNET_BLOCK_TYPE_FS_DBLOCK,
+                             NULL,
                              GNUNET_BLOCK_EO_NONE,
                              &key,
-                             NULL, 0,
                              "bogus", 5,
                              NULL, 0))
     return 8;
diff --git a/src/gns/Makefile.am b/src/gns/Makefile.am
index 0b65a3d17..464bbbca1 100644
--- a/src/gns/Makefile.am
+++ b/src/gns/Makefile.am
@@ -12,7 +12,6 @@ SUBDIRS = . $(NSS_SUBDIR)
 EXTRA_DIST = \
   test_gns_defaults.conf \
   test_gns_lookup.conf \
-  test_gns_nick_shorten.conf \
   test_gns_proxy.conf \
   test_gns_simple_lookup.conf \
   gns-helper-service-w32.conf \
@@ -70,8 +69,8 @@ endif
 
 libexec_PROGRAMS = \
   gnunet-service-gns \
-  $(DO_W32_HELPER) \
   gnunet-dns2gns \
+  $(DO_W32_HELPER) \
   $(DO_PROXY)
 
 bin_PROGRAMS = \
@@ -114,7 +113,6 @@ gnunet_gns_SOURCES = \
 gnunet_gns_LDADD = \
   libgnunetgns.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
-  $(top_builddir)/src/namestore/libgnunetnamestore.la \
   $(top_builddir)/src/identity/libgnunetidentity.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(GN_LIBINTL)
@@ -134,11 +132,18 @@ gnunet_dns2gns_LDADD = \
   libgnunetgns.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/identity/libgnunetidentity.la \
-  $(top_builddir)/src/namestore/libgnunetnamestore.la \
   $(top_builddir)/src/dns/libgnunetdnsparser.la \
   $(top_builddir)/src/dns/libgnunetdnsstub.la \
   $(GN_LIBINTL)
 
+if LINUX
+HIJACKBIN = gnunet-dns2gns
+install-exec-hook:
+        $(SUDO_BINARY) setcap 'cap_net_bind_service=+ep' $(DESTDIR)$(libexecdir)/gnunet-dns2gns || true
+else
+install-exec-hook:
+endif
+
 gnunet_gns_proxy_SOURCES = \
  gnunet-gns-proxy.c
 gnunet_gns_proxy_CPPFLAGS = $(AM_CPPFLAGS) $(CPP_GNURL)
@@ -184,8 +189,6 @@ w32nsp_resolve_LDADD = -lws2_32
 gnunet_service_gns_SOURCES = \
  gnunet-service-gns.c \
  gnunet-service-gns_resolver.c gnunet-service-gns_resolver.h \
- gnunet-service-gns_reverser.c gnunet-service-gns_reverser.h \
- gnunet-service-gns_shorten.c gnunet-service-gns_shorten.h \
  gnunet-service-gns_interceptor.c gnunet-service-gns_interceptor.h
 gnunet_service_gns_LDADD = \
   -lm \
@@ -200,7 +203,6 @@ gnunet_service_gns_LDADD = \
   $(top_builddir)/src/dht/libgnunetdht.la \
   $(top_builddir)/src/tun/libgnunettun.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
-  $(top_builddir)/src/namestore/libgnunetnamestore.la \
   $(USE_VPN) \
   $(GN_LIBINTL)
 
@@ -227,6 +229,7 @@ libgnunet_plugin_block_gns_la_SOURCES = \
 libgnunet_plugin_block_gns_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/block/libgnunetblock.la \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la
 libgnunet_plugin_block_gns_la_LDFLAGS = \
   $(GN_PLUGIN_LDFLAGS)
@@ -253,18 +256,16 @@ check_SCRIPTS = \
   test_gns_gns2dns_lookup.sh \
   test_gns_dht_lookup.sh\
   test_gns_delegated_lookup.sh \
-  test_gns_nick_shorten.sh\
   test_gns_plus_lookup.sh\
   test_gns_zkey_lookup.sh\
   test_gns_rel_expiration.sh\
   test_gns_soa_lookup.sh\
   test_gns_revocation.sh\
-  test_gns_cname_lookup.sh \
-        test_gns_reverse_lookup.sh
+  test_gns_cname_lookup.sh
 
 if ENABLE_TEST_RUN
 if HAVE_SQLITE
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = $(check_SCRIPTS)
 endif
 endif
diff --git a/src/gns/gns.h b/src/gns/gns.h
index ca5525f80..d77bf53c6 100644
--- a/src/gns/gns.h
+++ b/src/gns/gns.h
@@ -72,51 +72,20 @@ struct LookupMessage
   int16_t options GNUNET_PACKED;
 
   /**
-   * Is a shorten key attached?
+   * Always 0.
    */
-  int16_t have_key GNUNET_PACKED;
+  int16_t reserved GNUNET_PACKED;
 
   /**
    * the type of record to look up
    */
   int32_t type GNUNET_PACKED;
 
-  /**
-   * The key for shorten, if @e have_key is set
-   */
-  struct GNUNET_CRYPTO_EcdsaPrivateKey shorten_key;
-
   /* Followed by the zero-terminated name to look up */
 };
 
 
 /**
- * Message from client to GNS service to lookup records.
- */
-struct ReverseLookupMessage
-{
-  /**
-   * Header of type #GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique identifier for this request (for key collisions).
-   */
-  uint32_t id GNUNET_PACKED;
-
-  /**
-   * Zone that is target for reverse lookup
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey zone_pkey;
-  
-  /**
-   * Root zone
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey root_pkey;
-};
-
-/**
  * Message from GNS service to client: new results.
  */
 struct LookupResultMessage
@@ -140,24 +109,6 @@ struct LookupResultMessage
 
 };
 
-/**
- * Message from GNS service to client: new results.
- */
-struct ReverseLookupResultMessage
-{
-  /**
-    * Header of type #GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP_RESULT
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique identifier for this request (for key collisions).
-   */
-  uint32_t id GNUNET_PACKED;
-
-  /* followed by the resulting name of the reverse lookup */
-};
-
 
 GNUNET_NETWORK_STRUCT_END
 
diff --git a/src/gns/gns_api.c b/src/gns/gns_api.c
index f6f9889ac..84c4ae189 100644
--- a/src/gns/gns_api.c
+++ b/src/gns/gns_api.c
@@ -79,49 +79,6 @@ struct GNUNET_GNS_LookupRequest
 
 };
 
-/**
- * Handle to a lookup request
- */
-struct GNUNET_GNS_ReverseLookupRequest
-{
-
-  /**
-   * DLL
-   */
-  struct GNUNET_GNS_ReverseLookupRequest *next;
-
-  /**
-   * DLL
-   */
-  struct GNUNET_GNS_ReverseLookupRequest *prev;
-
-  /**
-   * handle to gns
-   */
-  struct GNUNET_GNS_Handle *gns_handle;
-
-  /**
-   * processor to call on lookup result
-   */
-  GNUNET_GNS_ReverseLookupResultProcessor lookup_proc;
-
-  /**
-   * @e lookup_proc closure
-   */
-  void *proc_cls;
-
-  /**
-   * Envelope with the message for this queue entry.
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-  /**
-   * request id
-   */
-  uint32_t r_id;
-
-};
-
 
 /**
  * Connection to the GNS service.
@@ -150,15 +107,6 @@ struct GNUNET_GNS_Handle
   struct GNUNET_GNS_LookupRequest *lookup_tail;
 
   /**
-   * Head of linked list of active reverse lookup requests.
-   */
-  struct GNUNET_GNS_ReverseLookupRequest *rev_lookup_head;
-
-  /**
-   * Tail of linked list of active reverse lookup requests.
-   */
-  struct GNUNET_GNS_ReverseLookupRequest *rev_lookup_tail;
-  /**
    * Reconnect task
    */
   struct GNUNET_SCHEDULER_Task *reconnect_task;
@@ -232,71 +180,13 @@ mq_error_handler (void *cls,
                   enum GNUNET_MQ_Error error)
 {
   struct GNUNET_GNS_Handle *handle = cls;
-  LOG (GNUNET_ERROR_TYPE_WARNING, "Problem with message queue. error: %i\n",
+
+  LOG (GNUNET_ERROR_TYPE_WARNING,
+       "Problem with message queue. error: %i\n",
        error);
   force_reconnect (handle);
 }
 
-/**
- * Check validity of message received from the GNS service
- *
- * @param cls the `struct GNUNET_GNS_Handle *`
- * @param loookup_msg the incoming message
- */
-static int
-check_rev_result (void *cls,
-              const struct ReverseLookupResultMessage *lookup_msg)
-{
-  size_t mlen = ntohs (lookup_msg->header.size) - sizeof (*lookup_msg);
-  char *name;
-  
-  name = (char*) &lookup_msg[1];
-  if ('\0' != name[mlen-1])
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for messages received from the GNS service
- *
- * @param cls the `struct GNUNET_GNS_Handle *`
- * @param loookup_msg the incoming message
- */
-static void
-handle_rev_result (void *cls,
-                   const struct ReverseLookupResultMessage *lookup_msg)
-{
-  struct GNUNET_GNS_Handle *handle = cls;
-  char *name;
-  uint32_t r_id = ntohl (lookup_msg->id);
-  struct GNUNET_GNS_ReverseLookupRequest *rlr;
-  GNUNET_GNS_ReverseLookupResultProcessor proc;
-  void *proc_cls;
-
-  name = (char*)&lookup_msg[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received reverse lookup reply from GNS service (%s)\n",
-       name);
-  for (rlr = handle->rev_lookup_head; NULL != rlr; rlr = rlr->next)
-    if (rlr->r_id == r_id)
-      break;
-  if (NULL == rlr)
-    return;
-  proc = rlr->lookup_proc;
-  proc_cls = rlr->proc_cls;
-  GNUNET_CONTAINER_DLL_remove (handle->rev_lookup_head,
-                               handle->rev_lookup_tail,
-                               rlr);
-  GNUNET_free (rlr);
-  proc (proc_cls,
-        name);
-}
-
-
 
 /**
  * Check validity of message received from the GNS service
@@ -385,14 +275,9 @@ reconnect (struct GNUNET_GNS_Handle *handle)
                            GNUNET_MESSAGE_TYPE_GNS_LOOKUP_RESULT,
                            struct LookupResultMessage,
                            handle),
-    GNUNET_MQ_hd_var_size (rev_result,
-                           GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP_RESULT,
-                           struct ReverseLookupResultMessage,
-                           handle),
     GNUNET_MQ_handler_end ()
   };
   struct GNUNET_GNS_LookupRequest *lh;
-  struct GNUNET_GNS_ReverseLookupRequest *rlh;
 
   GNUNET_assert (NULL == handle->mq);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -407,9 +292,6 @@ reconnect (struct GNUNET_GNS_Handle *handle)
   for (lh = handle->lookup_head; NULL != lh; lh = lh->next)
     GNUNET_MQ_send_copy (handle->mq,
                          lh->env);
-  for (rlh = handle->rev_lookup_head; NULL != rlh; rlh = rlh->next)
-    GNUNET_MQ_send_copy (handle->mq,
-                         rlh->env);
 }
 
 
@@ -455,7 +337,6 @@ GNUNET_GNS_disconnect (struct GNUNET_GNS_Handle *handle)
     handle->reconnect_task = NULL;
   }
   GNUNET_assert (NULL == handle->lookup_head);
-  GNUNET_assert (NULL == handle->rev_lookup_head);
   GNUNET_free (handle);
 }
 
@@ -477,22 +358,6 @@ GNUNET_GNS_lookup_cancel (struct GNUNET_GNS_LookupRequest *lr)
   GNUNET_free (lr);
 }
 
-/**
- * Cancel pending reverse lookup request
- *
- * @param lr the lookup request to cancel
- */
-void
-GNUNET_GNS_reverse_lookup_cancel (struct GNUNET_GNS_ReverseLookupRequest *lr)
-{
-  struct GNUNET_GNS_Handle *handle = lr->gns_handle;
-
-  GNUNET_CONTAINER_DLL_remove (handle->rev_lookup_head,
-                               handle->rev_lookup_tail,
-                               lr);
-  GNUNET_MQ_discard (lr->env);
-  GNUNET_free (lr);
-}
 
 /**
  * Perform an asynchronous lookup operation on the GNS.
@@ -502,7 +367,6 @@ GNUNET_GNS_reverse_lookup_cancel (struct GNUNET_GNS_ReverseLookupRequest *lr)
  * @param zone the zone to start the resolution in
  * @param type the record type to look up
  * @param options local options for the lookup
- * @param shorten_zone_key the private key of the shorten zone (can be NULL)
  * @param proc processor to call on result
  * @param proc_cls closure for @a proc
  * @return handle to the get request
@@ -513,7 +377,6 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
                    const struct GNUNET_CRYPTO_EcdsaPublicKey *zone,
                    uint32_t type,
                    enum GNUNET_GNS_LocalOptions options,
-                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_zone_key,
                    GNUNET_GNS_LookupResultProcessor proc,
                    void *proc_cls)
 {
@@ -531,7 +394,7 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
        "Trying to lookup `%s' in GNS\n",
        name);
   nlen = strlen (name) + 1;
-  if (nlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*lr))
+  if (nlen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (*lr))
   {
     GNUNET_break (0);
     return NULL;
@@ -548,11 +411,6 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
   lookup_msg->options = htons ((uint16_t) options);
   lookup_msg->zone = *zone;
   lookup_msg->type = htonl (type);
-  if (NULL != shorten_zone_key)
-  {
-    lookup_msg->have_key = htons (GNUNET_YES);
-    lookup_msg->shorten_key = *shorten_zone_key;
-  }
   GNUNET_memcpy (&lookup_msg[1],
                  name,
                  nlen);
@@ -565,50 +423,4 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
   return lr;
 }
 
-/**
- * Perform an asynchronous reverse lookup operation on the GNS.
- *
- * @param handle handle to the GNS service
- * @param zone_key zone to find a name for
- * @param root_key our zone
- * @param proc processor to call on result
- * @param proc_cls closure for @a proc
- * @return handle to the request
- */
-struct GNUNET_GNS_ReverseLookupRequest*
-GNUNET_GNS_reverse_lookup (struct GNUNET_GNS_Handle *handle,
-                           const struct GNUNET_CRYPTO_EcdsaPublicKey *zone_key,
-                           const struct GNUNET_CRYPTO_EcdsaPublicKey *root_key,
-                           GNUNET_GNS_ReverseLookupResultProcessor proc,
-                           void *proc_cls)
-{
-  /* IPC to shorten gns names, return shorten_handle */
-  struct ReverseLookupMessage *rev_lookup_msg;
-  struct GNUNET_GNS_ReverseLookupRequest *lr;
-
-  if ((NULL == zone_key) || (NULL == root_key))
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Trying to reverse lookup in GNS\n");
-  lr = GNUNET_new (struct GNUNET_GNS_ReverseLookupRequest);
-  lr->gns_handle = handle;
-  lr->lookup_proc = proc;
-  lr->proc_cls = proc_cls;
-  lr->r_id = handle->r_id_gen++;
-  lr->env = GNUNET_MQ_msg (rev_lookup_msg,
-                           GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP);
-  rev_lookup_msg->id = htonl (lr->r_id);
-  rev_lookup_msg->zone_pkey = *zone_key;
-  rev_lookup_msg->root_pkey = *root_key;
-  GNUNET_CONTAINER_DLL_insert (handle->rev_lookup_head,
-                               handle->rev_lookup_tail,
-                               lr);
-  if (NULL != handle->mq)
-    GNUNET_MQ_send_copy (handle->mq,
-                         lr->env);
-  return lr;
-}
 /* end of gns_api.c */
diff --git a/src/gns/gnunet-bcd.c b/src/gns/gnunet-bcd.c
index 21471350d..0746d5c57 100644
--- a/src/gns/gnunet-bcd.c
+++ b/src/gns/gnunet-bcd.c
@@ -514,10 +514,14 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", "PORT",
-      gettext_noop ("Run HTTP serve on port PORT (default is 8888)"), 1,
-      &GNUNET_GETOPT_set_uint, &port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "port",
+                                   "PORT",
+                                   gettext_noop ("Run HTTP serve on port PORT (default is 8888)"),
+                                   &port),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/gns/gnunet-dns2gns.c b/src/gns/gnunet-dns2gns.c
index f58303789..424677d14 100644
--- a/src/gns/gnunet-dns2gns.c
+++ b/src/gns/gnunet-dns2gns.c
@@ -138,7 +138,7 @@ static char *dns_ip;
 /**
  * UDP Port we listen on for inbound DNS requests.
  */
-static unsigned int listen_port = 2853;
+static unsigned int listen_port = 53;
 
 /**
  * Which GNS zone do we translate incoming DNS requests to?
@@ -483,7 +483,6 @@ handle_request (struct GNUNET_NETWORK_Handle *lsock,
                                          &my_zone,
                                          type,
                                          GNUNET_NO,
-                                         NULL /* no shorten */,
                                          &result_processor,
                                          request);
   }
@@ -618,7 +617,7 @@ run_dnsd ()
   if (NULL != listen_socket4)
   {
     struct sockaddr_in v4;
-    
+
     memset (&v4, 0, sizeof (v4));
     v4.sin_family = AF_INET;
 #if HAVE_SOCKADDR_IN_SIN_LEN
@@ -641,7 +640,7 @@ run_dnsd ()
   if (NULL != listen_socket6)
   {
     struct sockaddr_in6 v6;
-    
+
     memset (&v6, 0, sizeof (v6));
     v6.sin6_family = AF_INET6;
 #if HAVE_SOCKADDR_IN_SIN_LEN
@@ -777,33 +776,53 @@ int
 main (int argc,
       char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'d', "dns", "IP",
-      gettext_noop ("IP of recursive DNS resolver to use (required)"), 1,
-      &GNUNET_GETOPT_set_string, &dns_ip},
-    {'f', "fcfs", "NAME",
-      gettext_noop ("Authoritative FCFS suffix to use (optional); default: fcfs.zkey.eu"), 1,
-      &GNUNET_GETOPT_set_string, &fcfs_suffix},
-    {'s', "suffix", "SUFFIX",
-      gettext_noop ("Authoritative DNS suffix to use (optional); default: zkey.eu"), 1,
-      &GNUNET_GETOPT_set_string, &dns_suffix},
-    {'p', "port", "UDPPORT",
-      gettext_noop ("UDP port to listen on for inbound DNS requests; default: 2853"), 1,
-      &GNUNET_GETOPT_set_uint, &listen_port},
-    {'z', "zone", "PUBLICKEY",
-      gettext_noop ("Public key of the GNS zone to use (overrides default)"), 1,
-      &GNUNET_GETOPT_set_string, &gns_zone_str},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('d',
+                                 "dns",
+                                 "IP",
+                                 gettext_noop ("IP of recursive DNS resolver to use (required)"),
+                                 &dns_ip),
+
+    GNUNET_GETOPT_option_string ('f',
+                                 "fcfs",
+                                 "NAME",
+                                 gettext_noop ("Authoritative FCFS suffix to use (optional); default: fcfs.zkey.eu"),
+                                 &fcfs_suffix),
+
+    GNUNET_GETOPT_option_string ('s',
+                                 "suffix",
+                                 "SUFFIX",
+                                 gettext_noop ("Authoritative DNS suffix to use (optional); default: zkey.eu"),
+                                 &dns_suffix),
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "port",
+                                   "UDPPORT",
+                                   gettext_noop ("UDP port to listen on for inbound DNS requests; default: 2853"),
+                                   &listen_port),
+
+    GNUNET_GETOPT_option_string ('z',
+                                 "zone",
+                                 "PUBLICKEY",
+                                 gettext_noop ("Public key of the GNS zone to use (overrides default)"),
+                                 &gns_zone_str), 
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
 
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv,
-                                                 &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
-  GNUNET_log_setup ("gnunet-dns2gns", "WARNING", NULL);
+  GNUNET_log_setup ("gnunet-dns2gns",
+                    "WARNING",
+                    NULL);
   ret =
       (GNUNET_OK ==
-       GNUNET_PROGRAM_run (argc, argv, "gnunet-dns2gns",
+       GNUNET_PROGRAM_run (argc, argv,
+                           "gnunet-dns2gns",
                            _("GNUnet DNS-to-GNS proxy (a DNS server)"),
                            options,
                            &run, NULL)) ? 0 : 1;
diff --git a/src/gns/gnunet-gns-helper-service-w32.c b/src/gns/gnunet-gns-helper-service-w32.c
index bfee2b498..a59cc5981 100644
--- a/src/gns/gnunet-gns-helper-service-w32.c
+++ b/src/gns/gnunet-gns-helper-service-w32.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -48,6 +48,7 @@ DEFINE_DNS_GUID(SVCID_DNS_TYPE_SRV, 0x0021);
 DEFINE_GUID(SVCID_HOSTNAME, 0x0002a800, 0x0000, 0x0000, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46);
 DEFINE_GUID(SVCID_INET_HOSTADDRBYNAME, 0x0002a803, 0x0000, 0x0000, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46);
 
+
 struct request
 {
   /**
@@ -60,14 +61,23 @@ struct request
    */
   struct request *prev;
 
-  struct GNUNET_SERVER_Client *client;
+  /**
+   * Client that issued the request
+   */
+  struct GNUNET_SERVICE_Client *client;
+
   GUID sc;
+
   int af;
+
   wchar_t *name;
+
   char *u8name;
+
   struct GNUNET_GNS_LookupRequest *lookup_request;
 };
 
+
 /**
  * Head of the doubly-linked list (for cleanup).
  */
@@ -99,14 +109,10 @@ static struct GNUNET_IDENTITY_Handle *identity;
 static struct GNUNET_CRYPTO_EcdsaPublicKey gns_master_pubkey;
 
 /**
- * Private key of the gns-short ego
- */
-static struct GNUNET_CRYPTO_EcdsaPrivateKey gns_short_privkey;
-
-/**
  * Set to 1 once egos are obtained.
  */
-static int got_egos = 0;
+static int got_egos;
+
 
 /**
  * Task run on shutdown.  Cleans up everything.
@@ -117,6 +123,7 @@ static void
 do_shutdown (void *cls)
 {
   struct request *rq;
+
   if (NULL != id_op)
   {
     GNUNET_IDENTITY_cancel (id_op);
@@ -130,8 +137,10 @@ do_shutdown (void *cls)
   while (NULL != (rq = rq_head))
   {
     if (NULL != rq->lookup_request)
-      GNUNET_GNS_lookup_cancel(rq->lookup_request);
-    GNUNET_CONTAINER_DLL_remove (rq_head, rq_tail, rq);
+      GNUNET_GNS_lookup_cancel (rq->lookup_request);
+    GNUNET_CONTAINER_DLL_remove (rq_head,
+                                 rq_tail,
+                                 rq);
     GNUNET_free_non_null (rq->name);
     if (rq->u8name)
       free (rq->u8name);
@@ -144,133 +153,6 @@ do_shutdown (void *cls)
   }
 }
 
-/**
- * Context for transmitting replies to clients.
- */
-struct TransmitCallbackContext
-{
-
-  /**
-   * We keep these in a doubly-linked list (for cleanup).
-   */
-  struct TransmitCallbackContext *next;
-
-  /**
-   * We keep these in a doubly-linked list (for cleanup).
-   */
-  struct TransmitCallbackContext *prev;
-
-  /**
-   * The message that we're asked to transmit.
-   */
-  struct GNUNET_MessageHeader *msg;
-
-  /**
-   * Handle for the transmission request.
-   */
-  struct GNUNET_SERVER_TransmitHandle *th;
-
-
-  /**
-   * Handle for the client to which to send
-   */
-  struct GNUNET_SERVER_Client *client;
-};
-
-
-/**
- * Head of the doubly-linked list (for cleanup).
- */
-static struct TransmitCallbackContext *tcc_head;
-
-/**
- * Tail of the doubly-linked list (for cleanup).
- */
-static struct TransmitCallbackContext *tcc_tail;
-
-/**
- * Have we already cleaned up the TCCs and are hence no longer
- * willing (or able) to transmit anything to anyone?
- */
-static int cleaning_done;
-
-
-/**
- * Function called to notify a client about the socket
- * being ready to queue more data.  "buf" will be
- * NULL and "size" zero if the socket was closed for
- * writing in the meantime.
- *
- * @param cls closure
- * @param size number of bytes available in buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
- */
-static size_t
-transmit_callback (void *cls, size_t size, void *buf)
-{
-  struct TransmitCallbackContext *tcc = cls;
-  size_t msize;
-
-  tcc->th = NULL;
-  GNUNET_CONTAINER_DLL_remove (tcc_head, tcc_tail, tcc);
-  msize = ntohs (tcc->msg->size);
-  if (size == 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Transmission to client failed!\n"));
-    GNUNET_free (tcc->msg);
-    GNUNET_free (tcc);
-    return 0;
-  }
-  GNUNET_assert (size >= msize);
-  GNUNET_memcpy (buf, tcc->msg, msize);
-  GNUNET_free (tcc->msg);
-  GNUNET_free (tcc);
-  for (tcc = tcc_head; tcc; tcc = tcc->next)
-  {
-    if (NULL == tcc->th)
-    {
-      tcc->th = GNUNET_SERVER_notify_transmit_ready (tcc->client,
-          ntohs (tcc->msg->size),
-          GNUNET_TIME_UNIT_FOREVER_REL,
-          &transmit_callback, tcc);
-      break;
-    }
-  }
-  return msize;
-}
-
-
-/**
- * Transmit the given message to the client.
- *
- * @param client target of the message
- * @param msg message to transmit, will be freed!
- */
-static void
-transmit (struct GNUNET_SERVER_Client *client,
-          struct GNUNET_MessageHeader *msg)
-{
-  struct TransmitCallbackContext *tcc;
-
-  if (GNUNET_YES == cleaning_done)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("Shutdown in progress, aborting transmission.\n"));
-    GNUNET_free (msg);
-    return;
-  }
-  tcc = GNUNET_new (struct TransmitCallbackContext);
-  tcc->msg = msg;
-  tcc->client = client;
-  tcc->th = GNUNET_SERVER_notify_transmit_ready (client,
-      ntohs (msg->size),
-      GNUNET_TIME_UNIT_FOREVER_REL,
-      &transmit_callback, tcc);
-  GNUNET_CONTAINER_DLL_insert (tcc_head, tcc_tail, tcc);
-}
-
 
 #define MarshallPtr(ptr, base, type) \
   if (ptr) \
@@ -280,7 +162,6 @@ transmit (struct GNUNET_SERVER_Client *client,
 void
 MarshallWSAQUERYSETW (WSAQUERYSETW *qs, GUID *sc)
 {
-  int i;
   MarshallPtr (qs->lpszServiceInstanceName, qs, wchar_t);
   MarshallPtr (qs->lpServiceClassId, qs, GUID);
   MarshallPtr (qs->lpVersion, qs, WSAVERSION);
@@ -288,7 +169,7 @@ MarshallWSAQUERYSETW (WSAQUERYSETW *qs, GUID *sc)
   MarshallPtr (qs->lpszContext, qs, wchar_t);
   MarshallPtr (qs->lpafpProtocols, qs, AFPROTOCOLS);
   MarshallPtr (qs->lpszQueryString, qs, wchar_t);
-  for (i = 0; i < qs->dwNumberOfCsAddrs; i++)
+  for (int i = 0; i < qs->dwNumberOfCsAddrs; i++)
   {
     MarshallPtr (qs->lpcsaBuffer[i].LocalAddr.lpSockaddr, qs, SOCKADDR);
     MarshallPtr (qs->lpcsaBuffer[i].RemoteAddr.lpSockaddr, qs, SOCKADDR);
@@ -297,12 +178,13 @@ MarshallWSAQUERYSETW (WSAQUERYSETW *qs, GUID *sc)
   if (IsEqualGUID (&SVCID_INET_HOSTADDRBYNAME, sc) && qs->lpBlob != NULL && qs->lpBlob->pBlobData != NULL)
   {
     struct hostent *he;
+
     he = (struct hostent *) qs->lpBlob->pBlobData;
-    for (i = 0; he->h_aliases[i] != NULL; i++)
+    for (int i = 0; he->h_aliases[i] != NULL; i++)
       MarshallPtr (he->h_aliases[i], he, char);
     MarshallPtr (he->h_aliases, he, char *);
     MarshallPtr (he->h_name, he, char);
-    for (i = 0; he->h_addr_list[i] != NULL; i++)
+    for (int i = 0; he->h_addr_list[i] != NULL; i++)
       MarshallPtr (he->h_addr_list[i], he, void);
     MarshallPtr (he->h_addr_list, he, char *);
     MarshallPtr (qs->lpBlob->pBlobData, qs, void);
@@ -312,13 +194,16 @@ MarshallWSAQUERYSETW (WSAQUERYSETW *qs, GUID *sc)
 
 
 static void
-process_lookup_result (void* cls, uint32_t rd_count,
-    const struct GNUNET_GNSRECORD_Data *rd)
+process_lookup_result (void *cls,
+                       uint32_t rd_count,
+                       const struct GNUNET_GNSRECORD_Data *rd)
 {
+  struct request *rq = cls;
   int i, j, csanum;
-  struct request *rq = (struct request *) cls;
   struct GNUNET_W32RESOLVER_GetMessage *msg;
+  struct GNUNET_MQ_Envelope *msg_env;
   struct GNUNET_MessageHeader *msgend;
+  struct GNUNET_MQ_Envelope *msgend_env;
   WSAQUERYSETW *qs;
   size_t size;
   size_t size_recalc;
@@ -327,18 +212,20 @@ process_lookup_result (void* cls, uint32_t rd_count,
   size_t blobaddrcount = 0;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-      "Got lookup result with count %u for rq %p with client %p\n",
-      rd_count, rq, rq->client);
+              "Got lookup result with count %u for rq %p with client %p\n",
+              rd_count,
+              rq,
+              rq->client);
   rq->lookup_request = NULL;
 
-  if (rd_count == 0)
+  if (0 == rd_count)
   {
-    size = sizeof (struct GNUNET_MessageHeader);
-    msg = GNUNET_malloc (size);
-    msg->header.size = htons (size);
-    msg->header.type = htons (GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
-    transmit (rq->client, &msg->header);
-    GNUNET_CONTAINER_DLL_remove (rq_head, rq_tail, rq);
+    msgend_env = GNUNET_MQ_msg (msgend, GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
+    GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (rq->client),
+                    msgend_env);
+    GNUNET_CONTAINER_DLL_remove (rq_head,
+                                 rq_tail,
+                                 rq);
     GNUNET_free_non_null (rq->name);
     if (rq->u8name)
       free (rq->u8name);
@@ -396,9 +283,9 @@ process_lookup_result (void* cls, uint32_t rd_count,
     size += blobsize;
   }
   size_recalc = sizeof (struct GNUNET_W32RESOLVER_GetMessage) + sizeof (WSAQUERYSETW);
-  msg = GNUNET_malloc (size);
-  msg->header.size = htons (size - sizeof (struct GNUNET_MessageHeader));
-  msg->header.type = htons (GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
+  msg_env = GNUNET_MQ_msg_extra (msg,
+                                 size - sizeof (struct GNUNET_MessageHeader),
+                                 GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
   msg->af = htonl (rq->af);
   msg->sc_data1 = htonl (rq->sc.Data1);
   msg->sc_data2 = htons (rq->sc.Data2);
@@ -557,19 +444,24 @@ process_lookup_result (void* cls, uint32_t rd_count,
     }
     he->h_addr_list[j] = NULL;
   }
-  msgend = GNUNET_new (struct GNUNET_MessageHeader);
-
-  msgend->type = htons (GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
-  msgend->size = htons (sizeof (struct GNUNET_MessageHeader));
+  msgend_env = GNUNET_MQ_msg (msgend, GNUNET_MESSAGE_TYPE_W32RESOLVER_RESPONSE);
 
   if ((char *) ptr - (char *) msg != size || size_recalc != size || size_recalc != ((char *) ptr - (char *) msg))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error in WSAQUERYSETW size calc: expected %lu, got %lu (recalc %lu)\n", size, (unsigned long) ((char *) ptr - (char *) msg), size_recalc);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Error in WSAQUERYSETW size calc: expected %u, got %lu (recalc %u)\n",
+                size,
+                (unsigned long) ((char *) ptr - (char *) msg),
+                size_recalc);
   }
   MarshallWSAQUERYSETW (qs, &rq->sc);
-  transmit (rq->client, &msg->header);
-  transmit (rq->client, msgend);
-  GNUNET_CONTAINER_DLL_remove (rq_head, rq_tail, rq);
+  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (rq->client),
+                  msg_env);
+  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (rq->client),
+                  msgend_env);
+  GNUNET_CONTAINER_DLL_remove (rq_head,
+                               rq_tail,
+                               rq);
   GNUNET_free_non_null (rq->name);
   if (rq->u8name)
     free (rq->u8name);
@@ -578,8 +470,10 @@ process_lookup_result (void* cls, uint32_t rd_count,
 
 
 static void
-get_ip_from_hostname (struct GNUNET_SERVER_Client *client,
-    const wchar_t *name, int af, GUID sc)
+get_ip_from_hostname (struct GNUNET_SERVICE_Client *client,
+                      const wchar_t *name,
+                      int af,
+                      GUID sc)
 {
   struct request *rq;
   char *hostname;
@@ -610,10 +504,19 @@ get_ip_from_hostname (struct GNUNET_SERVER_Client *client,
   else
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Unknown GUID: %08X-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
-                sc.Data1, sc.Data2, sc.Data3, sc.Data4[0], sc.Data4[1], sc.Data4[2],
-                sc.Data4[3], sc.Data4[4], sc.Data4[5], sc.Data4[6], sc.Data4[7]);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+                "Unknown GUID: %08lX-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
+                sc.Data1,
+                sc.Data2,
+                sc.Data3,
+                sc.Data4[0],
+                sc.Data4[1],
+                sc.Data4[2],
+                sc.Data4[3],
+                sc.Data4[4],
+                sc.Data4[5],
+                sc.Data4[6],
+                sc.Data4[7]);
+    GNUNET_SERVICE_client_drop (client);
     return;
   }
 
@@ -640,24 +543,31 @@ get_ip_from_hostname (struct GNUNET_SERVER_Client *client,
   if (namelen)
   {
     rq->name = GNUNET_malloc ((namelen + 1) * sizeof (wchar_t));
-    GNUNET_memcpy (rq->name, name, (namelen + 1) * sizeof (wchar_t));
+    GNUNET_memcpy (rq->name,
+                   name,
+                   (namelen + 1) * sizeof (wchar_t));
     rq->u8name = hostname;
   }
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Launching a lookup for client %p with rq %p\n",
-              client, rq);
-
-  rq->lookup_request = GNUNET_GNS_lookup (gns, hostname, &gns_master_pubkey,
-      rtype, GNUNET_NO /* Use DHT */, &gns_short_privkey, &process_lookup_result,
-      rq);
-
+              client,
+              rq);
+  rq->lookup_request = GNUNET_GNS_lookup (gns,
+                                          hostname,
+                                          &gns_master_pubkey,
+                                          rtype,
+                                          GNUNET_NO /* Use DHT */,
+                                          &process_lookup_result,
+                                          rq);
   if (NULL != rq->lookup_request)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Lookup launched, waiting for a reply\n");
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    GNUNET_CONTAINER_DLL_insert (rq_head, rq_tail, rq);
+    GNUNET_SERVICE_client_continue (client);
+    GNUNET_CONTAINER_DLL_insert (rq_head,
+                                 rq_tail,
+                                 rq);
   }
   else
   {
@@ -667,110 +577,105 @@ get_ip_from_hostname (struct GNUNET_SERVER_Client *client,
     if (rq->u8name)
       free (rq->u8name);
     GNUNET_free (rq);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    GNUNET_SERVICE_client_drop (client);
   }
 }
 
 
 /**
- * Handle GET-message.
+ * Check GET-message.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is well-formed
  */
-static void
-handle_get (void *cls, struct GNUNET_SERVER_Client *client,
-            const struct GNUNET_MessageHeader *message)
+static int
+check_get (void *cls,
+            const struct GNUNET_W32RESOLVER_GetMessage *msg)
 {
-  uint16_t msize;
-  const struct GNUNET_W32RESOLVER_GetMessage *msg;
-  GUID sc;
   uint16_t size;
-  int i;
   const wchar_t *hostname;
-  int af;
-
-  if (!got_egos)
-  {
-    /*
-     * FIXME: be done with GNUNET_OK, put the get request into a queue?
-     * or postpone GNUNET_SERVER_add_handlers() until egos are obtained?
-     */
-    GNUNET_SERVER_receive_done (client, GNUNET_NO);
-    return;
-  }
 
-  msize = ntohs (message->size);
-  if (msize <= sizeof (struct GNUNET_W32RESOLVER_GetMessage))
+  if (! got_egos)
   {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("Not ready to process requests, lacking ego data\n"));
+    return GNUNET_SYSERR;
   }
-  msg = (const struct GNUNET_W32RESOLVER_GetMessage *) message;
-  size = msize - sizeof (struct GNUNET_W32RESOLVER_GetMessage);
-  af = ntohl (msg->af);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-      "Got NBO GUID: %08X-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
-      msg->sc_data1, msg->sc_data2, msg->sc_data3, msg->sc_data4[0], msg->sc_data4[1],
-      msg->sc_data4[2], msg->sc_data4[3], msg->sc_data4[4], msg->sc_data4[5],
-      msg->sc_data4[6], msg->sc_data4[7]);
-  sc.Data1 = ntohl (msg->sc_data1);
-  sc.Data2 = ntohs (msg->sc_data2);
-  sc.Data3 = ntohs (msg->sc_data3);
-  for (i = 0; i < 8; i++)
-    sc.Data4[i] = msg->sc_data4[i];
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got GUID: %08X-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
-              sc.Data1, sc.Data2, sc.Data3, sc.Data4[0], sc.Data4[1], sc.Data4[2],
-              sc.Data4[3], sc.Data4[4], sc.Data4[5], sc.Data4[6], sc.Data4[7]);
-
+  size = ntohs (msg->header.size) - sizeof (struct GNUNET_W32RESOLVER_GetMessage);
   hostname = (const wchar_t *) &msg[1];
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "name of %u bytes (last word is 0x%0X): %*S\n",
-        size, hostname[size / 2 - 2], size / 2, hostname);
   if (hostname[size / 2 - 1] != L'\0')
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "name of length %u, not 0-terminated (%d-th word is 0x%0X): %*S\n",
-        size, size / 2 - 1, hostname[size / 2 - 1], size, hostname);
     GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    return GNUNET_SYSERR;
   }
-  get_ip_from_hostname (client, hostname, af, sc);
+  return GNUNET_OK;
 }
 
 
 /**
- * Method called to with the ego we are to use for shortening
- * during the lookup.
+ * Handle GET-message.
  *
- * @param cls closure (NULL, unused)
- * @param ego ego handle, NULL if not found
- * @param ctx context for application to store data for this ego
- *                 (during the lifetime of this process, initially NULL)
- * @param name name assigned by the user for this ego,
- *                   NULL if the user just deleted the ego and it
- *                   must thus no longer be used
+ * @param cls identification of the client
+ * @param msg the actual message
  */
 static void
-identity_shorten_cb (void *cls,
-         struct GNUNET_IDENTITY_Ego *ego,
-         void **ctx,
-         const char *name)
+handle_get (void *cls,
+            const struct GNUNET_W32RESOLVER_GetMessage *msg)
 {
-  id_op = NULL;
-  if (NULL == ego)
-  {
-    fprintf (stderr,
-       _("Ego for `gns-short' not found. This is not really fatal, but i'll pretend that it is and refuse to perform a lookup.  Did you run gnunet-gns-import.sh?\n"));
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  gns_short_privkey = *GNUNET_IDENTITY_ego_get_private_key (ego);
-  got_egos = 1;
+  struct GNUNET_SERVICE_Client *client = cls;
+  GUID sc;
+  uint16_t size;
+  const wchar_t *hostname;
+  int af;
+
+  size = ntohs (msg->header.size) - sizeof (struct GNUNET_W32RESOLVER_GetMessage);
+  af = ntohl (msg->af);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Got NBO GUID: %08X-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
+              msg->sc_data1,
+              msg->sc_data2,
+              msg->sc_data3,
+              msg->sc_data4[0],
+              msg->sc_data4[1],
+              msg->sc_data4[2],
+              msg->sc_data4[3],
+              msg->sc_data4[4],
+              msg->sc_data4[5],
+              msg->sc_data4[6],
+              msg->sc_data4[7]);
+  sc.Data1 = ntohl (msg->sc_data1);
+  sc.Data2 = ntohs (msg->sc_data2);
+  sc.Data3 = ntohs (msg->sc_data3);
+  for (int i = 0; i < 8; i++)
+    sc.Data4[i] = msg->sc_data4[i];
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Got GUID: %08lX-%04X-%04X-%02X-%02X-%02X-%02X-%02X-%02X-%02X-%02X\n",
+              sc.Data1,
+              sc.Data2,
+              sc.Data3,
+              sc.Data4[0],
+              sc.Data4[1],
+              sc.Data4[2],
+              sc.Data4[3],
+              sc.Data4[4],
+              sc.Data4[5],
+              sc.Data4[6],
+              sc.Data4[7]);
+  hostname = (const wchar_t *) &msg[1];
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Name of %u bytes (last word is 0x%0X): %*S\n",
+              size,
+              hostname[size / 2 - 2],
+              size / 2,
+              hostname);
+  get_ip_from_hostname (client,
+                        hostname,
+                        af,
+                        sc);
 }
 
+
 /**
  * Method called to with the ego we are to use for the lookup,
  * when the ego is the one for the default master zone.
@@ -785,22 +690,21 @@ identity_shorten_cb (void *cls,
  */
 static void
 identity_master_cb (void *cls,
-        struct GNUNET_IDENTITY_Ego *ego,
-        void **ctx,
-        const char *name)
+                    struct GNUNET_IDENTITY_Ego *ego,
+                    void **ctx,
+                    const char *name)
 {
   id_op = NULL;
   if (NULL == ego)
   {
-    fprintf (stderr,
-       _("Ego for `gns-master' not found, cannot perform lookup.  Did you run gnunet-gns-import.sh?\n"));
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("Ego for `gns-master' not found, cannot perform lookup.  Did you run gnunet-gns-import.sh?\n"));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  GNUNET_IDENTITY_ego_get_public_key (ego, &gns_master_pubkey);
-  id_op = GNUNET_IDENTITY_get (identity, "gns-short", &identity_shorten_cb,
-      NULL);
-  GNUNET_assert (NULL != id_op);
+  GNUNET_IDENTITY_ego_get_public_key (ego,
+                                      &gns_master_pubkey);
+  got_egos = 1;
 }
 
 
@@ -808,59 +712,90 @@ identity_master_cb (void *cls,
  * Start up gns-helper-w32 service.
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-    const struct GNUNET_CONFIGURATION_Handle *cfg)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {&handle_get, NULL, GNUNET_MESSAGE_TYPE_W32RESOLVER_REQUEST, 0},
-    {NULL, NULL, 0, 0}
-  };
-
   gns = GNUNET_GNS_connect (cfg);
   if (NULL == gns)
   {
-    fprintf (stderr, _("Failed to connect to GNS\n"));
+    fprintf (stderr,
+             _("Failed to connect to GNS\n"));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
   GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                  NULL);
-
-  identity = GNUNET_IDENTITY_connect (cfg, NULL, NULL);
+  identity = GNUNET_IDENTITY_connect (cfg,
+                                      NULL,
+                                      NULL);
   if (NULL == identity)
   {
-    fprintf (stderr, _("Failed to connect to identity service\n"));
+    fprintf (stderr,
+             _("Failed to connect to identity service\n"));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-
-  id_op = GNUNET_IDENTITY_get (identity, "gns-master", &identity_master_cb,
-      NULL);
+  id_op = GNUNET_IDENTITY_get (identity,
+                               "gns-master",
+                               &identity_master_cb,
+                               NULL);
   GNUNET_assert (NULL != id_op);
-
-  GNUNET_SERVER_add_handlers (server, handlers);
 }
 
 
 /**
- * The main function for gns-helper-w32.
+ * Handle client connecting to the service.
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls NULL
+ * @param client the new client
+ * @param mq the message queue of @a client
+ * @return @a client
  */
-int
-main (int argc, char *const *argv)
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *client,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  int ret;
+  return client;
+}
 
-  ret = GNUNET_SERVICE_run (argc, argv, "gns-helper-service-w32",
-      GNUNET_SERVICE_OPTION_NONE, &run, NULL);
-  return (GNUNET_OK == ret) ? 0 : 1;
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
+{
+  GNUNET_assert (internal_cls == client);
 }
 
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("gns-helper-service-w32",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (get,
+                        GNUNET_MESSAGE_TYPE_W32RESOLVER_REQUEST,
+                        struct GNUNET_W32RESOLVER_GetMessage,
+                        NULL),
+ GNUNET_MQ_handler_end());
+
+
 /* end of gnunet-gns-helper-service-w32.c */
diff --git a/src/gns/gnunet-gns-import.c b/src/gns/gnunet-gns-import.c
index e98babfa8..49f6e495f 100644
--- a/src/gns/gnunet-gns-import.c
+++ b/src/gns/gnunet-gns-import.c
@@ -224,45 +224,52 @@ zone_iterator (void *cls,
     else if (0 == strcmp (rname, "pin"))
       check_pkey (rd_len, rd, pin_zone_pkey, &found_pin_rec);
   }
-  if (NULL == rname && 0 == rd_len && NULL == rd)
+  GNUNET_NAMESTORE_zone_iterator_next (list_it);
+}
+
+static void
+zone_iteration_error (void *cls)
+{
+  enum GNUNET_OS_ProcessStatusType st;
+  unsigned long code;
+  if (!found_private_rec)
   {
-    enum GNUNET_OS_ProcessStatusType st;
-    unsigned long code;
-    if (!found_private_rec)
+    if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
+        "gnunet-namestore",
+        "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "private", "-p", "-t", "PKEY", "-V", private_zone_pkey, NULL))
     {
-      if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
-          "gnunet-namestore",
-          "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "private", "-p", "-t", "PKEY", "-V", private_zone_pkey, NULL))
-      {
-        ret = 8;
-        return;
-      }
+      ret = 8;
+      return;
     }
-    if (!found_short_rec)
+  }
+  if (!found_short_rec)
+  {
+    if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
+        "gnunet-namestore",
+        "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "short", "-p", "-t", "PKEY", "-V", short_zone_pkey, NULL))
     {
-      if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
-          "gnunet-namestore",
-          "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "short", "-p", "-t", "PKEY", "-V", short_zone_pkey, NULL))
-      {
-        ret = 9;
-        return;
-      }
+      ret = 9;
+      return;
     }
-    if (!found_pin_rec)
+  }
+  if (!found_pin_rec)
+  {
+    if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
+        "gnunet-namestore",
+        "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "pin", "-p", "-t", "PKEY", "-V", pin_zone_pkey, NULL))
     {
-      if (0 != run_process_and_wait (GNUNET_NO, GNUNET_OS_INHERIT_STD_OUT_AND_ERR, NULL, NULL, &st, &code,
-          "gnunet-namestore",
-          "gnunet-namestore", "-z", "master-zone", "-a", "-e", "never", "-n", "pin", "-p", "-t", "PKEY", "-V", pin_zone_pkey, NULL))
-      {
-        ret = 10;
-        return;
-      }
+      ret = 10;
+      return;
     }
-    list_it = NULL;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
   }
-  GNUNET_NAMESTORE_zone_iterator_next (list_it);
+  list_it = NULL;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+zone_iteration_finished (void *cls)
+{
 }
 
 
@@ -317,7 +324,7 @@ get_ego (void *cls,
       return;
     }
     list_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
-        &master_pk, &zone_iterator, NULL);
+        &master_pk, &zone_iteration_error, NULL, &zone_iterator, NULL, &zone_iteration_finished, NULL);
     if (NULL == list_it)
     {
       ret = 12;
diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c
index 3a38970a8..2a6de1c30 100644
--- a/src/gns/gnunet-gns-proxy.c
+++ b/src/gns/gnunet-gns-proxy.c
@@ -606,7 +606,7 @@ struct Socks5Request
    * Headers from response
    */
   struct HttpResponseHeader *header_tail;
-  
+
   /**
    * SSL Certificate status
    */
@@ -621,7 +621,7 @@ struct Socks5Request
 /**
  * The port the proxy is running on (default 7777)
  */
-static unsigned long port = GNUNET_GNS_PROXY_PORT;
+static unsigned long long port = GNUNET_GNS_PROXY_PORT;
 
 /**
  * The CA file (pem) to use for the proxy CA
@@ -695,16 +695,6 @@ static struct Socks5Request *s5r_tail;
 static struct GNUNET_CRYPTO_EcdsaPublicKey local_gns_zone;
 
 /**
- * The users local shorten zone
- */
-static struct GNUNET_CRYPTO_EcdsaPrivateKey local_shorten_zone;
-
-/**
- * Is shortening enabled?
- */
-static int do_shorten;
-
-/**
  * The CA for SSL certificate generation
  */
 static struct ProxyCA proxy_ca;
@@ -873,9 +863,10 @@ check_ssl_certificate (struct Socks5Request *s5r)
   gnutls_x509_crt_t x509_cert;
   int rc;
   const char *name;
-  
+
   s5r->ssl_checked = GNUNET_YES;
-  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "XXXXXX\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Checking SSL certificate\n");
   if (CURLE_OK !=
       curl_easy_getinfo (s5r->curl,
                          CURLINFO_TLS_SESSION,
@@ -1039,7 +1030,7 @@ curl_check_hdr (void *buffer, size_t size, size_t nmemb, void *cls)
   size_t delta_cdomain;
   int domain_matched;
   char *tok;
-  
+
   /* first, check SSL certificate */
   if ((GNUNET_YES != s5r->ssl_checked) &&
       (HTTPS_PORT == s5r->port))
@@ -1047,7 +1038,7 @@ curl_check_hdr (void *buffer, size_t size, size_t nmemb, void *cls)
       if (GNUNET_OK != check_ssl_certificate (s5r))
         return 0;
   }
-  
+
   ndup = GNUNET_strndup (buffer, bytes);
   hdr_type = strtok (ndup, ":");
   if (NULL == hdr_type)
@@ -1287,7 +1278,7 @@ curl_upload_cb (void *buf, size_t size, size_t nmemb, void *cls)
   struct Socks5Request *s5r = cls;
   size_t len = size * nmemb;
   size_t to_copy;
-  
+
   if ( (0 == s5r->io_len) &&
        (SOCKS5_SOCKET_UPLOAD_DONE != s5r->state) )
   {
@@ -1763,7 +1754,7 @@ create_response (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Processing %u bytes UPLOAD\n",
                 (unsigned int) *upload_data_size);
-    
+
     /* FIXME: This must be set or a header with Transfer-Encoding: chunked. Else
      * upload callback is not called!
      */
@@ -1892,19 +1883,22 @@ mhd_connection_cb (void *cls,
       {
         if (GNUNET_NETWORK_get_fd (s5r->sock) == sock)
         {
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Context set...\n");
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      "Context set...\n");
+          s5r->ssl_checked = GNUNET_NO;
           *con_cls = s5r;
           break;
         }
       }
-      s5r->ssl_checked = GNUNET_NO;
       break;
     case MHD_CONNECTION_NOTIFY_CLOSED:
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connection closed... cleaning up\n");
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connection closed... cleaning up\n");
       s5r = *con_cls;
       if (NULL == s5r)
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Connection stale!\n");
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "Connection stale!\n");
         return;
       }
       cleanup_s5r (s5r);
@@ -2827,7 +2821,6 @@ do_s5r_read (void *cls)
                                                  &local_gns_zone,
                                                  GNUNET_DNSPARSER_TYPE_A,
                                                  GNUNET_NO /* only cached */,
-                                                 (GNUNET_YES == do_shorten) ? &local_shorten_zone : NULL,
                                                  &handle_gns_result,
                                                  s5r);
             break;
@@ -3115,7 +3108,7 @@ run_cont ()
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Proxy listens on port %lu\n",
+              "Proxy listens on port %llu\n",
               port);
 
   /* start MHD daemon for HTTP */
@@ -3141,46 +3134,6 @@ run_cont ()
 
 
 /**
- * Method called to inform about the egos of the shorten zone of this peer.
- *
- * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
- * this function is only called ONCE, and 'NULL' being passed in
- * @a ego does indicate an error (i.e. name is taken or no default
- * value is known).  If @a ego is non-NULL and if '*ctx'
- * is set in those callbacks, the value WILL be passed to a subsequent
- * call to the identity callback of #GNUNET_IDENTITY_connect (if
- * that one was not NULL).
- *
- * @param cls closure, NULL
- * @param ego ego handle
- * @param ctx context for application to store data for this ego
- *                 (during the lifetime of this process, initially NULL)
- * @param name name assigned by the user for this ego,
- *                   NULL if the user just deleted the ego and it
- *                   must thus no longer be used
- */
-static void
-identity_shorten_cb (void *cls,
-                     struct GNUNET_IDENTITY_Ego *ego,
-                     void **ctx,
-                     const char *name)
-{
-  id_op = NULL;
-  if (NULL == ego)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                _("No ego configured for `shorten-zone`\n"));
-  }
-  else
-  {
-    local_shorten_zone = *GNUNET_IDENTITY_ego_get_private_key (ego);
-    do_shorten = GNUNET_YES;
-  }
-  run_cont ();
-}
-
-
-/**
  * Method called to inform about the egos of the master zone of this peer.
  *
  * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
@@ -3216,10 +3169,7 @@ identity_master_cb (void *cls,
   }
   GNUNET_IDENTITY_ego_get_public_key (ego,
                                       &local_gns_zone);
-  id_op = GNUNET_IDENTITY_get (identity,
-                               "gns-short",
-                               &identity_shorten_cb,
-                               NULL);
+  run_cont ();
 }
 
 
@@ -3232,7 +3182,9 @@ identity_master_cb (void *cls,
  * @param c configuration
  */
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
   char* cafile_cfg = NULL;
@@ -3309,13 +3261,20 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", NULL,
-      gettext_noop ("listen on specified port (default: 7777)"), 1,
-      &GNUNET_GETOPT_set_ulong, &port},
-    {'a', "authority", NULL,
-      gettext_noop ("pem file to use as CA"), 1,
-      &GNUNET_GETOPT_set_string, &cafile_opt},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_ulong ('p',
+                                    "port",
+                                    NULL,
+                                    gettext_noop ("listen on specified port (default: 7777)"),
+                                    &port),
+
+    GNUNET_GETOPT_option_string ('a',
+                                 "authority",
+                                 NULL,
+                                 gettext_noop ("pem file to use as CA"),
+                                 &cafile_opt),
+
     GNUNET_GETOPT_OPTION_END
   };
   static const char* page =
@@ -3323,22 +3282,26 @@ main (int argc, char *const *argv)
     "</head><body>cURL fail</body></html>";
   int ret;
 
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                                 &argc, &argv))
     return 2;
-  GNUNET_log_setup ("gnunet-gns-proxy", "WARNING", NULL);
-  curl_failure_response = MHD_create_response_from_buffer (strlen (page),
-                                                           (void*)page,
-                                                           MHD_RESPMEM_PERSISTENT);
+  GNUNET_log_setup ("gnunet-gns-proxy",
+                    "WARNING",
+                    NULL);
+  curl_failure_response
+    = MHD_create_response_from_buffer (strlen (page),
+                                       (void *) page,
+                                       MHD_RESPMEM_PERSISTENT);
 
   ret =
     (GNUNET_OK ==
-     GNUNET_PROGRAM_run (argc, argv, "gnunet-gns-proxy",
+     GNUNET_PROGRAM_run (argc, argv,
+                         "gnunet-gns-proxy",
                          _("GNUnet GNS proxy"),
                          options,
                          &run, NULL)) ? 0 : 1;
   MHD_destroy_response (curl_failure_response);
   GNUNET_free_non_null ((char *) argv);
-  GNUNET_CRYPTO_ecdsa_key_clear (&local_shorten_zone);
   return ret;
 }
 
diff --git a/src/gns/gnunet-gns.c b/src/gns/gnunet-gns.c
index 17fe4cbda..c0de0f30c 100644
--- a/src/gns/gnunet-gns.c
+++ b/src/gns/gnunet-gns.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012-2013 GNUnet e.V.
+     Copyright (C) 2012-2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -66,16 +66,6 @@ static char *zone_ego_name;
 static char *public_key;
 
 /**
- * Reverse key
- */
-static char *reverse_key;
-
-/**
- * Reverse key
- */
-static struct GNUNET_CRYPTO_EcdsaPublicKey rkey;
-
-/**
  * Set to GNUNET_GNS_LO_LOCAL_MASTER if we are looking up in the master zone.
  */
 static enum GNUNET_GNS_LocalOptions local_options;
@@ -96,11 +86,6 @@ static int rtype;
 static struct GNUNET_GNS_LookupRequest *lookup_request;
 
 /**
- * Handle to reverse lookup request
- */
-static struct GNUNET_GNS_ReverseLookupRequest *rev_lookup_request;
-
-/**
  * Lookup an ego with the identity service.
  */
 static struct GNUNET_IDENTITY_EgoLookup *el;
@@ -174,24 +159,6 @@ do_timeout (void *cls)
   GNUNET_SCHEDULER_shutdown ();
 }
 
-static void
-process_reverse_result (void *cls,
-                        const char *name)
-{
-  rev_lookup_request = NULL;
-  if (NULL == name)
-  {
-    printf ("No name found.\n");
-    return;
-  }
-  if (raw)
-    printf ("%s\n", name);
-  else
-    printf ("%s is known as %s\n",
-            reverse_key,
-            name);
-  GNUNET_SCHEDULER_shutdown ();
-}
 
 /**
  * Function called with the result of a GNS lookup.
@@ -201,7 +168,8 @@ process_reverse_result (void *cls,
  * @param rd array of @a rd_count records with the results
  */
 static void
-process_lookup_result (void *cls, uint32_t rd_count,
+process_lookup_result (void *cls,
+                       uint32_t rd_count,
                        const struct GNUNET_GNSRECORD_Data *rd)
 {
   const char *name = cls;
@@ -253,11 +221,9 @@ process_lookup_result (void *cls, uint32_t rd_count,
  * identified by the given public key and the shorten zone.
  *
  * @param pkey public key to use for the zone, can be NULL
- * @param shorten_key private key used for shortening, can be NULL
  */
 static void
-lookup_with_keys (const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey,
-                  const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_key)
+lookup_with_public_key (const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey)
 {
   if (NULL != lookup_type)
     rtype = GNUNET_GNSRECORD_typename_to_number (lookup_type);
@@ -277,18 +243,9 @@ lookup_with_keys (const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey,
                                         pkey,
                                         rtype,
                                         local_options,
-                                        shorten_key,
                                         &process_lookup_result,
                                         lookup_name);
   }
-  else if (NULL != reverse_key)
-  {
-    rev_lookup_request = GNUNET_GNS_reverse_lookup (gns,
-                                                &rkey,
-                                                pkey,
-                                                &process_reverse_result,
-                                                NULL);
-  }
   else
   {
     fprintf (stderr,
@@ -300,63 +257,6 @@ lookup_with_keys (const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey,
 
 
 /**
- * Method called to with the ego we are to use for shortening
- * during the lookup.
- *
- * @param cls closure contains the public key to use
- * @param ego ego handle, NULL if not found
- * @param ctx context for application to store data for this ego
- *                 (during the lifetime of this process, initially NULL)
- * @param name name assigned by the user for this ego,
- *                   NULL if the user just deleted the ego and it
- *                   must thus no longer be used
- */
-static void
-identity_shorten_cb (void *cls,
-                     struct GNUNET_IDENTITY_Ego *ego,
-                     void **ctx,
-                     const char *name)
-{
-  struct GNUNET_CRYPTO_EcdsaPublicKey *pkeym = cls;
-
-  id_op = NULL;
-  if (NULL == ego)
-    lookup_with_keys (pkeym, NULL);
-  else
-    lookup_with_keys (pkeym,
-                      GNUNET_IDENTITY_ego_get_private_key (ego));
-  GNUNET_free (pkeym);
-}
-
-
-/**
- * Perform the actual resolution, starting with the zone
- * identified by the given public key.
- *
- * @param pkey public key to use for the zone
- */
-static void
-lookup_with_public_key (const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey)
-{
-  struct GNUNET_CRYPTO_EcdsaPublicKey *pkeym;
-
-  GNUNET_assert (NULL != pkey);
-  pkeym = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPublicKey);
-  *pkeym = *pkey;
-  GNUNET_break (NULL == id_op);
-  id_op = GNUNET_IDENTITY_get (identity,
-                               "gns-short",
-                               &identity_shorten_cb,
-                               pkeym);
-  if (NULL == id_op)
-  {
-    GNUNET_break (0);
-    lookup_with_keys (pkey, NULL);
-  }
-}
-
-
-/**
  * Method called to with the ego we are to use for the lookup,
  * when the ego is determined by a name.
  *
@@ -449,7 +349,6 @@ run (void *cls,
 
   cfg = c;
   gns = GNUNET_GNS_connect (cfg);
-  identity = GNUNET_IDENTITY_connect (cfg, NULL, NULL);
   if (NULL == gns)
   {
     fprintf (stderr,
@@ -457,22 +356,13 @@ run (void *cls,
     return;
   }
   tt = GNUNET_SCHEDULER_add_delayed (timeout,
-                                     &do_timeout, NULL);
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
-  if (NULL != reverse_key)
-  {
-    if (GNUNET_OK !=
-        GNUNET_CRYPTO_ecdsa_public_key_from_string (reverse_key,
-                                                    strlen (reverse_key),
-                                                    &rkey))
-    {
-      fprintf (stderr,
-               _("Reverse key `%s' is not well-formed\n"),
-               reverse_key);
-      GNUNET_SCHEDULER_shutdown ();
-      return;
-    }
-  }
+                                     &do_timeout,
+                                     NULL);
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
+  identity = GNUNET_IDENTITY_connect (cfg,
+                                      NULL,
+                                      NULL);
   if (NULL != public_key)
   {
     if (GNUNET_OK !=
@@ -489,20 +379,6 @@ run (void *cls,
     lookup_with_public_key (&pkey);
     return;
   }
-  if (NULL != reverse_key)
-  {
-    if (GNUNET_OK !=
-        GNUNET_CRYPTO_ecdsa_public_key_from_string (reverse_key,
-                                                    strlen (reverse_key),
-                                                    &rkey))
-    {
-      fprintf (stderr,
-               _("Reverse key `%s' is not well-formed\n"),
-               reverse_key);
-      GNUNET_SCHEDULER_shutdown ();
-      return;
-    }
-  }
   if (NULL != zone_ego_name)
   {
     el = GNUNET_IDENTITY_ego_lookup (cfg,
@@ -541,42 +417,62 @@ run (void *cls,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'u', "lookup", "NAME",
-      gettext_noop ("Lookup a record for the given name"), 1,
-      &GNUNET_GETOPT_set_string, &lookup_name},
-    {'t', "type", "TYPE",
-      gettext_noop ("Specify the type of the record to lookup"), 1,
-      &GNUNET_GETOPT_set_string, &lookup_type},
-    { 'T', "timeout", "DELAY",
-      gettext_noop ("Specify timeout for the lookup"), 1,
-      &GNUNET_GETOPT_set_relative_time, &timeout },
-    {'r', "raw", NULL,
-      gettext_noop ("No unneeded output"), 0,
-      &GNUNET_GETOPT_set_one, &raw},
-    {'p', "public-key", "PKEY",
-      gettext_noop ("Specify the public key of the zone to lookup the record in"), 1,
-      &GNUNET_GETOPT_set_string, &public_key},
-    {'z', "zone", "NAME",
-      gettext_noop ("Specify the name of the ego of the zone to lookup the record in"), 1,
-      &GNUNET_GETOPT_set_string, &zone_ego_name},
-    {'R', "reverse", "PKEY",
-      gettext_noop ("Specify the public key of the zone to reverse lookup a name for"), 1,
-      &GNUNET_GETOPT_set_string, &reverse_key},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('u',
+                                 "lookup",
+                                 "NAME",
+                                 gettext_noop ("Lookup a record for the given name"),
+                                 &lookup_name),
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("Specify the type of the record to lookup"),
+                                 &lookup_type),
+
+    GNUNET_GETOPT_option_relative_time ('T',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("Specify timeout for the lookup"),
+                                            &timeout),
+
+    GNUNET_GETOPT_option_flag ('r',
+                                  "raw",
+                                  gettext_noop ("No unneeded output"),
+                                  &raw),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "public-key",
+                                 "PKEY",
+                                 gettext_noop ("Specify the public key of the zone to lookup the record in"),
+                                 &public_key),
+    
+    GNUNET_GETOPT_option_string ('z',
+                                 "zone",
+                                 "NAME",
+                                 gettext_noop ("Specify the name of the ego of the zone to lookup the record in"),
+                                 &zone_ego_name),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
 
   timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                                 &argc, &argv))
     return 2;
 
-  GNUNET_log_setup ("gnunet-gns", "WARNING", NULL);
+  GNUNET_log_setup ("gnunet-gns",
+                    "WARNING",
+                    NULL);
   ret =
     (GNUNET_OK ==
-     GNUNET_PROGRAM_run (argc, argv, "gnunet-gns",
+     GNUNET_PROGRAM_run (argc, argv,
+                         "gnunet-gns",
                          _("GNUnet GNS resolver tool"),
                          options,
                          &run, NULL)) ? 0 : 1;
diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c
index cec31ff48..0ca25ac19 100644
--- a/src/gns/gnunet-service-gns.c
+++ b/src/gns/gnunet-service-gns.c
@@ -29,14 +29,11 @@
 #include "gnunet_dnsparser_lib.h"
 #include "gnunet_dht_service.h"
 #include "gnunet_namecache_service.h"
-#include "gnunet_namestore_service.h"
 #include "gnunet_identity_service.h"
 #include "gnunet_gns_service.h"
 #include "gnunet_statistics_service.h"
 #include "gns.h"
 #include "gnunet-service-gns_resolver.h"
-#include "gnunet-service-gns_reverser.h"
-#include "gnunet-service-gns_shorten.h"
 #include "gnunet-service-gns_interceptor.h"
 #include "gnunet_protocols.h"
 
@@ -61,7 +58,7 @@ struct ClientLookupHandle
    * We keep these in a DLL.
    */
   struct ClientLookupHandle *prev;
-  
+
   /**
    * Client handle
    */
@@ -73,11 +70,6 @@ struct ClientLookupHandle
   struct GNS_ResolverHandle *lookup;
 
   /**
-   * Active handle for a reverse lookup
-   */
-  struct GNS_ReverserHandle *rev_lookup;
-
-  /**
    * request id
    */
   uint32_t request_id;
@@ -119,11 +111,6 @@ static struct GNUNET_DHT_Handle *dht_handle;
 static struct GNUNET_NAMECACHE_Handle *namecache_handle;
 
 /**
- * Our handle to the namestore service
- */
-static struct GNUNET_NAMESTORE_Handle *namestore_handle;
-
-/**
  * Our handle to the identity service
  */
 static struct GNUNET_IDENTITY_Handle *identity_handle;
@@ -173,19 +160,12 @@ shutdown_task (void *cls)
     identity_handle = NULL;
   }
   GNS_resolver_done ();
-  GNS_reverse_done ();
-  GNS_shorten_done ();
   if (NULL != statistics)
   {
     GNUNET_STATISTICS_destroy (statistics,
                                GNUNET_NO);
     statistics = NULL;
   }
-  if (NULL != namestore_handle)
-  {
-    GNUNET_NAMESTORE_disconnect (namestore_handle);
-    namestore_handle = NULL;
-  }
   if (NULL != namecache_handle)
   {
     GNUNET_NAMECACHE_disconnect (namecache_handle);
@@ -221,15 +201,13 @@ client_disconnect_cb (void *cls,
   {
     if (NULL != clh->lookup)
       GNS_resolver_lookup_cancel (clh->lookup);
-    if (NULL != clh->rev_lookup)
-      GNS_reverse_lookup_cancel (clh->rev_lookup);
     GNUNET_CONTAINER_DLL_remove (gc->clh_head,
                                  gc->clh_tail,
                                  clh);
     GNUNET_free (clh);
   }
 
-  GNUNET_free (gc); 
+  GNUNET_free (gc);
 }
 
 
@@ -288,7 +266,9 @@ send_lookup_response (void* cls,
                                       (char*) &rmsg[1]);
   GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq(clh->gc->client),
                   env);
-  GNUNET_CONTAINER_DLL_remove (clh->gc->clh_head, clh->gc->clh_tail, clh);
+  GNUNET_CONTAINER_DLL_remove (clh->gc->clh_head,
+                               clh->gc->clh_tail,
+                               clh);
   GNUNET_free (clh);
   GNUNET_STATISTICS_update (statistics,
                             "Completed lookups", 1,
@@ -299,47 +279,6 @@ send_lookup_response (void* cls,
                             GNUNET_NO);
 }
 
-/**
- * Reply to client with the result from our reverse lookup.
- *
- * @param cls the closure (our client lookup handle)
- * @param rd_count the number of records in @a rd
- * @param rd the record data
- */
-static void
-send_reverse_lookup_response (void* cls,
-                              const char *name)
-{
-  struct ClientLookupHandle *clh = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct ReverseLookupResultMessage *rmsg;
-  size_t len;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending LOOKUP_RESULT message with %s\n",
-              name);
-
-  if (NULL == name)
-    len = 1;
-  else
-    len = strlen (name) + 1;
-  env = GNUNET_MQ_msg_extra (rmsg,
-                             len,
-                             GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP_RESULT);
-  rmsg->id = clh->request_id;
-  if (1 < len)
-    GNUNET_memcpy ((char*) &rmsg[1],
-                   name,
-                   strlen (name));
-  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq(clh->gc->client),
-                  env);
-  GNUNET_CONTAINER_DLL_remove (clh->gc->clh_head, clh->gc->clh_tail, clh);
-  GNUNET_free (clh);
-  GNUNET_STATISTICS_update (statistics,
-                            "Completed reverse lookups", 1,
-                            GNUNET_NO);
-}
-
 
 /**
  * Checks a #GNUNET_MESSAGE_TYPE_GNS_LOOKUP message
@@ -371,6 +310,7 @@ check_lookup (void *cls,
   return GNUNET_OK;
 }
 
+
 /**
  * Handle lookup requests from client
  *
@@ -387,20 +327,18 @@ handle_lookup (void *cls,
   struct ClientLookupHandle *clh;
   char *nameptr = name;
   const char *utf_in;
-  const struct GNUNET_CRYPTO_EcdsaPrivateKey *key;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received LOOKUP message\n");
   GNUNET_SERVICE_client_continue (gc->client);
-  if (GNUNET_YES == ntohs (sh_msg->have_key))
-    key = &sh_msg->shorten_key;
-  else
-    key = NULL;
   utf_in = (const char *) &sh_msg[1];
   GNUNET_STRINGS_utf8_tolower (utf_in, nameptr);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received LOOKUP `%s' message\n",
+              name);
 
   clh = GNUNET_new (struct ClientLookupHandle);
-  GNUNET_CONTAINER_DLL_insert (gc->clh_head, gc->clh_tail, clh);
+  GNUNET_CONTAINER_DLL_insert (gc->clh_head,
+                               gc->clh_tail,
+                               clh);
   clh->gc = gc;
   clh->request_id = sh_msg->id;
   if ( (GNUNET_DNSPARSER_TYPE_A == ntohl (sh_msg->type)) &&
@@ -422,7 +360,6 @@ handle_lookup (void *cls,
   clh->lookup = GNS_resolver_lookup (&sh_msg->zone,
                                      ntohl (sh_msg->type),
                                      name,
-                                     key,
                                      (enum GNUNET_GNS_LocalOptions) ntohs (sh_msg->options),
                                      &send_lookup_response, clh);
   GNUNET_STATISTICS_update (statistics,
@@ -430,82 +367,6 @@ handle_lookup (void *cls,
                             1, GNUNET_NO);
 }
 
-/**
- * Handle reverse lookup requests from client
- *
- * @param cls the closure
- * @param client the client
- * @param message the message
- */
-static void
-handle_rev_lookup (void *cls,
-                   const struct ReverseLookupMessage *sh_msg)
-{
-  struct GnsClient *gc = cls;
-  struct ClientLookupHandle *clh;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received REVERSE_LOOKUP message\n");
-  GNUNET_SERVICE_client_continue (gc->client);
-
-  clh = GNUNET_new (struct ClientLookupHandle);
-  GNUNET_CONTAINER_DLL_insert (gc->clh_head, gc->clh_tail, clh);
-  clh->gc = gc;
-  clh->request_id = sh_msg->id;
-  clh->rev_lookup = GNS_reverse_lookup (&sh_msg->zone_pkey,
-                                        &sh_msg->root_pkey,
-                                        &send_reverse_lookup_response,
-                                        clh);
-  GNUNET_STATISTICS_update (statistics,
-                            "Reverse lookup attempts",
-                            1, GNUNET_NO);
-}
-
-
-/**
- * Method called to inform about the ego to be used for the master zone
- * for DNS interceptions.
- *
- * This function is only called ONCE, and 'NULL' being passed in
- * @a ego does indicate that interception is not configured.
- * If @a ego is non-NULL, we should start to intercept DNS queries
- * and resolve ".gnu" queries using the given ego as the master zone.
- *
- * @param cls closure, our `const struct GNUNET_CONFIGURATION_Handle *c`
- * @param ego ego handle
- * @param ctx context for application to store data for this ego
- *                 (during the lifetime of this process, initially NULL)
- * @param name name assigned by the user for this ego,
- *                   NULL if the user just deleted the ego and it
- *                   must thus no longer be used
- */
-static void
-identity_reverse_cb (void *cls,
-                       struct GNUNET_IDENTITY_Ego *ego,
-                       void **ctx,
-                       const char *name)
-{
-  identity_op = NULL;
-
-  if (NULL == ego)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                _("No ego configured for `%s`\n"),
-                "gns-master");
-
-    return;
-  }
-  if (GNUNET_SYSERR ==
-      GNS_reverse_init (namestore_handle,
-                        GNUNET_IDENTITY_ego_get_private_key (ego),
-                        name))
-  {
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_add_now (&shutdown_task, NULL);
-    return;
-  }
-}
-
 
 /**
  * Method called to inform about the ego to be used for the master zone
@@ -532,16 +393,10 @@ identity_intercept_cb (void *cls,
 {
   const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
   struct GNUNET_CRYPTO_EcdsaPublicKey dns_root;
-  identity_op = NULL;
 
+  identity_op = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Looking for gns-intercept ego\n");
-  identity_op = GNUNET_IDENTITY_get (identity_handle,
-                                     "gns-reverse",
-                                     &identity_reverse_cb,
-                                     (void*)cfg);
-
-
   if (NULL == ego)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -553,10 +408,12 @@ identity_intercept_cb (void *cls,
   GNUNET_IDENTITY_ego_get_public_key (ego,
                                       &dns_root);
   if (GNUNET_SYSERR ==
-      GNS_interceptor_init (&dns_root, cfg))
+      GNS_interceptor_init (&dns_root,
+                            cfg))
   {
     GNUNET_break (0);
-    GNUNET_SCHEDULER_add_now (&shutdown_task, NULL);
+    GNUNET_SCHEDULER_add_now (&shutdown_task,
+                              NULL);
     return;
   }
 }
@@ -577,16 +434,8 @@ run (void *cls,
   unsigned long long max_parallel_bg_queries = 16;
 
   v6_enabled = GNUNET_NETWORK_test_pf (PF_INET6);
-  v4_enabled = GNUNET_NETWORK_test_pf (PF_INET); 
-  namestore_handle = GNUNET_NAMESTORE_connect (c);
-  if (NULL == namestore_handle)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                _("Failed to connect to the namestore!\n"));
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
- namecache_handle = GNUNET_NAMECACHE_connect (c);
+  v4_enabled = GNUNET_NETWORK_test_pf (PF_INET);
+  namecache_handle = GNUNET_NAMECACHE_connect (c);
   if (NULL == namecache_handle)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -635,11 +484,9 @@ run (void *cls,
                      dht_handle,
                      c,
                      max_parallel_bg_queries);
-  GNS_shorten_init (namestore_handle,
-                    namecache_handle,
-                    dht_handle);
   statistics = GNUNET_STATISTICS_create ("gns", c);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
 }
 
 
@@ -657,10 +504,6 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_GNS_LOOKUP,
                         struct LookupMessage,
                         NULL),
- GNUNET_MQ_hd_fixed_size (rev_lookup,
-                          GNUNET_MESSAGE_TYPE_GNS_REVERSE_LOOKUP,
-                          struct ReverseLookupMessage,
-                          NULL),
  GNUNET_MQ_handler_end());
 
 
diff --git a/src/gns/gnunet-service-gns_interceptor.c b/src/gns/gnunet-service-gns_interceptor.c
index 7a3cfc0dd..a9e207891 100644
--- a/src/gns/gnunet-service-gns_interceptor.c
+++ b/src/gns/gnunet-service-gns_interceptor.c
@@ -333,7 +333,6 @@ handle_dns_request (void *cls,
     ilh->lookup = GNS_resolver_lookup (&zone,
                                        p->queries[0].type,
                                        p->queries[0].name,
-                                       NULL /* FIXME: enable shorten for DNS intercepts? */,
                                        GNUNET_NO,
                                        &reply_to_dns, ilh);
     return;
diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c
index 5e957871e..c58190599 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -30,7 +30,6 @@
 #include "gnunet_dht_service.h"
 #include "gnunet_gnsrecord_lib.h"
 #include "gnunet_namecache_service.h"
-#include "gnunet_namestore_service.h"
 #include "gnunet_dns_service.h"
 #include "gnunet_resolver_service.h"
 #include "gnunet_revocation_service.h"
@@ -39,7 +38,6 @@
 #include "gnunet_gns_service.h"
 #include "gns.h"
 #include "gnunet-service-gns_resolver.h"
-#include "gnunet-service-gns_shorten.h"
 #include "gnunet_vpn_service.h"
 
 
@@ -327,11 +325,6 @@ struct GNS_ResolverHandle
   struct AuthorityChain *ac_tail;
 
   /**
-   * Private key of the shorten zone, NULL to not shorten.
-   */
-  struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_key;
-
-  /**
    * ID of a task associated with the resolution process.
    */
   struct GNUNET_SCHEDULER_Task * task_id;
@@ -1750,23 +1743,6 @@ handle_gns_resolution_result (void *cls,
       } /* end: switch */
     } /* end: for rd_count */
 
-    /* trigger shortening */
-    if ((NULL != rh->shorten_key) &&
-        (NULL != shorten_ac) &&
-        (GNUNET_NO == shorten_ac->shortening_started) &&
-        (NULL != shorten_ac->suggested_shortening_label))
-    {
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    "Start shortening for label `%s' based on nick `%s'\n",
-                    shorten_ac->label,
-                    shorten_ac->suggested_shortening_label);
-        shorten_ac->shortening_started = GNUNET_YES;
-        GNS_shorten_start (shorten_ac->label,
-                         shorten_ac->suggested_shortening_label,
-                         &shorten_ac->authority_info.gns_authority,
-                         rh->shorten_key);
-    }
-
     /* yes, we are done, return result */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Returning GNS response for `%s' with %u answers\n",
@@ -2374,7 +2350,6 @@ start_resolver_lookup (struct GNS_ResolverHandle *rh)
  * @param zone the zone to perform the lookup in
  * @param record_type the record type to look up
  * @param name the name to look up
- * @param shorten_key a private key for use with PSEU import (can be NULL)
  * @param options local options to control local lookup
  * @param proc the processor to call on result
  * @param proc_cls the closure to pass to @a proc
@@ -2384,16 +2359,13 @@ struct GNS_ResolverHandle *
 GNS_resolver_lookup (const struct GNUNET_CRYPTO_EcdsaPublicKey *zone,
                      uint32_t record_type,
                      const char *name,
-                     const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_key,
                      enum GNUNET_GNS_LocalOptions options,
                      GNS_ResultProcessor proc, void *proc_cls)
 {
   struct GNS_ResolverHandle *rh;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              (NULL == shorten_key)
-              ? "Starting lookup for `%s' with shortening disabled\n"
-              : "Starting lookup for `%s' with shortening enabled\n",
+              "Starting lookup for `%s'\n",
               name);
   rh = GNUNET_new (struct GNS_ResolverHandle);
   GNUNET_CONTAINER_DLL_insert (rlh_head,
@@ -2406,11 +2378,6 @@ GNS_resolver_lookup (const struct GNUNET_CRYPTO_EcdsaPublicKey *zone,
   rh->record_type = record_type;
   rh->name = GNUNET_strdup (name);
   rh->name_resolution_pos = strlen (name);
-  if (NULL != shorten_key)
-  {
-    rh->shorten_key = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey);
-    *rh->shorten_key = *shorten_key;
-  }
   start_resolver_lookup (rh);
   return rh;
 }
@@ -2507,7 +2474,6 @@ GNS_resolver_lookup_cancel (struct GNS_ResolverHandle *rh)
                                  dr);
     GNUNET_free (dr);
   }
-  GNUNET_free_non_null (rh->shorten_key);
   GNUNET_free (rh->name);
   GNUNET_free (rh);
 }
diff --git a/src/gns/gnunet-service-gns_resolver.h b/src/gns/gnunet-service-gns_resolver.h
index 863398093..c71d3983d 100644
--- a/src/gns/gnunet-service-gns_resolver.h
+++ b/src/gns/gnunet-service-gns_resolver.h
@@ -65,9 +65,10 @@ struct GNS_ResolverHandle;
  * @param rd_count number of records in @a rd
  * @param rd records returned for the lookup
  */
-typedef void (*GNS_ResultProcessor)(void *cls,
-                                    uint32_t rd_count,
-                                    const struct GNUNET_GNSRECORD_Data *rd);
+typedef void
+(*GNS_ResultProcessor)(void *cls,
+                       uint32_t rd_count,
+                       const struct GNUNET_GNSRECORD_Data *rd);
 
 
 /**
@@ -77,7 +78,6 @@ typedef void (*GNS_ResultProcessor)(void *cls,
  * @param zone the zone to perform the lookup in
  * @param record_type the record type to look up
  * @param name the name to look up
- * @param shorten_key optional private key for authority caching, can be NULL
  * @param options options set to control local lookup
  * @param proc the processor to call
  * @param proc_cls the closure to pass to @a proc
@@ -87,7 +87,6 @@ struct GNS_ResolverHandle *
 GNS_resolver_lookup (const struct GNUNET_CRYPTO_EcdsaPublicKey *zone,
                      uint32_t record_type,
                      const char *name,
-                     const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_key,
                      enum GNUNET_GNS_LocalOptions options,
                      GNS_ResultProcessor proc,
                      void *proc_cls);
@@ -102,8 +101,6 @@ void
 GNS_resolver_lookup_cancel (struct GNS_ResolverHandle *rh);
 
 
-
-
 /**
  * Generic function to check for TLDs.  Checks if "name" ends in ".tld"
  *
diff --git a/src/gns/gnunet-service-gns_reverser.c b/src/gns/gnunet-service-gns_reverser.c
deleted file mode 100644
index b5b8b31b7..000000000
--- a/src/gns/gnunet-service-gns_reverser.c
+++ /dev/null
@@ -1,601 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file gns/gnunet-service-gns_reverser.c
- * @brief GNUnet GNS service
- * @author Martin Schanzenbach
- */
-
-
-#include "platform.h"
-#include "gnunet_gns_service.h"
-#include "gnunet-service-gns_resolver.h"
-#include "gnunet-service-gns_reverser.h"
-
-struct ReverseRecordEntry
-{
-  /**
-   * DLL
-   */
-  struct ReverseRecordEntry *next;
-
-  /**
-   * DLL
-   */
-  struct ReverseRecordEntry *prev;
-
-  /**
-   * ReverseRecord
-   */
-  struct GNUNET_GNSRECORD_ReverseRecord *record;
-
-  /**
-   * Record length
-   */
-  size_t record_len;
-
-};
-
-struct IteratorHandle
-{
-  /**
-   * Records found
-   */
-  struct ReverseRecordEntry *records_head;
-
-  /**
-   * Records found
-   */
-  struct ReverseRecordEntry *records_tail;
-
-  /**
-   * Record count
-   */
-  uint64_t record_count;
-
-  /**
-   * Current delegation to expect
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey target;
-
-  /**
-   * Queue entry
-   */
-  struct GNUNET_NAMESTORE_QueueEntry *ns_qe;
-
-};
-
-struct ReverseTreeNode
-{
-  /**
-   * DLL
-   */
-  struct ReverseTreeNode *next;
-
-  /**
-   * DLL
-   */
-  struct ReverseTreeNode *prev;
-
-  /**
-   * Resolved name until now
-   */
-  char *name;
-
-  /**
-   * Depth of the resolution at this node
-   */
-  uint8_t depth;
-
-  /**
-   * The pkey of the namespace
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
-
-};
-
-
-struct GNS_ReverserHandle
-{
-  /**
-   * GNS resolver handle
-   */
-  struct GNS_ResolverHandle *rh;
-
-  /**
-   * The authority to look for
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey authority;
-
-  /**
-   * Resolution candidate queue
-   */
-  struct ReverseTreeNode *node_queue_head;
-
-  /**
-   * Resolution candidate queue
-   */
-  struct ReverseTreeNode *node_queue_tail;
-
-  /**
-   * Max depth for the resolution
-   */
-  uint8_t max_depth;
-
-  /**
-   * Result callback
-   */
-  GNS_ReverseResultProcessor proc;
-
-  /**
-   * Callback closure
-   */
-  void *proc_cls;
-};
-
-/**
- * Reverse record collection task
- */
-static struct GNUNET_SCHEDULER_Task *reverse_record_check_task;
-
-/**
- * NS iterator task
- */
-static struct GNUNET_SCHEDULER_Task *it_task;
-
-/**
- * GNS lookup handle
- */
-static struct GNS_ResolverHandle *gns_lookup_reverse;
-
-/**
- * NS handle
- */
-static struct GNUNET_NAMESTORE_Handle *ns;
-
-/**
- * NS Iterator
- */
-static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
-
-/**
- * The zone target for reverse record resolution
- */
-static struct GNUNET_CRYPTO_EcdsaPublicKey myzone;
-
-/**
- * The zone target for reverse record resolution
- */
-static struct GNUNET_CRYPTO_EcdsaPrivateKey pzone;
-
-/**
- * The nick of our zone
- */
-static char *mynick;
-
-
-static void
-cleanup_handle (struct GNS_ReverserHandle *rh)
-{
-  struct ReverseTreeNode *rtn;
-
-  for (rtn = rh->node_queue_head; NULL != rtn; rtn = rh->node_queue_head)
-  {
-    if (NULL != rtn->name)
-      GNUNET_free (rtn->name);
-        GNUNET_CONTAINER_DLL_remove (rh->node_queue_head,
-                                 rh->node_queue_tail,
-                                 rtn);
-        GNUNET_free (rtn);
-  }
-  GNUNET_free (rh);
-}
-
-static void
-handle_gns_result (void *cls,
-                   uint32_t rd_count,
-                   const struct GNUNET_GNSRECORD_Data *rd)
-{
-  struct GNS_ReverserHandle *rh = cls;
-  const struct GNUNET_GNSRECORD_ReverseRecord *rr;
-  struct ReverseTreeNode *rtn;
-  char *result;
-  const char *name;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got result (%d)\n", rd_count);
-
-  for (int i = 0; i < rd_count; i++)
-  {
-    /**
-     * Check if we are in the delegation set
-     */
-    if (GNUNET_GNSRECORD_TYPE_REVERSE != rd[i].record_type)
-      continue;
-    rr = rd[i].data;
-    name = (const char*) &rr[1];
-    if (0 == memcmp (&rh->authority,
-                     &rr->pkey,
-                     sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)))
-    {
-      //Found!
-      GNUNET_asprintf (&result,
-                       "%s.%s.gnu",
-                       rh->node_queue_head->name,
-                       name);
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Found path from %s\n", result);
-
-      rh->proc (rh->proc_cls, result);
-      cleanup_handle (rh);
-      GNUNET_free (result);
-      return;
-    } else {
-      if (rh->node_queue_head->depth >= rh->max_depth)
-        break;
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Found REVERSE from %s\n", name);
-
-      rtn = GNUNET_new (struct ReverseTreeNode);
-      if (NULL == rh->node_queue_head->name)
-        rtn->name = GNUNET_strdup (name);
-      else
-        GNUNET_asprintf (&rtn->name,
-                         "%s.%s",
-                         rh->node_queue_head->name,
-                         name);
-      rtn->depth = rh->node_queue_head->depth + 1;
-      rtn->pkey = rr->pkey;
-      GNUNET_CONTAINER_DLL_insert_tail (rh->node_queue_head,
-                                        rh->node_queue_tail,
-                                        rtn);
-    }
-  }
-
-  /**
-   * Done here remove node from queue
-   */
-  rtn = rh->node_queue_head;
-  if (NULL != rtn)
-    GNUNET_CONTAINER_DLL_remove (rh->node_queue_head,
-                                 rh->node_queue_tail,
-                                 rtn);
-  if (NULL == rh->node_queue_head)
-  {
-    //No luck
-    rh->proc (rh->proc_cls, NULL);
-    cleanup_handle (rh);
-    return;
-  }
-  rh->rh = GNS_resolver_lookup (&rh->node_queue_head->pkey,
-                                GNUNET_GNSRECORD_TYPE_REVERSE,
-                                "+.gnu",
-                                NULL,
-                                GNUNET_GNS_LO_DEFAULT,
-                                &handle_gns_result,
-                                rh);
-}
-
-/**
- * Reverse lookup of a specific zone
- * calls RecordLookupProcessor on result or timeout
- *
- * @param target the zone to perform the lookup in
- * @param authority the authority
- * @param proc the processor to call
- * @param proc_cls the closure to pass to @a proc
- * @return handle to cancel operation
- */
-struct GNS_ReverserHandle *
-GNS_reverse_lookup (const struct GNUNET_CRYPTO_EcdsaPublicKey *target,
-                    const struct GNUNET_CRYPTO_EcdsaPublicKey *authority,
-                    GNS_ReverseResultProcessor proc,
-                    void *proc_cls)
-{
-  struct GNS_ReverserHandle *rh;
-  struct ReverseTreeNode *rtn;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Starting reverse resolution\n");
-  rh = GNUNET_new (struct GNS_ReverserHandle);
-  rh->proc = proc;
-  rh->proc_cls = proc_cls;
-  rtn = GNUNET_new (struct ReverseTreeNode);
-  rtn->name = NULL;
-  rtn->pkey = *target;
-  rtn->depth = 0;
-  GNUNET_CONTAINER_DLL_insert (rh->node_queue_head,
-                               rh->node_queue_tail,
-                               rtn);
-  rh->authority = *authority;
-  rh->max_depth = 3; //TODO make argument
-  rh->rh = GNS_resolver_lookup (target,
-                                GNUNET_GNSRECORD_TYPE_REVERSE,
-                                "+.gnu",
-                                NULL,
-                                GNUNET_GNS_LO_DEFAULT,
-                                &handle_gns_result,
-                                rh);
-  return rh;
-}
-
-/**
- * Cancel active resolution (i.e. client disconnected).
- *
- * @param rh resolution to abort
- */
-void
-GNS_reverse_lookup_cancel (struct GNS_ReverserHandle *rh)
-{
-  cleanup_handle (rh);
-  return;
-}
-
-/********************************************
- * Reverse iterator
- * ******************************************/
-
-
-static void
-next_it (void *cls);
-
-static void
-handle_gns_result_iter (void *cls,
-                        uint32_t rd_count,
-                        const struct GNUNET_GNSRECORD_Data *rd)
-{
-  struct IteratorHandle *ith = cls;
-  struct ReverseRecordEntry *rr;
-  gns_lookup_reverse = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "GNS for REVERSE (%s)\n", mynick);
-
-
-  if ((rd_count != 1) ||
-      (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "GNS invalid REVERSE (%s)\n", mynick);
-    gns_lookup_reverse = NULL;
-    it_task = GNUNET_SCHEDULER_add_now (&next_it, ith);
-    return;
-  }
-
-
-  rr = GNUNET_new (struct ReverseRecordEntry);
-  rr->record_len = sizeof (struct GNUNET_GNSRECORD_ReverseRecord)
-    + strlen (mynick) + 1;
-  rr->record = GNUNET_malloc (rr->record_len);
-  rr->record->pkey = ith->target;
-  rr->record->expiration.abs_value_us = rd->expiration_time;
-  GNUNET_memcpy ((char*)&rr->record[1],
-                 mynick,
-                 strlen (mynick));
-  GNUNET_CONTAINER_DLL_insert (ith->records_head,
-                               ith->records_tail,
-                               rr);
-  ith->record_count++;
-  it_task = GNUNET_SCHEDULER_add_now (&next_it, ith);
-}
-
-
-static void
-next_it (void *cls)
-{
-  it_task = NULL;
-  GNUNET_assert (NULL != namestore_iter);
-  GNUNET_NAMESTORE_zone_iterator_next (namestore_iter);
-}
-
-
-static void
-iterator_cb (void *cls,
-             const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
-             const char *label,
-             unsigned int rd_count,
-             const struct GNUNET_GNSRECORD_Data *rd)
-{
-  struct IteratorHandle *ith = cls;
-  struct GNUNET_CRYPTO_EcdsaPublicKey zone;
-  char *name;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "iterating for REVERSE (%s / %s)\n",
-              label,
-              mynick);
-
-
-  if ((rd_count != 1) ||
-      (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "wrong format (%s)\n", mynick);
-
-
-    it_task = GNUNET_SCHEDULER_add_now (&next_it, ith);
-    return;
-  }
-  GNUNET_CRYPTO_ecdsa_key_get_public (key,
-                                      &zone);
-  if (0 != memcmp (&zone, &myzone,
-                   sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "wrong zone (%s)\n", mynick);
-
-
-    it_task = GNUNET_SCHEDULER_add_now (&next_it, ith);
-    return;
-  }
-  ith->target = *((struct GNUNET_CRYPTO_EcdsaPublicKey *) rd->data);
-  GNUNET_asprintf (&name,
-                  "%s.gnu",
-                  mynick);
-  gns_lookup_reverse = GNS_resolver_lookup (&ith->target,
-                                            GNUNET_GNSRECORD_TYPE_PKEY,
-                                            name,
-                                            NULL,
-                                            GNUNET_GNS_LO_DEFAULT,
-                                            &handle_gns_result_iter,
-                                            ith);
-  GNUNET_free (name);
-}
-
-static void check_reverse_records (void *cls);
-
-static void
-store_reverse (void *cls,
-               int32_t success,
-               const char *emsg)
-{
-  struct IteratorHandle *ith = cls;
-  struct ReverseRecordEntry *rr;
-
-  if (GNUNET_SYSERR == success)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "%s\n",
-                emsg);
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Stored records (%s)\n", mynick);
-
-  for (rr = ith->records_head; NULL != rr; rr = ith->records_head)
-  {
-    GNUNET_CONTAINER_DLL_remove (ith->records_head,
-                                 ith->records_tail,
-                                 rr);
-    GNUNET_free (rr->record);
-    GNUNET_free (rr);
-  }
-  reverse_record_check_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_DAYS,
-                                                            &check_reverse_records,
-                                                            NULL);
-  GNUNET_free (ith);
-}
-
-
-static void
-finished_cb (void *cls)
-{
-  struct IteratorHandle *ith = cls;
-  struct ReverseRecordEntry *rr;
-  struct GNUNET_GNSRECORD_Data rd[ith->record_count];
-
-  memset (rd, 0, sizeof (struct GNUNET_GNSRECORD_Data) * ith->record_count);
-
-  rr = ith->records_head;
-  for (int i = 0; i < ith->record_count; i++)
-  {
-    rd[i].data_size = rr->record_len;
-    rd[i].data = GNUNET_malloc (rr->record_len);
-    rd[i].record_type = GNUNET_GNSRECORD_TYPE_REVERSE;
-    rd[i].expiration_time = rr->record->expiration.abs_value_us;
-    GNUNET_memcpy ((char*) rd[i].data,
-                   rr->record,
-                   rr->record_len);
-    rr = rr->next;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Finished iterating for REVERSE\n");
-
-  ith->ns_qe = GNUNET_NAMESTORE_records_store (ns,
-                                               &pzone,
-                                               "+",
-                                               ith->record_count,
-                                               rd,
-                                               &store_reverse,
-                                               ith);
-  namestore_iter = NULL;
-
-}
-
-
-static void
-it_error (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-              "Error iterating for REVERSE\n");
-}
-
-
-static void
-check_reverse_records (void *cls)
-{
-  struct IteratorHandle *ith;
-  ith = GNUNET_new (struct IteratorHandle);
-  ith->record_count = 0;
-  reverse_record_check_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Start iterating for REVERSE (%s)\n", mynick);
-  namestore_iter = GNUNET_NAMESTORE_zone_iteration_start (ns,
-                                                          NULL,
-                                                          &it_error,
-                                                          ith,
-                                                          &iterator_cb,
-                                                          ith,
-                                                          &finished_cb,
-                                                          ith);
-}
-
-
-/**
- * Initialize reverser
- *
- * @param nh handle to a namestore
- * @param key the private key of the gns-reverse zone
- * @param name the name of the gns-reverse zone
- * @return GNUNET_OK
- */
-int
-GNS_reverse_init (struct GNUNET_NAMESTORE_Handle *nh,
-                  const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
-                  const char *nick)
-{
-  GNUNET_asprintf (&mynick,
-                   "%s",
-                   nick);
-  GNUNET_CRYPTO_ecdsa_key_get_public (zone,
-                                      &myzone);
-  GNUNET_memcpy (&pzone,
-                 zone,
-                 sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey));
-  ns = nh;
-  reverse_record_check_task = GNUNET_SCHEDULER_add_now (&check_reverse_records,
-                                                        NULL);
-  return GNUNET_OK;
-}
-
-/**
- * Cleanup reverser
- */
-void
-GNS_reverse_done ()
-{
-  if (NULL != mynick)
-    GNUNET_free (mynick);
-  if (NULL != it_task)
-    GNUNET_SCHEDULER_cancel (it_task);
-  if (NULL != reverse_record_check_task)
-    GNUNET_SCHEDULER_cancel (reverse_record_check_task);
-  if (NULL != gns_lookup_reverse)
-    GNS_resolver_lookup_cancel (gns_lookup_reverse);
-  if (NULL != namestore_iter)
-    GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
-}
-
diff --git a/src/gns/gnunet-service-gns_reverser.h b/src/gns/gnunet-service-gns_reverser.h
deleted file mode 100644
index fc9680a29..000000000
--- a/src/gns/gnunet-service-gns_reverser.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file gns/gnunet-service-gns_reverser.h
- * @brief GNUnet GNS service
- * @author Martin Schanzenbach
- */
-#ifndef GNS_REVERSER_H
-#define GNS_REVERSER_H
-#include "gns.h"
-#include "gnunet_gns_service.h"
-
-/**
- * Handle for an active request.
- */
-struct GNS_ReverserHandle;
-
-
-/**
- * Function called with results for a GNS resolution.
- *
- * @param cls closure
- * @param rd_count number of records in @a rd
- * @param rd records returned for the lookup
- */
-typedef void (*GNS_ReverseResultProcessor)(void *cls,
-                                           const char *name);
-
-
-/**
- * Reverse lookup of a specific zone
- * calls RecordLookupProcessor on result or timeout
- *
- * @param target the zone to perform the lookup in
- * @param authority the authority
- * @param proc the processor to call
- * @param proc_cls the closure to pass to @a proc
- * @return handle to cancel operation
- */
-struct GNS_ReverserHandle *
-GNS_reverse_lookup (const struct GNUNET_CRYPTO_EcdsaPublicKey *target,
-                    const struct GNUNET_CRYPTO_EcdsaPublicKey *authority,
-                    GNS_ReverseResultProcessor proc,
-                    void *proc_cls);
-
-
-/**
- * Cancel active resolution (i.e. client disconnected).
- *
- * @param rh resolution to abort
- */
-void
-GNS_reverse_lookup_cancel (struct GNS_ReverserHandle *rh);
-
-/**
- * Initialize reverser
- *
- * @param nh handle to a namestore
- * @param key the private key of the gns-reverse zone
- * @param name the name of the gns-reverse zone
- * @return GNUNET_OK
- */
-int
-GNS_reverse_init (struct GNUNET_NAMESTORE_Handle *nh,
-                  const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
-                  const char *name);
-
-/**
- * Cleanup reverser
- */
-void
-GNS_reverse_done ();
-
-#endif
diff --git a/src/gns/gnunet-service-gns_shorten.c b/src/gns/gnunet-service-gns_shorten.c
deleted file mode 100644
index 9aa0419aa..000000000
--- a/src/gns/gnunet-service-gns_shorten.c
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file gns/gnunet-service-gns_shorten.c
- * @brief GNUnet GNS shortening logic
- * @author Martin Schanzenbach
- * @author Christian Grothoff
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_gnsrecord_lib.h"
-#include "gnunet_namestore_service.h"
-#include "gnunet_resolver_service.h"
-#include "gnunet_gns_service.h"
-#include "gns.h"
-#include "gnunet-service-gns_shorten.h"
-#include "gnunet_vpn_service.h"
-
-
-/**
- * Default DHT timeout for lookups.
- */
-#define DHT_LOOKUP_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
-
-/**
- * DHT replication level
- */
-#define DHT_GNS_REPLICATION_LEVEL 5
-
-
-/**
- * Handle for a PSEU lookup used to shorten names.
- */
-struct GetPseuAuthorityHandle
-{
-  /**
-   * DLL
-   */
-  struct GetPseuAuthorityHandle *next;
-
-  /**
-   * DLL
-   */
-  struct GetPseuAuthorityHandle *prev;
-
-  /**
-   * Private key of the (shorten) zone to store the resulting
-   * pseudonym in.
-   */
-  struct GNUNET_CRYPTO_EcdsaPrivateKey shorten_zone_key;
-
-  /**
-   * Original label (used if no PSEU record is found).
-   */
-  char label[GNUNET_DNSPARSER_MAX_LABEL_LENGTH + 1];
-
-  /**
-   * Suggested label based on NICK record
-   */
-  char * suggested_label;
-
-  /**
-   * Label we are currently trying out
-   */
-  char *current_label;
-
-  /**
-   * The zone for which we are trying to find the PSEU record.
-   */
-  struct GNUNET_CRYPTO_EcdsaPublicKey target_zone;
-
-  /**
-   * Handle for DHT lookups. Should be NULL if no lookups are in progress
-   */
-  struct GNUNET_DHT_GetHandle *get_handle;
-
-  /**
-   * Handle to namestore request
-   */
-  struct GNUNET_NAMESTORE_QueueEntry *namestore_task;
-
-  /**
-   * Handle to namecache request
-   */
-  struct GNUNET_NAMECACHE_QueueEntry *namecache_task;
-
-  /**
-   * Task to abort DHT lookup operation.
-   */
-  struct GNUNET_SCHEDULER_Task * timeout_task;
-
-};
-
-
-/**
- * Head of PSEU/shorten operations list.
- */
-static struct GetPseuAuthorityHandle *gph_head;
-
-/**
- * Tail of PSEU/shorten operations list.
- */
-static struct GetPseuAuthorityHandle *gph_tail;
-
-/**
- * Our handle to the namestore service
- */
-static struct GNUNET_NAMESTORE_Handle *namestore_handle;
-
-/**
- * Our handle to the namecache service
- */
-static struct GNUNET_NAMECACHE_Handle *namecache_handle;
-
-/**
- * Resolver handle to the dht
- */
-static struct GNUNET_DHT_Handle *dht_handle;
-
-/**
- * Cleanup a 'struct GetPseuAuthorityHandle', terminating all
- * pending activities.
- *
- * @param gph handle to terminate
- */
-static void
-free_get_pseu_authority_handle (struct GetPseuAuthorityHandle *gph)
-{
-  if (NULL != gph->get_handle)
-  {
-    GNUNET_DHT_get_stop (gph->get_handle);
-    gph->get_handle = NULL;
-  }
-  if (NULL != gph->namestore_task)
-  {
-    GNUNET_NAMESTORE_cancel (gph->namestore_task);
-    gph->namestore_task = NULL;
-  }
-  if (NULL != gph->namecache_task)
-  {
-    GNUNET_NAMECACHE_cancel (gph->namecache_task);
-    gph->namecache_task = NULL;
-  }
-  if (NULL != gph->timeout_task)
-  {
-    GNUNET_SCHEDULER_cancel (gph->timeout_task);
-    gph->timeout_task = NULL;
-  }
-  GNUNET_CONTAINER_DLL_remove (gph_head, gph_tail, gph);
-  GNUNET_free_non_null (gph->current_label);
-  GNUNET_free (gph);
-}
-
-
-/**
- * Continuation for pkey record creation (shorten)
- *
- * @param cls a GetPseuAuthorityHandle
- * @param success unused
- * @param emsg unused
- */
-static void
-create_pkey_cont (void* cls,
-                  int32_t success,
-                  const char *emsg)
-{
-  struct GetPseuAuthorityHandle* gph = cls;
-
-  gph->namestore_task = NULL;
-  free_get_pseu_authority_handle (gph);
-}
-
-
-/**
- * Namestore calls this function if we have record for this name.
- * (or with rd_count=0 to indicate no matches).
- *
- * @param cls the pending query
- * @param rd_count the number of records with 'name'
- * @param rd the record data
- */
-static void
-process_pseu_lookup_ns (void *cls,
-                        unsigned int rd_count,
-                        const struct GNUNET_GNSRECORD_Data *rd);
-
-
-/**
- * We obtained a result for our query to the shorten zone from
- * the namestore.  Try to decrypt.
- *
- * @param cls the handle to our shorten operation
- * @param block resulting encrypted block
- */
-static void
-process_pseu_block_ns (void *cls,
-                       const struct GNUNET_GNSRECORD_Block *block)
-{
-  struct GetPseuAuthorityHandle *gph = cls;
-  struct GNUNET_CRYPTO_EcdsaPublicKey pub;
-
-  gph->namecache_task = NULL;
-  if (NULL == block)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Namecache did not return information for label `%s' \n",
-                gph->current_label);
-    process_pseu_lookup_ns (gph, 0, NULL);
-    return;
-  }
-  GNUNET_CRYPTO_ecdsa_key_get_public (&gph->shorten_zone_key,
-                                    &pub);
-  if (GNUNET_OK !=
-      GNUNET_GNSRECORD_block_decrypt (block,
-                                      &pub,
-                                      gph->current_label,
-                                      &process_pseu_lookup_ns,
-                                      gph))
-  {
-    GNUNET_break (0);
-    free_get_pseu_authority_handle (gph);
-    return;
-  }
-}
-
-
-/**
- * Lookup in the namecache for the shorten zone the given label.
- *
- * @param gph the handle to our shorten operation
- * @param label the label to lookup
- */
-static void
-perform_nick_lookup (struct GetPseuAuthorityHandle *gph,
-                     const char *label)
-{
-  struct GNUNET_CRYPTO_EcdsaPublicKey pub;
-  struct GNUNET_HashCode query;
-
-  GNUNET_CRYPTO_ecdsa_key_get_public (&gph->shorten_zone_key,
-                                    &pub);
-  GNUNET_free_non_null (gph->current_label);
-  gph->current_label = GNUNET_strdup (label);
-  GNUNET_GNSRECORD_query_from_public_key (&pub,
-                                          label,
-                                          &query);
-  gph->namecache_task = GNUNET_NAMECACHE_lookup_block (namecache_handle,
-                                                       &query,
-                                                       &process_pseu_block_ns,
-                                                       gph);
-}
-
-
-/**
- * Namestore calls this function if we have record for this name.
- * (or with rd_count=0 to indicate no matches).
- *
- * @param cls the pending query
- * @param rd_count the number of records with 'name'
- * @param rd the record data
- */
-static void
-process_pseu_lookup_ns (void *cls,
-                        unsigned int rd_count,
-                        const struct GNUNET_GNSRECORD_Data *rd)
-{
-  struct GetPseuAuthorityHandle *gph = cls;
-  struct GNUNET_GNSRECORD_Data new_pkey;
-
-  gph->namestore_task = NULL;
-  if (rd_count > 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Name `%s' already taken, cannot shorten.\n",
-                gph->current_label);
-    /* if this was not yet the original label, try one more
-       time, this time not using PSEU but the original label */
-    if (0 == strcmp (gph->current_label,
-                     gph->label))
-    {
-      free_get_pseu_authority_handle (gph);
-    }
-    else
-    {
-      perform_nick_lookup (gph, gph->label);
-    }
-    return;
-  }
-  /* name is available */
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Shortening `%s' to `%s'\n",
-              GNUNET_GNSRECORD_z2s (&gph->target_zone),
-              gph->current_label);
-  new_pkey.expiration_time = UINT64_MAX;
-  new_pkey.data_size = sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
-  new_pkey.data = &gph->target_zone;
-  new_pkey.record_type = GNUNET_GNSRECORD_TYPE_PKEY;
-  new_pkey.flags = GNUNET_GNSRECORD_RF_NONE
-                 | GNUNET_GNSRECORD_RF_PRIVATE;
-  gph->namestore_task
-    = GNUNET_NAMESTORE_records_store (namestore_handle,
-                                      &gph->shorten_zone_key,
-                                      gph->current_label,
-                                      1, &new_pkey,
-                                      &create_pkey_cont, gph);
-}
-
-
-/**
- * Encountered an error in zone-to-name lookup, give up on shortening.
- */
-static void
-zone_to_name_error_cb (void *cls)
-{
-  struct GetPseuAuthorityHandle* gph = cls;
-
-  gph->namestore_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "Shortening aborted, internal error talking to namestore\n");
-  free_get_pseu_authority_handle (gph);
-}
-
-
-/**
- * Callback called by namestore for a zone to name result.  We're
- * trying to see if a short name for a given zone already exists.
- *
- * @param cls the closure
- * @param zone_key the zone we queried
- * @param name the name found or NULL
- * @param rd_len number of records for the name
- * @param rd the record data (PKEY) for the name
- */
-static void
-process_zone_to_name_discover (void *cls,
-                               const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
-                               const char *name,
-                               unsigned int rd_len,
-                               const struct GNUNET_GNSRECORD_Data *rd)
-{
-  struct GetPseuAuthorityHandle* gph = cls;
-
-  gph->namestore_task = NULL;
-  if (0 != rd_len)
-  {
-    /* we found a match in our own zone */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Shortening aborted, name `%s' already reserved for the zone\n",
-                name);
-    free_get_pseu_authority_handle (gph);
-    return;
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Shortening continuing, no name not reserved in shorten zone\n");
-  }
-  /* record does not yet exist, check if suggested label is available */
-  perform_nick_lookup (gph, gph->suggested_label);
-}
-
-
-/**
- * Start shortening algorithm, try to allocate a nice short
- * canonical name for @a pub in @a shorten_zone, using
- * @a original_label as one possible suggestion.
- *
- * @param original_label original label for the zone
- * @param suggested_label suggested label for the zone
- * @param pub public key of the zone to shorten
- * @param shorten_zone private key of the target zone for the new record
- */
-void
-GNS_shorten_start (const char *original_label,
-                   const char *suggested_label,
-                   const struct GNUNET_CRYPTO_EcdsaPublicKey *pub,
-                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_zone)
-{
-  struct GetPseuAuthorityHandle *gph;
-  struct GNUNET_CRYPTO_EcdsaPublicKey shorten_pub;
-
-  if (strlen (original_label) > GNUNET_DNSPARSER_MAX_LABEL_LENGTH)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  GNUNET_CRYPTO_ecdsa_key_get_public (shorten_zone, &shorten_pub);
-  if (0 == memcmp (&shorten_pub, pub, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)))
-  {
-    /* Do not shorten the shorten zone */
-    return;
-  }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Starting shortening process for `%s' with old label `%s' and suggested nickname `%s'\n",
-              GNUNET_GNSRECORD_z2s (pub),
-              original_label, suggested_label);
-  gph = GNUNET_new (struct GetPseuAuthorityHandle);
-  gph->shorten_zone_key = *shorten_zone;
-  gph->target_zone = *pub;
-  gph->suggested_label = GNUNET_strdup (suggested_label);
-  strcpy (gph->label, original_label);
-  GNUNET_CONTAINER_DLL_insert (gph_head, gph_tail, gph);
-  /* first, check if we *already* have a record for this zone */
-  gph->namestore_task = GNUNET_NAMESTORE_zone_to_name (namestore_handle,
-                                                       shorten_zone,
-                                                       pub,
-                                                       &zone_to_name_error_cb,
-                                                       gph,
-                                                       &process_zone_to_name_discover,
-                                                       gph);
-}
-
-
-/**
- * Initialize the shortening subsystem
- *
- * @param nh the namestore handle
- * @param nc the namecache handle
- * @param dht the dht handle
- */
-void
-GNS_shorten_init (struct GNUNET_NAMESTORE_Handle *nh,
-                  struct GNUNET_NAMECACHE_Handle *nc,
-                  struct GNUNET_DHT_Handle *dht)
-{
-  namestore_handle = nh;
-  namecache_handle = nc;
-  dht_handle = dht;
-}
-
-
-/**
- * Shutdown shortening.
- */
-void
-GNS_shorten_done ()
-{
-  /* abort active shorten operations */
-  while (NULL != gph_head)
-    free_get_pseu_authority_handle (gph_head);
-  dht_handle = NULL;
-  namestore_handle = NULL;
-  namecache_handle = NULL;
-}
-
-/* end of gnunet-service-gns_shorten.c */
diff --git a/src/gns/gnunet-service-gns_shorten.h b/src/gns/gnunet-service-gns_shorten.h
deleted file mode 100644
index d82bb52f7..000000000
--- a/src/gns/gnunet-service-gns_shorten.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file gns/gnunet-service-gns_shorten.h
- * @brief GNUnet GNS shortening API
- * @author Martin Schanzenbach
- */
-#ifndef GNS_SHORTEN_H
-#define GNS_SHORTEN_H
-#include "gns.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_namecache_service.h"
-#include "gnunet_namestore_service.h"
-
-
-/**
- * Initialize the shorten subsystem.
- * MUST be called before #GNS_shorten_start.
- *
- * @param nh handle to the namestore
- * @param nc the namecache handle
- * @param dht handle to the dht
- */
-void
-GNS_shorten_init (struct GNUNET_NAMESTORE_Handle *nh,
-                  struct GNUNET_NAMECACHE_Handle *nc,
-                  struct GNUNET_DHT_Handle *dht);
-
-
-/**
- * Cleanup shorten: Terminate pending lookups
- */
-void
-GNS_shorten_done (void);
-
-
-/**
- * Start shortening algorithm, try to allocate a nice short
- * canonical name for @a pub in @a shorten_zone, using
- * @a original_label as one possible suggestion.
- *
- * @param original_label original label for the zone
- * @param pub public key of the zone to shorten
- * @param shorten_zone private key of the target zone for the new record
- */
-void
-GNS_shorten_start (const char *original_label,
-                   const char *suggested_label,
-                   const struct GNUNET_CRYPTO_EcdsaPublicKey *pub,
-                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_zone);
-
-
-#endif
diff --git a/src/gns/plugin_block_gns.c b/src/gns/plugin_block_gns.c
index f0e34a04b..a42fef953 100644
--- a/src/gns/plugin_block_gns.c
+++ b/src/gns/plugin_block_gns.c
@@ -22,9 +22,11 @@
  * @file gns/plugin_block_gns.c
  * @brief blocks used for GNS records
  * @author Martin Schanzenbach
+ * @author Christian Grothoff
  */
 
 #include "platform.h"
+#include "gnunet_block_group_lib.h"
 #include "gnunet_block_plugin.h"
 #include "gnunet_namestore_service.h"
 #include "gnunet_signatures.h"
@@ -36,6 +38,59 @@
 #define BLOOMFILTER_K 16
 
 /**
+ * How big is the BF we use for GNS blocks?
+ */
+#define GNS_BF_SIZE 8
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_gns_create_group (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               uint32_t nonce,
+                               const void *raw_data,
+                               size_t raw_data_size,
+                               va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = GNS_BF_SIZE;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
+
+/**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the reply_block.
  * Note that it is assumed that the reply has already been
@@ -43,11 +98,11 @@
  * be done with the "get_key" function.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param bg block group to use for evaluation
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with @a query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extrended query data (can be NULL, depending on @a type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -56,11 +111,11 @@
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_gns_evaluate (void *cls,
+                           struct GNUNET_BLOCK_Context *ctx,
                            enum GNUNET_BLOCK_Type type,
+                           struct GNUNET_BLOCK_Group *bg,
                            enum GNUNET_BLOCK_EvaluationOptions eo,
                            const struct GNUNET_HashCode *query,
-                           struct GNUNET_CONTAINER_BloomFilter **bf,
-                           int32_t bf_mutator,
                            const void *xquery,
                            size_t xquery_size,
                            const void *reply_block,
@@ -69,7 +124,6 @@ block_plugin_gns_evaluate (void *cls,
   const struct GNUNET_GNSRECORD_Block *block;
   struct GNUNET_HashCode h;
   struct GNUNET_HashCode chash;
-  struct GNUNET_HashCode mhash;
 
   if (type != GNUNET_BLOCK_TYPE_GNS_NAMERECORD)
     return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -110,21 +164,13 @@ block_plugin_gns_evaluate (void *cls,
       GNUNET_break_op (0);
       return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
     }
-  if (NULL != bf)
-    {
-      GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash);
-      GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash);
-      if (NULL != *bf)
-        {
-          if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test(*bf, &mhash))
-            return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-        }
-      else
-        {
-          *bf = GNUNET_CONTAINER_bloomfilter_init(NULL, 8, BLOOMFILTER_K);
-        }
-      GNUNET_CONTAINER_bloomfilter_add(*bf, &mhash);
-    }
+  GNUNET_CRYPTO_hash (reply_block,
+                      reply_block_size,
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_OK_MORE;
 }
 
@@ -141,9 +187,11 @@ block_plugin_gns_evaluate (void *cls,
  *         (or if extracting a key from a block of this type does not work)
  */
 static int
-block_plugin_gns_get_key (void *cls, enum GNUNET_BLOCK_Type type,
-                         const void *reply_block, size_t reply_block_size,
-                         struct GNUNET_HashCode *key)
+block_plugin_gns_get_key (void *cls,
+                          enum GNUNET_BLOCK_Type type,
+                          const void *reply_block,
+                          size_t reply_block_size,
+                          struct GNUNET_HashCode *key)
 {
   const struct GNUNET_GNSRECORD_Block *block;
 
@@ -178,6 +226,7 @@ libgnunet_plugin_block_gns_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_gns_evaluate;
   api->get_key = &block_plugin_gns_get_key;
+  api->create_group = &block_plugin_gns_create_group;
   api->types = types;
   return api;
 }
@@ -189,7 +238,7 @@ libgnunet_plugin_block_gns_init (void *cls)
 void *
 libgnunet_plugin_block_gns_done (void *cls)
 {
-  struct GNUNET_TRANSPORT_PluginFunctions *api = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
 
   GNUNET_free (api);
   return NULL;
diff --git a/src/gns/plugin_gnsrecord_gns.c b/src/gns/plugin_gnsrecord_gns.c
index 5faca4578..5d611e19e 100644
--- a/src/gns/plugin_gnsrecord_gns.c
+++ b/src/gns/plugin_gnsrecord_gns.c
@@ -279,9 +279,10 @@ gns_string_to_value (void *cls,
         }
         *data_size = sizeof (struct GNUNET_TUN_GnsVpnRecord) + strlen (s_serv) + 1;
         *data = vpn = GNUNET_malloc (*data_size);
-        if (GNUNET_OK != GNUNET_CRYPTO_eddsa_public_key_from_string ((char*) s_peer,
-                                                                     strlen (s_peer),
-                                                                     &vpn->peer.public_key))
+        if (GNUNET_OK !=
+            GNUNET_CRYPTO_eddsa_public_key_from_string ((char*) s_peer,
+                                                        strlen (s_peer),
+                                                        &vpn->peer.public_key))
         {
           GNUNET_free (vpn);
           *data_size = 0;
@@ -362,9 +363,14 @@ gns_string_to_value (void *cls,
         }
         *data_size = sizeof (struct GNUNET_GNSRECORD_ReverseRecord) + strlen (known_by) + 1;
         *data = rev = GNUNET_malloc (*data_size);
-        GNUNET_CRYPTO_ecdsa_public_key_from_string (pkey_str,
-                                                    strlen (pkey_str),
-                                                    &rev->pkey);
+        if (GNUNET_OK !=
+            GNUNET_CRYPTO_ecdsa_public_key_from_string (pkey_str,
+                                                        strlen (pkey_str),
+                                                        &rev->pkey))
+        {
+          GNUNET_free (rev);
+          return GNUNET_SYSERR;
+        }
         rev->expiration = expiration;
         GNUNET_memcpy (&rev[1],
                        known_by,
diff --git a/src/gns/plugin_rest_gns.c b/src/gns/plugin_rest_gns.c
index 3cddbc246..b7775e4ea 100644
--- a/src/gns/plugin_rest_gns.c
+++ b/src/gns/plugin_rest_gns.c
@@ -336,10 +336,9 @@ process_lookup_result (void *cls, uint32_t rd_count,
  * identified by the given public key and the shorten zone.
  *
  * @param pkey public key to use for the zone, can be NULL
- * @param shorten_key private key used for shortening, can be NULL
  */
 static void
-lookup_with_keys (struct LookupHandle *handle, const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_key)
+lookup_with_public_key (struct LookupHandle *handle)
 {
   if (UINT32_MAX == handle->type)
   {
@@ -354,7 +353,6 @@ lookup_with_keys (struct LookupHandle *handle, const struct GNUNET_CRYPTO_EcdsaP
                                                 &handle->pkey,
                                                 handle->type,
                                                 handle->options,
-                                                shorten_key,
                                                 &process_lookup_result,
                                                 handle);
   }
@@ -365,55 +363,6 @@ lookup_with_keys (struct LookupHandle *handle, const struct GNUNET_CRYPTO_EcdsaP
   }
 }
 
-/**
- * Method called to with the ego we are to use for shortening
- * during the lookup.
- *
- * @param cls closure contains the public key to use
- * @param ego ego handle, NULL if not found
- * @param ctx context for application to store data for this ego
- *                 (during the lifetime of this process, initially NULL)
- * @param name name assigned by the user for this ego,
- *                   NULL if the user just deleted the ego and it
- *                   must thus no longer be used
- */
-static void
-identity_shorten_cb (void *cls,
-                     struct GNUNET_IDENTITY_Ego *ego,
-                     void **ctx,
-                     const char *name)
-{
-  struct LookupHandle *handle = cls;
-
-  handle->id_op = NULL;
-  if (NULL == ego)
-    lookup_with_keys (handle, NULL);
-  else
-    lookup_with_keys (handle,
-                      GNUNET_IDENTITY_ego_get_private_key (ego));
-}
-
-/**
- * Perform the actual resolution, starting with the zone
- * identified by the given public key.
- *
- * @param pkey public key to use for the zone
- */
-static void
-lookup_with_public_key (struct LookupHandle *handle)
-{
-  handle->pkeym = handle->pkey;
-  GNUNET_break (NULL == handle->id_op);
-  handle->id_op = GNUNET_IDENTITY_get (handle->identity,
-                                       "gns-short",
-                                       &identity_shorten_cb,
-                                       handle);
-  if (NULL == handle->id_op)
-  {
-    GNUNET_break (0);
-    lookup_with_keys (handle, NULL);
-  }
-}
 
 /**
  * Method called to with the ego we are to use for the lookup,
@@ -444,6 +393,7 @@ identity_zone_cb (void *cls,
   json_decref(handle->json_root);
 }
 
+
 /**
  * Method called to with the ego we are to use for the lookup,
  * when the ego is the one for the default master zone.
@@ -579,6 +529,7 @@ get_gns_cont (struct GNUNET_REST_RequestHandle *conndata_handle,
   {
     handle->pkey_str = GNUNET_CONTAINER_multihashmap_get (conndata_handle->url_param_map,
                                                           &key);
+    GNUNET_assert (NULL != handle->pkey_str);
     if (GNUNET_OK !=
         GNUNET_CRYPTO_ecdsa_public_key_from_string (handle->pkey_str,
                                                     strlen(handle->pkey_str),
diff --git a/src/gns/test_gns_nick_shorten.sh b/src/gns/test_gns_nick_shorten.sh
deleted file mode 100755
index df69bbba9..000000000
--- a/src/gns/test_gns_nick_shorten.sh
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/bin/bash
-trap "gnunet-arm -e -c test_gns_nick_shorten.conf" SIGINT
-which timeout &> /dev/null && DO_TIMEOUT="timeout 5"
-
-# This test tests shortening functionality based on NICK records:
-#
-# zone "delegatedego": Alice's zone
-# zone "testego": Local zone with delegation to alice
-
-LOCATION=$(which gnunet-config)
-if [ -z $LOCATION ]
-then
-  LOCATION="gnunet-config"
-fi
-$LOCATION --version 1> /dev/null
-if test $? != 0
-then
-        echo "GNUnet command line tools cannot be found, check environmental variables PATH and GNUNET_PREFIX"
-        exit 77
-fi
-
-# Deleting home directory from previous runs
-TEST_CONFIG="test_gns_nick_shorten.conf "
-rm -rf /tmp/test-gnunet-gns-peer-1/
-TEST_IP="127.0.0.1"
-TEST_IP="127.0.0.2"
-TEST_NICK_EGO="ego"
-TEST_NICK_DELEGATED="alice"
-TEST_NAME="www.mybestfriendalice.gnu"
-TEST_NAME_SHORT="www.alice.short.gnu"
-
-# export GNUNET_FORCE_LOG="namestore;;;;DEBUG/gns;;;;DEBUG/;;;;WARNING"
-
-# Start gnunet
-echo "Starting arm with configuration $TEST_CONFIG"
-gnunet-arm -s -c $TEST_CONFIG
-
-# Create initial identities: short-zone, delegated-zone, testego
-echo "Creating identities"
-gnunet-identity -d -c $TEST_CONFIG
-gnunet-identity -C short-zone -c $TEST_CONFIG
-gnunet-identity -C delegatedego -c $TEST_CONFIG
-gnunet-identity -e short-zone -s gns-short -c $TEST_CONFIG
-gnunet-identity -C testego -c $TEST_CONFIG
-
-echo "Adding nick names for identities"
-gnunet-namestore -z testego -i $TEST_NICK_EGO -c $TEST_CONFIG
-gnunet-namestore -z delegatedego -i $TEST_NICK_DELEGATED -c $TEST_CONFIG
-
-# Adding label www in Alice's delegatedego zone
-echo "Adding www record with IP $TEST_IP"
-gnunet-namestore -p -z delegatedego -a -n www -t A -V $TEST_IP -e never -c test_gns_nick_shorten.conf
-
-# Retrieve PKEYs for delegation
-DELEGATED_PKEY=$(gnunet-identity -d -c $TEST_CONFIG| grep delegatedego | awk '{print $3}')
-echo "Alice's PKEY is $DELEGATED_PKEY"
-
-SHORTEN_PKEY=$(gnunet-identity -c test_gns_nick_shorten.conf -d | grep short-zone | awk '{print $3}')
-echo "Shorten PKEY is $SHORTEN_PKEY"
-
-# Delegate the name "short" to shortenzone
-gnunet-namestore -p -z testego -a -n short -t PKEY -V $SHORTEN_PKEY -e never -c test_gns_nick_shorten.conf
-
-# Delegate the name "mybestfriendalice" to alice
-gnunet-namestore -p -z testego -a -n mybestfriendalice -t PKEY -V $DELEGATED_PKEY -e never -c test_gns_nick_shorten.conf
-
-# Perform lookup to shorten
-echo "Start gns..."
-gnunet-arm -c test_gns_nick_shorten.conf -i gns
-
-
-RES_IP=`$DO_TIMEOUT gnunet-gns --raw -z testego -u $TEST_NAME -t A -c test_gns_nick_shorten.conf`
-
-sleep 1
-
-echo "Lookup shortened names"
-PKEY_SHORT_RES=$($DO_TIMEOUT gnunet-gns --raw -c test_gns_nick_shorten.conf -z short-zone -u alice.gnu -t PKEY)
-echo "Resolving alice's PKEY in shorten zone: $PKEY_SHORT_RES"
-PKEY_RES=$($DO_TIMEOUT gnunet-gns --raw -c test_gns_nick_shorten.conf -z testego -u alice.short.gnu -t PKEY)
-echo "Resolving alice's PKEY in master zone: $PKEY_RES"
-
-RES=0
-if [ "$DELEGATED_PKEY" == "$PKEY_SHORT_RES" ]
-then
-  echo "PASS: Resolved delegation for shorten name in shortened zone"
-else
-  echo "FAIL: Expected PKEY in $DELEGATED_PKEY, received PKEY '$PKEY_SHORT_RES' in shorten zone."
-  RES=1
-fi
-
-if [ "$DELEGATED_PKEY" == "$PKEY_RES" ]
-then
-  echo "PASS: Resolved delegation for shorten name in master zone"
-else
-  echo "FAIL: Expected PKEY in $DELEGATED_PKEY, received PKEY $PKEY_SHORT_RES in master zone."
-  RES=1
-fi
-
-if [ $RES -eq 0 ]
-then
-        RES_IP=`$DO_TIMEOUT gnunet-gns --raw -z testego -u $TEST_NAME_SHORT -t A -c test_gns_nick_shorten.conf`
-        if [ "$RES_IP" == "$TEST_IP" ]
-        then
-                echo "PASS: Received $TEST_IP for $TEST_NAME_SHORT"
-        else
-                echo "FAIL: Expected IP in $TEST_IP, received IP '$RES_IP' for $TEST_SHORT_NAME."
-                RES=1
-        fi
-fi
-
-
-# Clean up
-echo "Clean up..."
-gnunet-namestore -z testego -d -n mybestfriendalice -t PKEY -V $DELEGATED_PKEY  -e never -c test_gns_nick_shorten.conf
-gnunet-namestore -z delegatedego -d -n www -t A -V $TEST_IP  -e never -c test_gns_nick_shorten.conf
-gnunet-identity -D -z testego -c $TEST_CONFIG
-gnunet-identity -D -z delegatedego -c $TEST_CONFIG
-gnunet-identity -D -z short-zone -c $TEST_CONFIG
-
-gnunet-arm -e -c test_gns_nick_shorten.conf
-rm -rf /tmp/test-gnunet-gns-peer-1/
-
-exit $RES
-
diff --git a/src/gns/test_gns_reverse_lookup.sh b/src/gns/test_gns_reverse_lookup.sh
deleted file mode 100755
index 189adef11..000000000
--- a/src/gns/test_gns_reverse_lookup.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-trap "gnunet-arm -e -c test_gns_lookup.conf" SIGINT
-which timeout &> /dev/null && DO_TIMEOUT="timeout 30"
-
-LOCATION=$(which gnunet-config)
-if [ -z $LOCATION ]
-then
-  LOCATION="gnunet-config"
-fi
-$LOCATION --version 1> /dev/null
-if test $? != 0
-then
-        echo "GNUnet command line tools cannot be found, check environmental variables PATH and GNUNET_PREFIX" 
-        exit 77
-fi
-
-TEST_NAME="dave.bob.alice.gnu"
-gnunet-arm -s -c test_gns_lookup.conf
-gnunet-identity -C bob -c test_gns_lookup.conf
-BOB_PKEY=$(gnunet-identity -d -c test_gns_lookup.conf | grep bob | awk '{print $3}')
-gnunet-identity -C daveego -c test_gns_lookup.conf
-DAVE_PKEY=$(gnunet-identity -d -c test_gns_lookup.conf | grep dave | awk '{print $3}')
-gnunet-identity -C aliceego -c test_gns_lookup.conf
-ALICE_PKEY=$(gnunet-identity -d -c test_gns_lookup.conf | grep alice | awk '{print $3}')
-gnunet-identity -C testego -c test_gns_lookup.conf
-ROOT_PKEY=$(gnunet-identity -d -c test_gns_lookup.conf | grep testego | awk '{print $3}')
-
-gnunet-identity -s gns-reverse -e bob -c test_gns_lookup.conf
-
-gnunet-namestore -p -z testego -a -n alice -t PKEY -V $ALICE_PKEY -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z aliceego -a -n bob -t PKEY -V $BOB_PKEY -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z aliceego -a -n + -t REVERSE -V "alice $ROOT_PKEY 0" -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z bob -a -n dave -t PKEY -V $DAVE_PKEY -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z bob -a -n alice -t PKEY -V $ALICE_PKEY -e never -c test_gns_lookup.conf
-#gnunet-namestore -p -z bob -a -n + -t REVERSE -V "bob $ALICE_PKEY 0" -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z daveego -a -n + -t REVERSE -V "dave $BOB_PKEY 0" -e never -c test_gns_lookup.conf
-gnunet-namestore -p -z daveego -a -n bob -t PKEY -V $BOB_PKEY -e never -c test_gns_lookup.conf
-gnunet-arm -i gns -c test_gns_lookup.conf
-sleep 10
-RES_NAME=`$DO_TIMEOUT gnunet-gns --raw -z testego -R $DAVE_PKEY -c test_gns_lookup.conf`
-gnunet-arm -e -c test_gns_lookup.conf
-rm -rf /tmp/test-gnunet-gns-peer-1/
-
-if [ "$RES_NAME" == "$TEST_NAME" ]
-then
-  exit 0
-else
-  echo "Failed to resolve to proper IP, got $RES_IP."
-  exit 1
-fi
diff --git a/src/gns/w32nsp-resolve.c b/src/gns/w32nsp-resolve.c
index 1de1a3657..82f15c6cc 100644
--- a/src/gns/w32nsp-resolve.c
+++ b/src/gns/w32nsp-resolve.c
@@ -22,6 +22,9 @@
  * @brief W32 integration for GNS
  * @author LRN
  */
+/* Instead of including gnunet_common.h */
+#define GNUNET_memcpy(dst,src,n) do { if (0 != n) { (void) memcpy (dst,src,n); } } while (0)
+
 #include <ws2tcpip.h>
 #include <windows.h>
 #include <nspapi.h>
diff --git a/src/gnsrecord/Makefile.am b/src/gnsrecord/Makefile.am
index e32751247..2fb427c69 100644
--- a/src/gnsrecord/Makefile.am
+++ b/src/gnsrecord/Makefile.am
@@ -22,7 +22,7 @@ check_PROGRAMS = \
  test_gnsrecord_block_expiration
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
   $(check_PROGRAMS) \
   $(check_SCRIPTS)
diff --git a/src/hello/Makefile.am b/src/hello/Makefile.am
index 563bcafb2..79003301b 100644
--- a/src/hello/Makefile.am
+++ b/src/hello/Makefile.am
@@ -29,7 +29,7 @@ check_PROGRAMS = \
  test_friend_hello
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/hello/hello.c b/src/hello/hello.c
index f9b21aa4f..27580275f 100644
--- a/src/hello/hello.c
+++ b/src/hello/hello.c
@@ -206,7 +206,7 @@ GNUNET_HELLO_create (const struct GNUNET_CRYPTO_EddsaPublicKey *public_key,
                      void *addrgen_cls,
                      int friend_only)
 {
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - 256 -
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1 - 256 -
               sizeof (struct GNUNET_HELLO_Message)];
   size_t max;
   size_t used;
diff --git a/src/hostlist/Makefile.am b/src/hostlist/Makefile.am
index 31ce5bf62..17f0bdfa0 100644
--- a/src/hostlist/Makefile.am
+++ b/src/hostlist/Makefile.am
@@ -70,7 +70,7 @@ endif
 
 if HAVE_MHD
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
  $(check_PROGRAMS)
 endif
diff --git a/src/hostlist/gnunet-daemon-hostlist.c b/src/hostlist/gnunet-daemon-hostlist.c
index a83d46e07..854340d3d 100644
--- a/src/hostlist/gnunet-daemon-hostlist.c
+++ b/src/hostlist/gnunet-daemon-hostlist.c
@@ -369,23 +369,26 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
 #if HAVE_MHD
-    {'a', "advertise", NULL,
-     gettext_noop ("advertise our hostlist to other peers"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &advertising},
+    GNUNET_GETOPT_option_flag ('a',
+                                  "advertise",
+                                  gettext_noop ("advertise our hostlist to other peers"),
+                                  &advertising),
 #endif
-    {'b', "bootstrap", NULL,
-     gettext_noop
-     ("bootstrap using hostlists (it is highly recommended that you always use this option)"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &bootstrapping},
-    {'e', "enable-learning", NULL,
-     gettext_noop ("enable learning about hostlist servers from other peers"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &learning},
+    GNUNET_GETOPT_option_flag ('b',
+                                  "bootstrap",
+                                  gettext_noop ("bootstrap using hostlists (it is highly recommended that you always use this option)"),
+                                  &bootstrapping),
+    GNUNET_GETOPT_option_flag ('e',
+                                  "enable-learning",
+                                  gettext_noop ("enable learning about hostlist servers from other peers"),
+                                  &learning),
 #if HAVE_MHD
-    {'p', "provide-hostlist", NULL,
-     gettext_noop ("provide a hostlist server"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &provide_hostlist},
+    GNUNET_GETOPT_option_flag ('p',
+                                  "provide-hostlist",
+                                  gettext_noop ("provide a hostlist server"),
+                                  &provide_hostlist),
 #endif
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/hostlist/gnunet-daemon-hostlist_client.c b/src/hostlist/gnunet-daemon-hostlist_client.c
index a973fcc28..207cc4a81 100644
--- a/src/hostlist/gnunet-daemon-hostlist_client.c
+++ b/src/hostlist/gnunet-daemon-hostlist_client.c
@@ -350,7 +350,7 @@ callback_download (void *ptr,
                    size_t nmemb,
                    void *ctx)
 {
-  static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
+  static char download_buffer[GNUNET_MAX_MESSAGE_SIZE - 1];
   const char *cbuf = ptr;
   const struct GNUNET_MessageHeader *msg;
   struct HelloOffer *ho;
@@ -373,7 +373,7 @@ callback_download (void *ptr,
   left = total;
   while ((left > 0) || (download_pos > 0))
   {
-    cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
+    cpy = GNUNET_MIN (left, GNUNET_MAX_MESSAGE_SIZE - 1 - download_pos);
     GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
     cbuf += cpy;
     download_pos += cpy;
@@ -1042,7 +1042,7 @@ download_hostlist ()
 #if 0
   CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
 #endif
-  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_MAX_MESSAGE_SIZE);
   if (0 == strncmp (current_url, "http", 4))
     CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
   CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
diff --git a/src/hostlist/gnunet-daemon-hostlist_server.c b/src/hostlist/gnunet-daemon-hostlist_server.c
index 48c1a5622..3df6f1ef8 100644
--- a/src/hostlist/gnunet-daemon-hostlist_server.c
+++ b/src/hostlist/gnunet-daemon-hostlist_server.c
@@ -489,7 +489,7 @@ connect_handler (void *cls,
     return NULL;
   size = strlen (hostlist_uri) + 1;
   if (size + sizeof (struct GNUNET_MessageHeader) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/identity-provider/gnunet-identity-token.c b/src/identity-provider/gnunet-identity-token.c
index 4bb3292be..30b63bfc4 100644
--- a/src/identity-provider/gnunet-identity-token.c
+++ b/src/identity-provider/gnunet-identity-token.c
@@ -158,13 +158,18 @@ run (void *cls,
 int
 main(int argc, char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'t', "token", NULL,
-      gettext_noop ("GNUid token"), 1,
-      &GNUNET_GETOPT_set_string, &token},
-    {'p', "print", NULL,
-      gettext_noop ("Print token contents"), 0,
-      &GNUNET_GETOPT_set_one, &print_token},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "token",
+                                 NULL,
+                                 gettext_noop ("GNUid token"),
+                                 &token),
+
+    GNUNET_GETOPT_option_flag ('p',
+                                  "print",
+                                  gettext_noop ("Print token contents"),
+                                  &print_token),
 
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/identity-provider/gnunet-service-identity-provider.c b/src/identity-provider/gnunet-service-identity-provider.c
index e97be55d1..f9e06fef9 100644
--- a/src/identity-provider/gnunet-service-identity-provider.c
+++ b/src/identity-provider/gnunet-service-identity-provider.c
@@ -480,6 +480,7 @@ handle_token_update (void *cls)
       {
         cur_value = GNUNET_CONTAINER_multihashmap_get (ego_entry->attr_map,
                                                        &key_hash);
+        GNUNET_assert (NULL != cur_value);
         GNUNET_CONTAINER_DLL_insert (new_token->attr_head,
                                      new_token->attr_tail,
                                      cur_value);
@@ -752,7 +753,7 @@ attribute_collect (void *cls,
       GNUNET_CONTAINER_DLL_insert (attr->val_head,
                                    attr->val_tail,
                                    val);
-      GNUNET_assert (GNUNET_OK == 
+      GNUNET_assert (GNUNET_OK ==
                      GNUNET_CONTAINER_multihashmap_put (ego_entry->attr_map,
                                                         &key,
                                                         attr,
@@ -989,7 +990,7 @@ create_issue_result_message (const char* label,
   struct IssueResultMessage *irm;
   char *tmp_str;
   size_t len;
-  
+
   GNUNET_asprintf (&tmp_str, "%s,%s,%s", label, ticket, token);
   len = strlen (tmp_str) + 1;
   env = GNUNET_MQ_msg_extra (irm,
@@ -1430,7 +1431,7 @@ check_exchange_message (void *cls,
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
-} 
+}
 
 /**
  *
@@ -1477,7 +1478,6 @@ handle_exchange_message (void *cls,
                          &xchange_handle->ticket->payload->identity_key,
                          GNUNET_GNSRECORD_TYPE_ID_TOKEN,
                          GNUNET_GNS_LO_LOCAL_MASTER,
-                         NULL,
                          &process_lookup_result,
                          xchange_handle);
   GNUNET_free (lookup_query);
diff --git a/src/identity-provider/identity_provider_api.c b/src/identity-provider/identity_provider_api.c
index 21ec235b6..04e3a54f9 100644
--- a/src/identity-provider/identity_provider_api.c
+++ b/src/identity-provider/identity_provider_api.c
@@ -499,7 +499,7 @@ GNUNET_IDENTITY_PROVIDER_exchange_ticket (struct GNUNET_IDENTITY_PROVIDER_Handle
   ticket_str = GNUNET_IDENTITY_PROVIDER_ticket_to_string (ticket);
 
   slen = strlen (ticket_str) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct ExchangeMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct ExchangeMessage))
   {
     GNUNET_free (ticket_str);
     GNUNET_break (0);
diff --git a/src/identity-provider/plugin_rest_identity_provider.c b/src/identity-provider/plugin_rest_identity_provider.c
index cd0c76989..5ea7b2821 100644
--- a/src/identity-provider/plugin_rest_identity_provider.c
+++ b/src/identity-provider/plugin_rest_identity_provider.c
@@ -602,6 +602,7 @@ issue_token_cont (struct GNUNET_REST_RequestHandle *con,
   }
   nonce_str = GNUNET_CONTAINER_multihashmap_get (handle->conndata_handle->url_param_map,
                                                  &key);
+  GNUNET_assert (NULL != nonce_str);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Request nonce: %s\n",
               nonce_str);
@@ -817,6 +818,7 @@ list_token_cont (struct GNUNET_REST_RequestHandle *con_handle,
   }
   ego_val = GNUNET_CONTAINER_multihashmap_get (handle->conndata_handle->url_param_map,
                                                &key);
+  GNUNET_assert (NULL != ego_val);
   //Remove non-matching egos
   for (ego_entry = handle->ego_head;
        NULL != ego_entry;)
@@ -889,6 +891,7 @@ exchange_cont (void *cls,
   }
   nonce_str = GNUNET_CONTAINER_multihashmap_get (handle->conndata_handle->url_param_map,
                                                  &key);
+  GNUNET_assert (NULL != nonce_str);
   GNUNET_assert (1 == sscanf (nonce_str, "%"SCNu64, &expected_nonce));
 
   if (ticket_nonce != expected_nonce)
diff --git a/src/identity/Makefile.am b/src/identity/Makefile.am
index c6f6625f4..94e8c5e94 100644
--- a/src/identity/Makefile.am
+++ b/src/identity/Makefile.am
@@ -92,7 +92,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS) 
 endif
 
diff --git a/src/identity/gnunet-identity.c b/src/identity/gnunet-identity.c
index f632aa0a8..9b66a1bc7 100644
--- a/src/identity/gnunet-identity.c
+++ b/src/identity/gnunet-identity.c
@@ -29,6 +29,12 @@
 #include "gnunet_util_lib.h"
 #include "gnunet_identity_service.h"
 
+
+/**
+ * Return value from main on timeout.
+ */
+#define TIMEOUT_STATUS_CODE 40
+
 /**
  * Handle to IDENTITY service.
  */
@@ -79,6 +85,11 @@ static struct GNUNET_IDENTITY_Operation *create_op;
  */
 static struct GNUNET_IDENTITY_Operation *delete_op;
 
+/**
+ * Value to return from #main().
+ */
+static int global_ret;
+
 
 /**
  * Task run on shutdown.
@@ -120,7 +131,11 @@ test_finished ()
        (NULL == set_ego) &&
        (! list) &&
        (! monitor) )
+  {
+    if (TIMEOUT_STATUS_CODE == global_ret)
+      global_ret = 0;
     GNUNET_SCHEDULER_shutdown ();
+  }
 }
 
 
@@ -159,9 +174,12 @@ create_finished (void *cls,
 
   *op = NULL;
   if (NULL != emsg)
+  {
     fprintf (stderr,
              _("Failed to create ego: %s\n"),
              emsg);
+    global_ret = 1;
+  }
   test_finished ();
 }
 
@@ -178,9 +196,12 @@ set_done (void *cls,
 {
   set_op = NULL;
   if (NULL != emsg)
+  {
     fprintf (stderr,
              _("Failed to set default ego: %s\n"),
              emsg);
+    global_ret = 1;
+  }
   test_finished ();
 }
 
@@ -257,17 +278,23 @@ print_ego (void *cls,
   }
   if ( (NULL == ego) && (! monitor) )
   {
-    GNUNET_SCHEDULER_shutdown ();
+    list = 0;
+    test_finished ();
     return;
   }
   if (! (list | monitor))
     return;
   if (NULL == ego)
     return;
-  GNUNET_IDENTITY_ego_get_public_key (ego, &pk);
+  GNUNET_IDENTITY_ego_get_public_key (ego,
+                                      &pk);
   s = GNUNET_CRYPTO_ecdsa_public_key_to_string (&pk);
-  if ( (monitor) || (NULL != identifier) )
-    fprintf (stdout, "%s - %s\n", identifier, s);
+  if ( (monitor) ||
+       (NULL != identifier) )
+    fprintf (stdout,
+             "%s - %s\n",
+             identifier,
+             s);
   GNUNET_free (s);
 }
 
@@ -281,7 +308,9 @@ print_ego (void *cls,
  * @param cfg configuration
  */
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   if ( (NULL == set_subsystem) ^
@@ -291,7 +320,9 @@ run (void *cls, char *const *args, const char *cfgfile,
              "Options -e and -s must always be specified together\n");
     return;
   }
-  sh = GNUNET_IDENTITY_connect (cfg, &print_ego, NULL);
+  sh = GNUNET_IDENTITY_connect (cfg,
+                                &print_ego,
+                                NULL);
   if (NULL != delete_ego)
     delete_op = GNUNET_IDENTITY_delete (sh,
                                         delete_ego,
@@ -302,7 +333,8 @@ run (void *cls, char *const *args, const char *cfgfile,
                                         create_ego,
                                         &create_finished,
                                         &create_op);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
   test_finished ();
 }
 
@@ -317,42 +349,60 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  int res;
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_string ('C',
+                                 "create",
+                                 "NAME",
+                                 gettext_noop ("create ego NAME"),
+                                 &create_ego),
+
+    GNUNET_GETOPT_option_string ('D',
+                                 "delete",
+                                 "NAME",
+                                 gettext_noop ("delete ego NAME "),
+                                 &delete_ego),
+
+    GNUNET_GETOPT_option_flag ('d',
+                                  "display",
+                                  gettext_noop ("display all egos"),
+                                  &list),
+    
+    GNUNET_GETOPT_option_string ('e',
+                                 "ego",
+                                 "NAME",
+                                 gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -s)"),
+                                 &set_ego),
+
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("run in monitor mode egos"),
+                                  &monitor),
+
+    GNUNET_GETOPT_option_string ('s',
+                                 "set",
+                                 "SUBSYSTEM",
+                                 gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -e)"),
+                                 &set_subsystem),
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "create", "NAME",
-     gettext_noop ("create ego NAME"),
-     1, &GNUNET_GETOPT_set_string, &create_ego},
-    {'D', "delete", "NAME",
-     gettext_noop ("delete ego NAME "),
-     1, &GNUNET_GETOPT_set_string, &delete_ego},
-    {'d', "display", NULL,
-     gettext_noop ("display all egos"),
-     0, &GNUNET_GETOPT_set_one, &list},
-    {'e', "ego", "NAME",
-     gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -s)"),
-     1, &GNUNET_GETOPT_set_string, &set_ego},
-    {'m', "monitor", NULL,
-     gettext_noop ("run in monitor mode egos"),
-     0, &GNUNET_GETOPT_set_one, &monitor},
-    {'s', "set", "SUBSYSTEM",
-     gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -e)"),
-     1, &GNUNET_GETOPT_set_string, &set_subsystem},
     GNUNET_GETOPT_OPTION_END
   };
+  int res;
 
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
-    return 2;
-
-  res = GNUNET_PROGRAM_run (argc, argv, "gnunet-identity",
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
+    return 4;
+  global_ret = TIMEOUT_STATUS_CODE; /* timeout */
+  res = GNUNET_PROGRAM_run (argc, argv,
+                            "gnunet-identity",
                             gettext_noop ("Maintain egos"),
                             options, &run,
                             NULL);
   GNUNET_free ((void *) argv);
 
   if (GNUNET_OK != res)
-    return 1;
-  return 0;
+    return 3;
+  return global_ret;
 }
 
 /* end of gnunet-identity.c */
diff --git a/src/identity/identity_api.c b/src/identity/identity_api.c
index 905b3fd8b..1601ae2fd 100644
--- a/src/identity/identity_api.c
+++ b/src/identity/identity_api.c
@@ -671,7 +671,7 @@ GNUNET_IDENTITY_get (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (service_name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct GetDefaultMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct GetDefaultMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -722,7 +722,7 @@ GNUNET_IDENTITY_set (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (service_name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct SetDefaultMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct SetDefaultMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -773,7 +773,7 @@ GNUNET_IDENTITY_create (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct CreateRequestMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct CreateRequestMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -830,9 +830,9 @@ GNUNET_IDENTITY_rename (struct GNUNET_IDENTITY_Handle *h,
     return NULL;
   slen_old = strlen (old_name) + 1;
   slen_new = strlen (new_name) + 1;
-  if ( (slen_old >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen_new >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen_old + slen_new >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct RenameMessage)) )
+  if ( (slen_old >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen_new >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen_old + slen_new >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct RenameMessage)) )
   {
     GNUNET_break (0);
     return NULL;
@@ -885,7 +885,7 @@ GNUNET_IDENTITY_delete (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct DeleteMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct DeleteMessage))
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index bf3ffe482..b745da125 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -38,7 +38,6 @@ gnunetinclude_HEADERS = \
   gnunet_common.h \
   gnunet_constants.h \
   gnunet_configuration_lib.h \
-  gnunet_connection_lib.h \
   gnunet_consensus_service.h \
   gnunet_container_lib.h \
   gnunet_conversation_service.h \
@@ -107,13 +106,13 @@ gnunetinclude_HEADERS = \
   gnunet_scalarproduct_service.h \
   gnunet_scheduler_lib.h \
   gnunet_secretsharing_service.h \
-  gnunet_server_lib.h \
   gnunet_service_lib.h \
   gnunet_set_service.h \
   gnunet_signal_lib.h \
   gnunet_signatures.h \
   gnunet_social_service.h \
   gnunet_speaker_lib.h \
+  gnunet_sq_lib.h \
   gnunet_statistics_service.h \
   gnunet_strings_lib.h \
   gnunet_testbed_service.h \
diff --git a/src/include/gnunet_arm_service.h b/src/include/gnunet_arm_service.h
index 7fb14d3ac..f8d71bd8b 100644
--- a/src/include/gnunet_arm_service.h
+++ b/src/include/gnunet_arm_service.h
@@ -62,25 +62,9 @@ enum GNUNET_ARM_RequestStatus
   GNUNET_ARM_REQUEST_SENT_OK = 0,
 
   /**
-   * Misconfiguration (can't connect to the ARM service).
-   */
-  GNUNET_ARM_REQUEST_CONFIGURATION_ERROR = 1,
-
-  /**
    * We disconnected from ARM, and request was not sent.
    */
-  GNUNET_ARM_REQUEST_DISCONNECTED = 2,
-
-  /**
-   * ARM API is busy (probably trying to connect to ARM),
-   * and request was not sent. Try again later.
-   */
-  GNUNET_ARM_REQUEST_BUSY = 3,
-
-  /**
-   * Request time ran out before we had a chance to send it.
-   */
-  GNUNET_ARM_REQUEST_TIMEOUT = 5
+  GNUNET_ARM_REQUEST_DISCONNECTED = 2
 
 };
 
diff --git a/src/include/gnunet_bandwidth_lib.h b/src/include/gnunet_bandwidth_lib.h
index 178ddaaac..967d50dea 100644
--- a/src/include/gnunet_bandwidth_lib.h
+++ b/src/include/gnunet_bandwidth_lib.h
@@ -133,7 +133,7 @@ struct GNUNET_BANDWIDTH_Tracker
   /**
    * Maximum number of seconds over which bandwidth may "accumulate".
    * Note that additionally, we also always allow at least
-   * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.
+   * #GNUNET_MAX_MESSAGE_SIZE to accumulate.
    */
   uint32_t max_carry_s__;
 };
@@ -214,10 +214,10 @@ GNUNET_BANDWIDTH_value_max (struct GNUNET_BANDWIDTH_Value32NBO b1,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
@@ -238,10 +238,10 @@ GNUNET_BANDWIDTH_tracker_init (struct GNUNET_BANDWIDTH_Tracker *av,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
diff --git a/src/include/gnunet_block_group_lib.h b/src/include/gnunet_block_group_lib.h
new file mode 100644
index 000000000..3a3dfb2e2
--- /dev/null
+++ b/src/include/gnunet_block_group_lib.h
@@ -0,0 +1,114 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @author Christian Grothoff
+ *
+ * @file
+ * Library for creating block groups (to be used by block plugins)
+ *
+ * @defgroup block  Block group library
+ * Library for data group management
+ * @{
+ */
+#ifndef GNUNET_BLOCK_GROUP_LIB_H
+#define GNUNET_BLOCK_GROUP_LIB_H
+
+#include "gnunet_util_lib.h"
+#include "gnunet_block_lib.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+
+/**
+ * How many bytes should a bloomfilter be if we have already seen
+ * entry_count responses?  Sized so that do not have to
+ * re-size the filter too often (to keep it cheap).
+ *
+ * Since other peers will also add entries but not resize the filter,
+ * we should generally pick a slightly larger size than what the
+ * strict math would suggest.
+ *
+ * @param entry_count expected number of entries in the Bloom filter
+ * @param k number of bits set per entry
+ * @return must be a power of two and smaller or equal to 2^15.
+ */
+size_t
+GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count,
+                                             unsigned int k);
+
+
+/**
+ * Create a new block group that filters duplicates using a Bloom filter.
+ *
+ * @param ctx block context in which the block group is created
+ * @param bf_size size of the Bloom filter
+ * @param bf_k K-value for the Bloom filter
+ * @param type block type
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+struct GNUNET_BLOCK_Group *
+GNUNET_BLOCK_GROUP_bf_create (void *cls,
+                              size_t bf_size,
+                              unsigned int bf_k,
+                              enum GNUNET_BLOCK_Type type,
+                              uint32_t nonce,
+                              const void *raw_data,
+                              size_t raw_data_size);
+
+
+/**
+ * Test if @a hc is contained in the Bloom filter of @a bg.  If so,
+ * return #GNUNET_YES.  If not, add @a hc to the Bloom filter and
+ * return #GNUNET_NO.
+ *
+ * @param bg block group to use for testing
+ * @param hc hash of element to evaluate
+ * @return #GNUNET_YES if @a hc is (likely) a duplicate
+ *         #GNUNET_NO if @a hc was definitively not in @bg (but now is)
+ */
+int
+GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg,
+                                    const struct GNUNET_HashCode *hc);
+
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+/* ifndef GNUNET_BLOCK_GROUP_LIB_H */
+#endif
+
+/** @} */  /* end of group */
+
+/* end of gnunet_block_group_lib.h */
diff --git a/src/include/gnunet_block_lib.h b/src/include/gnunet_block_lib.h
index b21b3496b..d8ee68c21 100644
--- a/src/include/gnunet_block_lib.h
+++ b/src/include/gnunet_block_lib.h
@@ -112,6 +112,11 @@ enum GNUNET_BLOCK_Type
   GNUNET_BLOCK_TYPE_GNS_NAMERECORD = 11,
 
   /**
+   * Block type for a revocation message by which a key is revoked.
+   */
+  GNUNET_BLOCK_TYPE_REVOCATION = 12,
+
+  /**
    * Block to store a cadet regex state
    */
   GNUNET_BLOCK_TYPE_REGEX = 22,
@@ -119,7 +124,19 @@ enum GNUNET_BLOCK_Type
   /**
    * Block to store a cadet regex accepting state
    */
-  GNUNET_BLOCK_TYPE_REGEX_ACCEPT = 23
+  GNUNET_BLOCK_TYPE_REGEX_ACCEPT = 23,
+
+  /**
+   * Block for testing set/consensus.  If first byte of the block
+   * is non-zero, the block is considered invalid.
+   */
+  GNUNET_BLOCK_TYPE_SET_TEST = 24,
+
+  /**
+   * Block type for consensus elements.
+   * Contains either special marker elements or a nested block.
+   */
+  GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT = 25,
 };
 
 
@@ -230,6 +247,61 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx);
 
 
 /**
+ * Handle for a group of elements that will be evaluated together.
+ * They must all be of the same type.  A block group allows the
+ * plugin to keep some state across individual evaluations.
+ */
+struct GNUNET_BLOCK_Group;
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param ... type-specific additional data, can be empty
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+struct GNUNET_BLOCK_Group *
+GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
+                           enum GNUNET_BLOCK_Type type,
+                           uint32_t nonce,
+                           const void *raw_data,
+                           size_t raw_data_size,
+                           ...);
+
+
+/**
+ * Serialize state of a block group.
+ *
+ * @param bg group to serialize
+ * @param[out] nonce set to the nonce of the @a bg
+ * @param[out] raw_data set to the serialized state
+ * @param[out] raw_data_size set to the number of bytes in @a raw_data
+ * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
+ *         supported, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg,
+                              uint32_t *nonce,
+                              void **raw_data,
+                              size_t *raw_data_size);
+
+
+/**
+ * Destroy resources used by a block group.
+ *
+ * @param bg group to destroy, NULL is allowed
+ */
+void
+GNUNET_BLOCK_group_destroy (struct GNUNET_BLOCK_Group *bg);
+
+
+/**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the @a reply_block.
  * Note that it is assumed that the reply has already been
@@ -238,10 +310,9 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx);
  *
  * @param ctx block contxt
  * @param type block type
+ * @param group block group to use for evaluation
  * @param eo evaluation options to control evaluation
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -251,10 +322,9 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx);
 enum GNUNET_BLOCK_EvaluationResult
 GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx,
                        enum GNUNET_BLOCK_Type type,
+                       struct GNUNET_BLOCK_Group *group,
                        enum GNUNET_BLOCK_EvaluationOptions eo,
                        const struct GNUNET_HashCode *query,
-                       struct GNUNET_CONTAINER_BloomFilter **bf,
-                       int32_t bf_mutator,
                        const void *xquery,
                        size_t xquery_size,
                        const void *reply_block,
@@ -279,24 +349,41 @@ GNUNET_BLOCK_get_key (struct GNUNET_BLOCK_Context *ctx,
                       enum GNUNET_BLOCK_Type type,
                       const void *block,
                       size_t block_size,
-                      struct GNUNET_HashCode * key);
-
+                      struct GNUNET_HashCode *key);
 
 
 /**
- * Construct a bloom filter that would filter out the given
- * results.
+ * Update block group to filter out the given results.  Note that the
+ * use of a hash for seen results implies that the caller magically
+ * knows how the specific block engine hashes for filtering
+ * duplicates, so this API may not always apply.
  *
  * @param bf_mutator mutation value to use
  * @param seen_results results already seen
  * @param seen_results_count number of entries in @a seen_results
- * @return NULL if seen_results_count is 0, otherwise a BF
- *         that would match the given results.
+ * @return #GNUNET_SYSERR if not supported, #GNUNET_OK on success
  */
-struct GNUNET_CONTAINER_BloomFilter *
-GNUNET_BLOCK_construct_bloomfilter (int32_t bf_mutator,
-                                    const struct GNUNET_HashCode *seen_results,
-                                    unsigned int seen_results_count);
+int
+GNUNET_BLOCK_group_set_seen (struct GNUNET_BLOCK_Group *bg,
+                             const struct GNUNET_HashCode *seen_results,
+                             unsigned int seen_results_count);
+
+
+/**
+ * Try merging two block groups.  Afterwards, @a bg1 should remain
+ * valid and contain the rules from both @a bg1 and @bg2, and
+ * @a bg2 should be destroyed (as part of this call).  The latter
+ * should happen even if merging is not supported.
+ *
+ * @param[in,out] bg1 first group to merge, is updated
+ * @param bg2 second group to merge, is destroyed
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_NO if merge failed due to different nonce
+ *         #GNUNET_SYSERR if merging is not supported
+ */
+int
+GNUNET_BLOCK_group_merge (struct GNUNET_BLOCK_Group *bg1,
+                          struct GNUNET_BLOCK_Group *bg2);
 
 
 #if 0                           /* keep Emacsens' auto-indent happy */
diff --git a/src/include/gnunet_block_plugin.h b/src/include/gnunet_block_plugin.h
index 5c320457e..3eb031573 100644
--- a/src/include/gnunet_block_plugin.h
+++ b/src/include/gnunet_block_plugin.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2010,2013 GNUnet e.V.
+     Copyright (C) 2010,2013,2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -40,6 +40,130 @@
 
 
 /**
+ * Mark elements as "seen" using a hash of the element. Not supported
+ * by all block plugins.
+ *
+ * @param bg group to update
+ * @param seen_results results already seen
+ * @param seen_results_count number of entries in @a seen_results
+ */
+typedef void
+(*GNUNET_BLOCK_GroupMarkSeenFunction)(struct GNUNET_BLOCK_Group *bg,
+                                      const struct GNUNET_HashCode *seen_results,
+                                      unsigned int seen_results_count);
+
+
+/**
+ * Merge two groups, if possible. Not supported by all block plugins,
+ * can also fail if the nonces were different.
+ *
+ * @param bg1 group to update
+ * @param bg2 group to merge into @a bg1
+ * @return #GNUNET_OK on success, #GNUNET_NO if the nonces were different and thus
+ *         we failed.
+ */
+typedef int
+(*GNUNET_BLOCK_GroupMergeFunction)(struct GNUNET_BLOCK_Group *bg1,
+                                   const struct GNUNET_BLOCK_Group *bg2);
+
+
+/**
+ * Serialize state of a block group.
+ *
+ * @param bg group to serialize
+ * @param[out] nonce set to the nonce of the @a bg
+ * @param[out] raw_data set to the serialized state
+ * @param[out] raw_data_size set to the number of bytes in @a raw_data
+ * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
+ *         supported, #GNUNET_SYSERR on error
+ */
+typedef int
+(*GNUNET_BLOCK_GroupSerializeFunction)(struct GNUNET_BLOCK_Group *bg,
+                                       uint32_t *nonce,
+                                       void **raw_data,
+                                       size_t *raw_data_size);
+
+
+/**
+ * Destroy resources used by a block group.
+ *
+ * @param bg group to destroy, NULL is allowed
+ */
+typedef void
+(*GNUNET_BLOCK_GroupDestroyFunction)(struct GNUNET_BLOCK_Group *bg);
+
+
+/**
+ * Block group data.  The plugin must initialize the callbacks
+ * and can use the @e internal_cls as it likes.
+ */
+struct GNUNET_BLOCK_Group
+{
+
+  /**
+   * Context owning the block group. Set by the main block library.
+   */
+  struct GNUENT_BLOCK_Context *ctx;
+
+  /**
+   * Type for the block group.  Set by the main block library.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Serialize the block group data, can be NULL if
+   * not supported.
+   */
+  GNUNET_BLOCK_GroupSerializeFunction serialize_cb;
+
+  /**
+   * Function to call to mark elements as seen in the group.
+   * Can be NULL if not supported.
+   */
+  GNUNET_BLOCK_GroupMarkSeenFunction mark_seen_cb;
+
+  /**
+   * Function to call to merge two groups.
+   * Can be NULL if not supported.
+   */
+  GNUNET_BLOCK_GroupMergeFunction merge_cb;
+
+  /**
+   * Function to call to destroy the block group.
+   * Must not be NULL.
+   */
+  GNUNET_BLOCK_GroupDestroyFunction destroy_cb;
+
+  /**
+   * Internal data structure of the plugin.
+   */
+  void *internal_cls;
+
+};
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+typedef struct GNUNET_BLOCK_Group *
+(*GNUNET_BLOCK_GroupCreateFunction)(void *cls,
+                                    enum GNUNET_BLOCK_Type type,
+                                    uint32_t nonce,
+                                    const void *raw_data,
+                                    size_t raw_data_size,
+                                    va_list va);
+
+
+/**
  * Function called to validate a reply or a request.  For
  * request evaluation, simply pass "NULL" for the @a reply_block.
  * Note that it is assumed that the reply has already been
@@ -47,11 +171,11 @@
  * be done with the "get_key" function.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param group which block group to use for evaluation
  * @param eo evaluation options to control evaluation
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for @a bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -60,11 +184,11 @@
  */
 typedef enum GNUNET_BLOCK_EvaluationResult
 (*GNUNET_BLOCK_EvaluationFunction) (void *cls,
+                                    struct GNUNET_BLOCK_Context *ctx,
                                     enum GNUNET_BLOCK_Type type,
+                                    struct GNUNET_BLOCK_Group *group,
                                     enum GNUNET_BLOCK_EvaluationOptions eo,
                                     const struct GNUNET_HashCode *query,
-                                    struct GNUNET_CONTAINER_BloomFilter **bf,
-                                    int32_t bf_mutator,
                                     const void *xquery,
                                     size_t xquery_size,
                                     const void *reply_block,
@@ -121,6 +245,11 @@ struct GNUNET_BLOCK_PluginFunctions
    */
   GNUNET_BLOCK_GetKeyFunction get_key;
 
+  /**
+   * Create a block group to process a bunch of blocks in a shared
+   * context (i.e. to detect duplicates).
+   */
+  GNUNET_BLOCK_GroupCreateFunction create_group;
 };
 
 #endif
diff --git a/src/include/gnunet_cadet_service.h b/src/include/gnunet_cadet_service.h
index 8d10c3d8d..f76f17a51 100644
--- a/src/include/gnunet_cadet_service.h
+++ b/src/include/gnunet_cadet_service.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2014 GNUnet e.V.
+     Copyright (C) 2009-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -49,7 +49,7 @@ extern "C"
 /**
  * Version number of GNUnet-cadet API.
  */
-#define GNUNET_CADET_VERSION 0x00000004
+#define GNUNET_CADET_VERSION 0x00000005
 
 
 /**
@@ -69,6 +69,33 @@ struct GNUNET_CADET_Port;
 
 
 /**
+ * Hash uniquely identifying a connection below a tunnel.
+ */
+struct GNUNET_CADET_ConnectionTunnelIdentifier
+{
+  struct GNUNET_ShortHashCode connection_of_tunnel;
+};
+
+
+/**
+ * Number identifying a CADET channel within a tunnel.
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+{
+  /**
+   * Which number does this channel have that uniquely identfies
+   * it within its tunnel, in network byte order.
+   *
+   * Given two peers, both may initiate channels over the same tunnel.
+   * The @e cn must be greater or equal to 0x80000000 (high-bit set)
+   * for tunnels initiated with the peer that has the larger peer
+   * identity as compared using #GNUNET_CRYPTO_cmp_peer_identity().
+   */
+  uint32_t cn GNUNET_PACKED;
+};
+
+
+/**
  * Channel options.  Second line indicates filed in the
  * CadetChannelInfo union carrying the answer.
  */
@@ -108,118 +135,67 @@ enum GNUNET_CADET_ChannelOption
 
 
 /**
- * Functions with this signature are called whenever a message is
- * received.
+ * Method called whenever a peer connects to a port in MQ-based CADET.
  *
- * Each time the function must call #GNUNET_CADET_receive_done on the channel
- * in order to receive the next message. This doesn't need to be immediate:
- * can be delayed if some processing is done on the message.
- *
- * @param cls Closure (set from #GNUNET_CADET_connect).
- * @param channel Connection to the other end.
- * @param channel_ctx Place to store local state associated with the channel.
- * @param message The actual message.
- * @return #GNUNET_OK to keep the channel open,
- *         #GNUNET_SYSERR to close it (signal serious error).
- */
-typedef int
-(*GNUNET_CADET_MessageCallback) (void *cls,
-                                 struct GNUNET_CADET_Channel *channel,
-                                 void **channel_ctx,
-                                 const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Message handler.  Each struct specifies how to handle on particular
- * type of message received.
+ * @param cls Closure from #GNUNET_CADET_open_port.
+ * @param channel New handle to the channel.
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
  */
-struct GNUNET_CADET_MessageHandler
-{
-  /**
-   * Function to call for messages of type @e type.
-   */
-  GNUNET_CADET_MessageCallback callback;
-
-  /**
-   * Type of the message this handler covers.
-   */
-  uint16_t type;
-
-  /**
-   * Expected size of messages of this type.  Use 0 for variable-size.
-   * If non-zero, messages of the given type will be discarded if they
-   * do not have the right size.
-   */
-  uint16_t expected_size;
-};
+typedef void *
+(*GNUNET_CADET_ConnectEventHandler) (void *cls,
+                                     struct GNUNET_CADET_Channel *channel,
+                                     const struct GNUNET_PeerIdentity *source);
 
 
 /**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- * Only called (once) upon reception of data with a message type which was
- * subscribed to in #GNUNET_CADET_connect.
- *
- * A call to #GNUNET_CADET_channel_destroy causes te channel to be ignored. In
- * this case the handler MUST return NULL.
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
  *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port Port this channel is for.
- * @param options CadetOption flag field, with all active option bits set to 1.
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
  *
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
  */
-typedef void *
-(GNUNET_CADET_InboundChannelNotificationHandler) (void *cls,
-                                                  struct GNUNET_CADET_Channel *channel,
-                                                  const struct GNUNET_PeerIdentity *initiator,
-                                                  const struct GNUNET_HashCode *port,
-                                                  enum GNUNET_CADET_ChannelOption options);
+typedef void
+(*GNUNET_CADET_DisconnectEventHandler) (void *cls,
+                                        const struct GNUNET_CADET_Channel *channel);
 
 
 /**
- * Function called whenever a channel is destroyed.  Should clean up
- * any associated state, including cancelling any pending transmission on this
- * channel.
+ * Function called whenever an MQ-channel's transmission window size changes.
  *
- * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end --- FIXME: drop?
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative. -- FIXME: make unsigned, we never call negative?
  */
 typedef void
-(GNUNET_CADET_ChannelEndHandler) (void *cls,
-                                  const struct GNUNET_CADET_Channel *channel,
-                                  void *channel_ctx);
+(*GNUNET_CADET_WindowSizeEventHandler) (void *cls,
+                                        const struct GNUNET_CADET_Channel *channel,
+                                        int window_size);
 
 
 /**
- * Connect to the cadet service.
+ * Connect to the MQ-based cadet service.
  *
  * @param cfg Configuration to use.
- * @param cls Closure for the various callbacks that follow (including
- *            handlers in the handlers array).
- * @param cleaner Function called when a channel is destroyed.
- *                It is called immediately if #GNUNET_CADET_channel_destroy
- *                is called on the channel.
- * @param handlers Callbacks for messages we care about, NULL-terminated. Each
- *                 one must call #GNUNET_CADET_receive_done on the channel to
- *                 receive the next message.  Messages of a type that is not
- *                 in the handlers array are ignored if received.
- *
- * @return handle to the cadet service NULL on error
- *         (in this case, init is never called)
+ * @return Handle to the cadet service NULL on error.
  */
 struct GNUNET_CADET_Handle *
-GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      void *cls,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      const struct GNUNET_CADET_MessageHandler *handlers);
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg);
 
 
 /**
@@ -233,21 +209,29 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
 void
 GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle);
 
+
 /**
- * Open a port to receive incomming channels.
+ * Open a port to receive incomming MQ-based channels.
  *
  * @param h CADET handle.
- * @param port Hash representing the port number.
- * @param new_channel Function called when an channel is received.
- * @param new_channel_cls Closure for @a new_channel.
- *
+ * @param port Hash identifying the port.
+ * @param connects Function called when an incoming channel is connected.
+ * @param connects_cls Closure for the @a connects handler.
+ * @param window_changes Function called when the transmit window size changes.
+ *                       Can be NULL.
+ * @param disconnects Function called when a channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
  * @return Port handle.
  */
 struct GNUNET_CADET_Port *
 GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                        void *new_channel_cls);
+                        const struct GNUNET_HashCode *port,
+                        GNUNET_CADET_ConnectEventHandler connects,
+                        void *connects_cls,
+                        GNUNET_CADET_WindowSizeEventHandler window_changes,
+                        GNUNET_CADET_DisconnectEventHandler disconnects,
+                        const struct GNUNET_MQ_MessageHandler *handlers);
+
 
 /**
  * Close a port opened with @a GNUNET_CADET_open_port.
@@ -258,27 +242,38 @@ GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
 void
 GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p);
 
+
 /**
  * Create a new channel towards a remote peer.
  *
  * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
+ * does not accept the channel, @a disconnects will be called
  * for this channel.
  *
- * @param h cadet handle
- * @param channel_ctx client's channel context to associate with the channel
- * @param peer peer identity the channel should go to
- * @param port Port hash (port number).
+ * @param h CADET handle.
+ * @param channel_cls Closure for the channel. It's given to:
+ *                    - The management handler @a window_changes.
+ *                    - The disconnect handler @a disconnects
+ *                    - Each message type callback in @a handlers
+ * @param destination Peer identity the channel should go to.
+ * @param port Identification of the destination port.
  * @param options CadetOption flag field, with all desired option bits set to 1.
- *
- * @return handle to the channel
+ * @param window_changes Function called when the transmit window size changes.
+ *                       Can be NULL if this data is of no interest.
+ * TODO                  Not yet implemented.
+ * @param disconnects Function called when the channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Handle to the channel.
  */
 struct GNUNET_CADET_Channel *
 GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
-                            void *channel_ctx,
-                            const struct GNUNET_PeerIdentity *peer,
-                            const struct GNUNET_HashCode *port,
-                            enum GNUNET_CADET_ChannelOption options);
+                             void *channel_cls,
+                             const struct GNUNET_PeerIdentity *destination,
+                             const struct GNUNET_HashCode *port,
+                             enum GNUNET_CADET_ChannelOption options,
+                             GNUNET_CADET_WindowSizeEventHandler window_changes,
+                             GNUNET_CADET_DisconnectEventHandler disconnects,
+                             const struct GNUNET_MQ_MessageHandler *handlers);
 
 
 /**
@@ -295,6 +290,52 @@ GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel);
 
 
 /**
+ * Obtain the message queue for a connected channel.
+ *
+ * @param channel The channel handle from which to get the MQ.
+ * @return The message queue of the channel.
+ */
+struct GNUNET_MQ_Handle *
+GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel);
+
+
+/**
+ * Indicate readiness to receive the next message on a channel.
+ *
+ * Should only be called once per handler called.
+ *
+ * @param channel Channel that will be allowed to call another handler.
+ */
+void
+GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel);
+
+
+/******************************************************************************/
+/********************       MONITORING /DEBUG API     *************************/
+/******************************************************************************/
+/* The following calls are not useful for normal CADET operation, but for      */
+/* debug and monitoring of the cadet state. They can be safely ignored.        */
+/* The API can change at any point without notice.                            */
+/* Please contact the developer if you consider any of this calls useful for  */
+/* normal cadet applications.                                                  */
+/******************************************************************************/
+
+
+/**
+ * Transitional function to convert an unsigned int port to a hash value.
+ * WARNING: local static value returned, NOT reentrant!
+ * WARNING: do not use this function for new code!
+ *
+ * @param port Numerical port (unsigned int format).
+ *
+ * @return A GNUNET_HashCode usable for the new CADET API.
+ */
+const struct GNUNET_HashCode *
+GC_u2h (uint32_t port);
+
+
+
+/**
  * Struct to retrieve info about a channel.
  */
 union GNUNET_CADET_ChannelInfo
@@ -327,76 +368,6 @@ GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
 
 
 /**
- * Handle for a transmission request.
- */
-struct GNUNET_CADET_TransmitHandle;
-
-
-/**
- * Ask the cadet to call @a notify once it is ready to transmit the
- * given number of bytes to the specified channel.
- * Only one call can be active at any time, to issue another request,
- * wait for the callback or cancel the current request.
- *
- * @param channel channel to use for transmission
- * @param cork is corking allowed for this transmission?
- * @param maxdelay how long can the message wait?
- * @param notify_size how many bytes of buffer space does notify want?
- * @param notify function to call when buffer space is available;
- *        will be called with NULL on timeout or if the overall queue
- *        for this peer is larger than queue_size and this is currently
- *        the message with the lowest priority
- * @param notify_cls closure for @a notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we can not even queue the request (insufficient
- *         memory); if NULL is returned, @a notify will NOT be called.
- */
-struct GNUNET_CADET_TransmitHandle *
-GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
-                                   int cork,
-                                   struct GNUNET_TIME_Relative maxdelay,
-                                   size_t notify_size,
-                                   GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                   void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready notification.
- *
- * #DEPRECATED
- * Since soon we will send immediately with mq (via request_data),
- * there will be time or need to cancel a "pending" transmission.
- *
- * @param th handle that was returned by "notify_transmit_ready".
- */
-void
-GNUNET_CADET_notify_transmit_ready_cancel (struct GNUNET_CADET_TransmitHandle *th);
-
-
-/**
- * Indicate readiness to receive the next message on a channel.
- *
- * Should only be called once per handler called.
- *
- * @param channel Channel that will be allowed to call another handler.
- */
-void
-GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel);
-
-
-
-/******************************************************************************/
-/********************       MONITORING /DEBUG API     *************************/
-/******************************************************************************/
-/* The following calls are not useful for normal CADET operation, but for      */
-/* debug and monitoring of the cadet state. They can be safely ignored.        */
-/* The API can change at any point without notice.                            */
-/* Please contact the developer if you consider any of this calls useful for  */
-/* normal cadet applications.                                                  */
-/******************************************************************************/
-
-
-/**
  * Method called to retrieve information about a specific channel the cadet peer
  * is aware of, including all transit nodes.
  *
@@ -456,7 +427,7 @@ typedef void
                         int tunnel,
                         int neighbor,
                         unsigned int n_paths,
-                        struct GNUNET_PeerIdentity *paths);
+                        const struct GNUNET_PeerIdentity *paths);
 
 
 /**
@@ -482,33 +453,6 @@ typedef void
 
 
 /**
- * Hash uniquely identifying a connection below a tunnel.
- */
-struct GNUNET_CADET_ConnectionTunnelIdentifier
-{
-  struct GNUNET_ShortHashCode connection_of_tunnel;
-};
-
-
-/**
- * Number identifying a CADET channel within a tunnel.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-{
-  /**
-   * Which number does this channel have that uniquely identfies
-   * it within its tunnel, in network byte order.
-   *
-   * Given two peers, both may initiate channels over the same tunnel.
-   * The @e cn must be greater or equal to 0x80000000 (high-bit set)
-   * for tunnels initiated with the peer that has the larger peer
-   * identity as compared using #GNUNET_CRYPTO_cmp_peer_identity().
-   */
-  uint32_t cn GNUNET_PACKED;
-};
-
-
-/**
  * Method called to retrieve information about a specific tunnel the cadet peer
  * has established, o`r is trying to establish.
  *
@@ -667,188 +611,6 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
                          void *callback_cls);
 
 
-/**
- * Create a message queue for a cadet channel.
- * The message queue can only be used to transmit messages,
- * not to receive them.
- *
- * @param channel the channel to create the message qeue for
- * @return a message queue to messages over the channel
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_mq_create (struct GNUNET_CADET_Channel *channel);
-
-
-/**
- * Transitional function to convert an unsigned int port to a hash value.
- * WARNING: local static value returned, NOT reentrant!
- * WARNING: do not use this function for new code!
- *
- * @param port Numerical port (unsigned int format).
- *
- * @return A GNUNET_HashCode usable for the new CADET API.
- */
-const struct GNUNET_HashCode *
-GC_u2h (uint32_t port);
-
-
-/******************************************************************************/
-/******************************* MQ-BASED API *********************************/
-/******************************************************************************/
-
-/**
- * Function called after #GNUNET_CADET_connecT has succeeded (or failed
- * for good). Implementations of this function must not call
- * #GNUNET_CADET_disconnecT (other than by scheduling a new task to
- * do this later).
- *
- * @param cls closure
- * @param connected #GNUNET_YES if successfully connected, #GNUNET_NO otherwise.
- */
-typedef void
-(*GNUNET_CADET_StartupCallback) (void *cls, int connected);
-
-
-/**
- * Method called whenever a given peer connects in mq-based CADET.
- *
- * @param cls Closure given to @a GNUNET_CADET_connecT.
- * @param channel New handle to the channel.
- * @param channel_cls Closure given to @a GNUNET_CADET_open_porT.
- *        NOTE: do we need two cls? I'd get rid of this one.
- * @param peer Peer that started this channel.
- *
- * NOTE: to keep symmetry between incoming and outgoing channels, this call
- *       does not provide the *mq, since we cannot cleanly return an mq
- *       from @a GNUNET_CADET_channel_create.
- *       The client must always call @a GNUNET_CADET_get_mq to the *mq
- *       Alternatively, we can provide the mq here and add and out **mq
- *       to @a GNUNET_CADET_channel_create
- *
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
- */
-typedef void *
-(*GNUNET_CADET_ConnectEventHandler) (void *cls,
-                                     struct GNUNET_CADET_Channel *channel,
-                                     const struct GNUNET_PeerIdentity *peer);
-
-/**
- * Function called whenever an mq-channel is destroyed.  Should clean up
- * any associated state, including cancelling any pending transmission on this
- * channel.
- *
- * It must NOT call @a GNUNET_CADET_channel_destroy on the channel.
- *
- * @param cls Closure (set from @a GNUNET_CADET_connecT).
- * @param channel Connection to the other end (henceforth invalid).
- * @param channel_ctx Context (set from @a GNUNET_CADET_ConnectEventHandler).
- */
-typedef void
-(GNUNET_CADET_DisconnectEventHandler) (void *cls,
-                                       const struct GNUNET_CADET_Channel *channel,
-                                       void *channel_ctx);
-
-/**
- * Connect to the mq-based cadet service.
- *
- * NOTE: it would be more elegant to provide a separate @a handlers and
- *       @a disconnects for each port, giving them to @a GNUNET_CADET_open_porT,
- *       but how do we handle *incoming* channels?
- *
- * @param cfg Configuration to use.
- * @param cls Closure for the various callbacks that follow (including
- *            handlers in the handlers array).
- * @param init callback to call once we have successfully connected
- *             to the cadet service
- * @param disconnects Function called when a channel is destroyed.
- *                    It is called immediately if the channel is destroyed by
- *                    calling @a GNUNET_CADET_channel_destroy.
- * @param handlers Callbacks for messages we care about, NULL-terminated.
- *                 Messages of a type that is not in the handlers array
- *                 are ignored if received.
- *
- * @return handle to the cadet service NULL on error
- *         (in this case, init is never called)
- */
-struct GNUNET_CADET_Handle *
-GNUNET_CADET_connecT (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      void *cls,
-                      GNUNET_CADET_StartupCallback init,
-                      GNUNET_CADET_DisconnectEventHandler disconnects,
-                      const struct GNUNET_MQ_MessageHandler *handlers);
-
-/**
- * Disconnect from the mq-based cadet service. All channels will be destroyed.
- * All channel disconnect callbacks will be called on any still connected peers,
- * notifying about their disconnection. The registered inbound channel cleaner
- * will be called should any inbound channels still exist.
- *
- * @param handle connection to cadet to disconnect
- */
-void
-GNUNET_CADET_disconnecT (struct GNUNET_CADET_Handle *handle);
-
-/**
- * Open a port to receive incomming mq-based channels.
- *
- * @param h CADET handle.
- * @param port Hash representing the port number.
- * @param new_channel Function called when an channel is received.
- * @param new_channel_cls Closure for @a new_channel.
- *        NOTE: get rid of this cls?
- *
- * @return Port handle.
- */
-struct GNUNET_CADET_Port *
-GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_ConnectEventHandler new_channel,
-                        void *new_channel_cls);
-
-/**
- * Close a port opened with @a GNUNET_CADET_open_porT.
- * The @a new_channel callback will no longer be called.
- *
- * @param p Port handle.
- */
-void
-GNUNET_CADET_close_porT (struct GNUNET_CADET_Port *p);
-
-/**
- * Obtain the message queue for a connected peer.
- *
- * @param h the cadet handle
- * @param channel the identity of the peer
- *
- * @return NULL if @a channel is not yet connected.
- *         NOTE: provide an mq before a channel is connected?
- *               provide a callback to notify a client a channel connected?
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_get_mq (const struct GNUNET_CADET_Handle *h,
-                     const struct GNUNET_CADET_Channel *channel);
-
-/* NOTE:
- * GNUNET_CADET_channel_create and _destroy can stay the same.
- * Monitor API can stay the same (low-priority).
-
-struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
-                            void *channel_ctx,
-                            const struct GNUNET_PeerIdentity *peer,
-                            const struct GNUNET_HashCode *port,
-                            enum GNUNET_CADET_ChannelOption options);
-void
-GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel);
-
-*/
-
-/******************************************************************************/
-/******************************* MQ-BASED API *********************************/
-/******************************************************************************/
-
-
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_common.h b/src/include/gnunet_common.h
index fdcae66fa..bda011fb2 100644
--- a/src/include/gnunet_common.h
+++ b/src/include/gnunet_common.h
@@ -391,7 +391,7 @@ GNUNET_get_log_call_status (int caller_level,
  */
 void
 GNUNET_log_nocheck (enum GNUNET_ErrorType kind, const char *message, ...)
-  __attribute__ ((format (printf, 2, 3)));
+  __attribute__ ((format (gnu_printf, 2, 3)));
 
 /* from glib */
 #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)
diff --git a/src/include/gnunet_configuration_lib.h b/src/include/gnunet_configuration_lib.h
index 746dea61f..c1537e4f8 100644
--- a/src/include/gnunet_configuration_lib.h
+++ b/src/include/gnunet_configuration_lib.h
@@ -281,8 +281,8 @@ GNUNET_CONFIGURATION_get_value_number (const struct GNUNET_CONFIGURATION_Handle
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 int
-GNUNET_CONFIGURATION_get_value_float  (const struct GNUNET_CONFIGURATION_Handle
-                                       *cfg, const char *section,
+GNUNET_CONFIGURATION_get_value_float  (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                                       const char *section,
                                        const char *option,
                                        float *number);
 
@@ -454,9 +454,6 @@ GNUNET_CONFIGURATION_get_data (const struct GNUNET_CONFIGURATION_Handle *cfg,
                                size_t buf_size);
 
 
-
-
-
 /**
  * Expand an expression of the form "$FOO/BAR" to "DIRECTORY/BAR"
  * where either in the "PATHS" section or the environtment "FOO" is
diff --git a/src/include/gnunet_connection_lib.h b/src/include/gnunet_connection_lib.h
deleted file mode 100644
index e9dd95d1b..000000000
--- a/src/include/gnunet_connection_lib.h
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @author Christian Grothoff
- *
- * @file include/gnunet_connection_lib.h
- * Basic, low-level TCP networking interface
- *
- * @defgroup connection  Connection library
- * Basic, low-level TCP networking interface
- * @{
- */
-#ifndef GNUNET_CONNECTION_LIB_H
-#define GNUNET_CONNECTION_LIB_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "gnunet_network_lib.h"
-#include "gnunet_scheduler_lib.h"
-#include "gnunet_time_lib.h"
-
-/**
- * Timeout we use on TCP connect before trying another
- * result from the DNS resolver.  Actual value used
- * is this value divided by the number of address families.
- * Default is 5s.
- */
-#define GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
-
-/**
- * @brief handle for a network connection
- */
-struct GNUNET_CONNECTION_Handle;
-
-
-/**
- * Credentials for UNIX domain sockets.
- */
-struct GNUNET_CONNECTION_Credentials
-{
-  /**
-   * UID of the other end of the connection.
-   */
-  uid_t uid;
-
-  /**
-   * GID of the other end of the connection.
-   */
-  gid_t gid;
-};
-
-
-/**
- * Function to call for access control checks.
- *
- * @param cls closure
- * @param ucred credentials, if available, otherwise NULL
- * @param addr address
- * @param addrlen length of address
- * @return GNUNET_YES to allow, GNUNET_NO to deny, GNUNET_SYSERR
- *   for unknown address family (will be denied).
- */
-typedef int (*GNUNET_CONNECTION_AccessCheck) (void *cls,
-                                              const struct
-                                              GNUNET_CONNECTION_Credentials *
-                                              ucred,
-                                              const struct sockaddr * addr,
-                                              socklen_t addrlen);
-
-
-/**
- * Callback function for data received from the network.  Note that
- * both "available" and "err" would be 0 if the read simply timed out.
- *
- * @param cls closure
- * @param buf pointer to received data
- * @param available number of bytes availabe in "buf",
- *        possibly 0 (on errors)
- * @param addr address of the sender
- * @param addrlen size of addr
- * @param errCode value of errno (on errors receiving)
- */
-typedef void (*GNUNET_CONNECTION_Receiver) (void *cls, const void *buf,
-                                            size_t available,
-                                            const struct sockaddr * addr,
-                                            socklen_t addrlen, int errCode);
-
-/**
- * Set the persist option on this connection handle.  Indicates
- * that the underlying socket or fd should never really be closed.
- * Used for indicating process death.
- *
- * @param connection the connection to set persistent
- */
-void
-GNUNET_CONNECTION_persist_ (struct GNUNET_CONNECTION_Handle *connection);
-
-/**
- * Disable the "CORK" feature for communication with the given socket,
- * forcing the OS to immediately flush the buffer on transmission
- * instead of potentially buffering multiple messages.  Essentially
- * reduces the OS send buffers to zero.
- * Used to make sure that the last messages sent through the connection
- * reach the other side before the process is terminated.
- *
- * @param connection the connection to make flushing and blocking
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_CONNECTION_disable_corking (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param s socket to connect
- * @param serv_addr server address
- * @param addrlen length of server address
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_connect_socket (struct GNUNET_NETWORK_Handle *s,
-                                  const struct sockaddr *serv_addr,
-                                  socklen_t addrlen);
-
-
-/**
- * Create a connection handle by boxing an existing OS socket.  The OS
- * socket should henceforth be no longer used directly.
- * #GNUNET_CONNECTION_destroy() will close it.
- *
- * @param osSocket existing socket to box
- * @return the boxed socket handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_existing (struct GNUNET_NETWORK_Handle *osSocket);
-
-
-/**
- * Create a connection handle by accepting on a listen socket.  This
- * function may block if the listen socket has no connection ready.
- *
- * @param access_cb function to use to check if access is allowed
- * @param access_cb_cls closure for @a access_cb
- * @param lsock listen socket
- * @return the connection handle, NULL on error (for example, access refused)
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
-                                      void *access_cb_cls,
-                                      struct GNUNET_NETWORK_Handle *lsock);
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param cfg configuration to use
- * @param hostname name of the host to connect to
- * @param port port to connect to
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_connect (const struct GNUNET_CONFIGURATION_Handle
-                                       *cfg, const char *hostname,
-                                       uint16_t port);
-
-
-/**
- * Create a connection handle by connecting to a UNIX domain service.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates UNIX connections.
- *
- * @param cfg configuration to use
- * @param unixpath path to connect to)
- * @return the connection handle, NULL on systems without UNIX support
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_connect_to_unixpath (const struct
-                                                   GNUNET_CONFIGURATION_Handle
-                                                   *cfg, const char *unixpath);
-
-
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param af_family address family to use
- * @param serv_addr server address
- * @param addrlen length of server address
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_sockaddr (int af_family,
-                                        const struct sockaddr *serv_addr,
-                                        socklen_t addrlen);
-
-/**
- * Check if connection is valid (no fatal errors have happened so far).
- * Note that a connection that is still trying to connect is considered
- * valid.
- *
- * @param connection handle to check
- * @return GNUNET_YES if valid, GNUNET_NO otherwise
- */
-int
-GNUNET_CONNECTION_check (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Obtain the network address of the other party.
- *
- * @param connection the client to get the address for
- * @param addr where to store the address
- * @param addrlen where to store the length of the address
- * @return GNUNET_OK on success
- */
-int
-GNUNET_CONNECTION_get_address (struct GNUNET_CONNECTION_Handle *connection,
-                               void **addr, size_t * addrlen);
-
-
-/**
- * Close the connection and free associated resources.  There must
- * not be any pending requests for reading or writing to the
- * connection at this time.
- *
- * @param connection connection to destroy
- */
-void
-GNUNET_CONNECTION_destroy (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Receive data from the given connection.  Note that this function will
- * call "receiver" asynchronously using the scheduler.  It will
- * "immediately" return.  Note that there MUST only be one active
- * receive call per connection at any given point in time (so do not
- * call receive again until the receiver callback has been invoked).
- *
- * @param connection connection handle
- * @param max maximum number of bytes to read
- * @param timeout maximum amount of time to wait
- * @param receiver function to call with received data
- * @param receiver_cls closure for receiver
- */
-void
-GNUNET_CONNECTION_receive (struct GNUNET_CONNECTION_Handle *connection, size_t max,
-                           struct GNUNET_TIME_Relative timeout,
-                           GNUNET_CONNECTION_Receiver receiver,
-                           void *receiver_cls);
-
-
-/**
- * Cancel receive job on the given connection.  Note that the
- * receiver callback must not have been called yet in order
- * for the cancellation to be valid.
- *
- * @param connection connection handle
- * @return closure of the original receiver callback closure
- */
-void *
-GNUNET_CONNECTION_receive_cancel (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Function called to notify a client about the connection begin ready
- * to queue more data.  @a buf will be NULL and @a size zero if the
- * connection was closed for writing in the meantime.
- *
- * @param cls closure
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to @a buf
- */
-typedef size_t
-(*GNUNET_CONNECTION_TransmitReadyNotify) (void *cls,
-                                          size_t size,
-                                          void *buf);
-
-
-/**
- * Opaque handle that can be used to cancel
- * a transmit-ready notification.
- */
-struct GNUNET_CONNECTION_TransmitHandle;
-
-/**
- * Ask the connection to call us once the specified number of bytes
- * are free in the transmission buffer.  Will never call the @a notify
- * callback in this task, but always first go into the scheduler.  Note that
- * this function will abort if "size" is greater than
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE.
- *
- * Note that "notify" will be called either when enough
- * buffer space is available OR when the connection is destroyed.
- * The size parameter given to notify is guaranteed to be
- * larger or equal to size if the buffer is ready, or zero
- * if the connection was destroyed (or at least closed for
- * writing).  Finally, any time before 'notify' is called, a
- * client may call "notify_transmit_ready_cancel" to cancel
- * the transmission request.
- *
- * Only one transmission request can be scheduled at the same
- * time.  Notify will be run with the same scheduler priority
- * as that of the caller.
- *
- * @param connection connection
- * @param size number of bytes to send
- * @param timeout after how long should we give up (and call
- *        notify with buf NULL and size 0)?
- * @param notify function to call when buffer space is available
- * @param notify_cls closure for notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we are already going to notify someone else (busy)
- */
-struct GNUNET_CONNECTION_TransmitHandle *
-GNUNET_CONNECTION_notify_transmit_ready (struct GNUNET_CONNECTION_Handle *connection,
-                                         size_t size,
-                                         struct GNUNET_TIME_Relative timeout,
-                                         GNUNET_CONNECTION_TransmitReadyNotify
-                                         notify, void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready
- * notification.
- *
- * @param th handle for notification to cancel
- */
-void
-GNUNET_CONNECTION_notify_transmit_ready_cancel (struct
-                                                GNUNET_CONNECTION_TransmitHandle
-                                                *th);
-
-
-/**
- * Create a connection to be proxied using a given connection.
- *
- * @param cph connection to proxy server
- * @return connection to be proxied
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_proxied_from_handshake (struct GNUNET_CONNECTION_Handle *cph);
-
-
-/**
- * Activate proxied connection and destroy initial proxy handshake connection.
- * There must not be any pending requests for reading or writing to the
- * proxy hadshake connection at this time.
- *
- * @param proxied connection connection to proxy server
- */
-void
-GNUNET_CONNECTION_acivate_proxied (struct GNUNET_CONNECTION_Handle *proxied);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CONNECTION_LIB_H */
-#endif
-
-/** @} */  /* end of group */
-
-/* end of gnunet_connection_lib.h */
diff --git a/src/include/gnunet_constants.h b/src/include/gnunet_constants.h
index 1d0232cea..1c48298a0 100644
--- a/src/include/gnunet_constants.h
+++ b/src/include/gnunet_constants.h
@@ -40,6 +40,8 @@ extern "C"
 #endif
 #endif
 
+
+
 /**
  * Bandwidth (in/out) to assume initially (before either peer has
  * communicated any particular preference).  Should be rather low; set
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h
index f3aaa943b..c77d82fd3 100644
--- a/src/include/gnunet_container_lib.h
+++ b/src/include/gnunet_container_lib.h
@@ -2073,6 +2073,59 @@ GNUNET_CONTAINER_multihashmap32_iterator_destroy (struct GNUNET_CONTAINER_MultiH
 
 
 
+/**
+ * Insertion sort of @a element into DLL from @a head to @a tail
+ * sorted by @a comparator.
+ *
+ * @param TYPE element type of the elements, i.e. `struct ListElement`
+ * @param comparator function like memcmp() to compare elements; takes
+ *                   three arguments, the @a comparator_cls and two elements,
+ *                   returns an `int` (-1, 0 or 1)
+ * @param comparator_cls closure for @a comparator
+ * @param[in,out] head head of DLL
+ * @param[in,out] tail tail of DLL
+ * @param element element to insert
+ */
+#define GNUNET_CONTAINER_DLL_insert_sorted(TYPE,comparator,comparator_cls,head,tail,element) do { \
+  if ( (NULL == head) || \
+       (0 < comparator (comparator_cls, \
+                        element, \
+                        head)) ) \
+  { \
+    /* insert at head, element < head */ \
+    GNUNET_CONTAINER_DLL_insert (head,                                \
+                                 tail, \
+                                 element); \
+  } \
+  else \
+  {          \
+    TYPE *pos; \
+    \
+    for (pos = head; \
+         NULL != pos; \
+         pos = pos->next) \
+      if (0 < \
+          comparator (comparator_cls, \
+                      element, \
+                      pos)) \
+        break; /* element < pos */ \
+    if (NULL == pos) /* => element > tail */ \
+    { \
+      GNUNET_CONTAINER_DLL_insert_tail (head,                             \
+                                        tail, \
+                                        element); \
+    } \
+    else /* prev < element < pos */ \
+    { \
+      GNUNET_CONTAINER_DLL_insert_after (head, \
+                                         tail, \
+                                         pos->prev, \
+                                         element); \
+    } \
+  } \
+} while (0)
+
+
 /* ******************** Heap *************** */
 
 
diff --git a/src/include/gnunet_core_service.h b/src/include/gnunet_core_service.h
index 8136770b7..ace223c11 100644
--- a/src/include/gnunet_core_service.h
+++ b/src/include/gnunet_core_service.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2014, 2016 GNUnet e.V.
+     Copyright (C) 2009-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -197,57 +197,6 @@ GNUNET_CORE_get_mq (const struct GNUNET_CORE_Handle *h,
 
 
 /**
- * Handle for a transmission request.
- */
-struct GNUNET_CORE_TransmitHandle;
-
-
-/**
- * Ask the core to call @a notify once it is ready to transmit the
- * given number of bytes to the specified @a target.  Must only be
- * called after a connection to the respective peer has been
- * established (and the client has been informed about this).  You may
- * have one request of this type pending for each connected peer at
- * any time.  If a peer disconnects, the application MUST call
- * #GNUNET_CORE_notify_transmit_ready_cancel() on the respective
- * transmission request, if one such request is pending.
- *
- * @param handle connection to core service
- * @param cork is corking allowed for this transmission?
- * @param priority how important is the message?
- * @param maxdelay how long can the message wait? Only effective if @a cork is #GNUNET_YES
- * @param target who should receive the message, never NULL (can be this peer's identity for loopback)
- * @param notify_size how many bytes of buffer space does @a notify want?
- * @param notify function to call when buffer space is available;
- *        will be called with NULL on timeout; clients MUST cancel
- *        all pending transmission requests DURING the disconnect
- *        handler
- * @param notify_cls closure for @a notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we can not even queue the request (request already pending);
- *         if NULL is returned, @a notify will NOT be called.
- */
-struct GNUNET_CORE_TransmitHandle *
-GNUNET_CORE_notify_transmit_ready (struct GNUNET_CORE_Handle *handle,
-                                   int cork,
-                                   enum GNUNET_CORE_Priority priority,
-                                   struct GNUNET_TIME_Relative maxdelay,
-                                   const struct GNUNET_PeerIdentity *target,
-                                   size_t notify_size,
-                                   GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                   void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready notification.
- *
- * @param th handle that was returned by #GNUNET_CORE_notify_transmit_ready().
- */
-void
-GNUNET_CORE_notify_transmit_ready_cancel (struct GNUNET_CORE_TransmitHandle *th);
-
-
-/**
  * Handle to a CORE monitoring operation.
  */
 struct GNUNET_CORE_MonitorHandle;
diff --git a/src/include/gnunet_credential_service.h b/src/include/gnunet_credential_service.h
index d04c0a253..6c9b477ea 100644
--- a/src/include/gnunet_credential_service.h
+++ b/src/include/gnunet_credential_service.h
@@ -262,8 +262,6 @@ typedef void (*GNUNET_CREDENTIAL_RemoveDelegateResultProcessor) (void *cls,
                                                                  uint32_t success);
 
 
-
-
 /**
  * Performs attribute verification.
  * Checks if there is a delegation chain from
diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h
index 43fd32a58..07cade0e3 100644
--- a/src/include/gnunet_crypto_lib.h
+++ b/src/include/gnunet_crypto_lib.h
@@ -1240,6 +1240,17 @@ GNUNET_CRYPTO_eddsa_key_create (void);
 
 /**
  * @ingroup crypto
+ * Create a new private key.  Clear with #GNUNET_CRYPTO_ecdhe_key_clear().
+ *
+ * @param[out] pk set to fresh private key;
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+int
+GNUNET_CRYPTO_ecdhe_key_create2 (struct GNUNET_CRYPTO_EcdhePrivateKey *pk);
+
+
+/**
+ * @ingroup crypto
  * Create a new private key. Caller must free return value.
  *
  * @return fresh private key; free using #GNUNET_free
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index 71c69ffaf..516ba525c 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -93,6 +93,7 @@ struct GNUNET_DATASTORE_PluginEnvironment
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum
  * @return #GNUNET_OK to keep the item
@@ -100,14 +101,15 @@ struct GNUNET_DATASTORE_PluginEnvironment
  */
 typedef int
 (*PluginDatumProcessor) (void *cls,
-                         const struct GNUNET_HashCode *key,
-                         uint32_t size,
-                         const void *data,
-                         enum GNUNET_BLOCK_Type type,
-                         uint32_t priority,
-                         uint32_t anonymity,
-                         struct GNUNET_TIME_Absolute expiration,
-                         uint64_t uid);
+                         const struct GNUNET_HashCode *key,
+                         uint32_t size,
+                         const void *data,
+                         enum GNUNET_BLOCK_Type type,
+                         uint32_t priority,
+                         uint32_t anonymity,
+                         uint32_t replication,
+                         struct GNUNET_TIME_Absolute expiration,
+                         uint64_t uid);
 
 
 /**
@@ -204,9 +206,9 @@ typedef void
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
- * @param key key to match, never NULL
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
+ * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
  *        Note that for DBlocks there is no difference
@@ -215,17 +217,18 @@ typedef void
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
  * @param proc function to call on the matching value;
- *        proc should be called with NULL if there is no result
+ *        will be called with NULL if nothing matches
  * @param proc_cls closure for @a proc
  */
 typedef void
 (*PluginGetKey) (void *cls,
-                 uint64_t offset,
-                 const struct GNUNET_HashCode *key,
-                 const struct GNUNET_HashCode *vhash,
-                 enum GNUNET_BLOCK_Type type,
-                 PluginDatumProcessor proc,
-                 void *proc_cls);
+                 uint64_t next_uid,
+                 bool random,
+                 const struct GNUNET_HashCode *key,
+                 const struct GNUNET_HashCode *vhash,
+                 enum GNUNET_BLOCK_Type type,
+                 PluginDatumProcessor proc,
+                 void *proc_cls);
 
 
 /**
@@ -258,19 +261,18 @@ typedef void
 
 
 /**
- * Update the priority for a particular key in the datastore.  If
- * the expiration time in value is different than the time found in
- * the datastore, the higher value should be kept.  For the
- * anonymity level, the lower value is to be used.  The specified
- * priority should be added to the existing priority, ignoring the
- * priority in value.
+ * Update the priority, replication and expiration for a particular
+ * unique ID in the datastore.  If the expiration time in value is
+ * different than the time found in the datastore, the higher value
+ * should be kept.  The specified priority and replication is added
+ * to the existing value.
  *
  * @param cls closure
  * @param uid unique identifier of the datum
- * @param delta by how much should the priority
- *     change?  If priority + delta < 0 the
- *     priority should be set to 0 (never go
- *     negative).
+ * @param priority by how much should the priority
+ *     change?
+ * @param replication by how much should the replication
+ *     change?
  * @param expire new expiration time should be the
  *     MAX of any existing expiration time and
  *     this value
@@ -279,31 +281,31 @@ typedef void
  */
 typedef void
 (*PluginUpdate) (void *cls,
-                 uint64_t uid,
-                 int delta,
-                 struct GNUNET_TIME_Absolute expire,
-                 PluginUpdateCont cont,
-                 void *cont_cls);
+                 uint64_t uid,
+                 uint32_t priority,
+                 uint32_t replication,
+                 struct GNUNET_TIME_Absolute expire,
+                 PluginUpdateCont cont,
+                 void *cont_cls);
 
 
 /**
- * Select a single item from the datastore at the specified offset
- * (among those applicable).
+ * Select a single item from the datastore (among those applicable).
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
  *        Must not be zero (ANY).
- * @param proc function to call on the matching value
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 typedef void
 (*PluginGetType) (void *cls,
-                  uint64_t offset,
-                  enum GNUNET_BLOCK_Type type,
-                  PluginDatumProcessor proc,
-                  void *proc_cls);
+                  uint64_t next_uid,
+                  enum GNUNET_BLOCK_Type type,
+                  PluginDatumProcessor proc,
+                  void *proc_cls);
 
 
 /**
@@ -356,9 +358,6 @@ struct GNUNET_DATASTORE_PluginFunctions
 
   /**
    * Get datum (of the specified type) with anonymity level zero.
-   * This function is allowed to ignore the 'offset' argument
-   * and instead return a random result (with zero anonymity of
-   * the correct type) if implementing an offset is expensive.
    */
   PluginGetType get_zero_anonymity;
 
diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h
index f594d8fa6..f851385c5 100644
--- a/src/include/gnunet_datastore_service.h
+++ b/src/include/gnunet_datastore_service.h
@@ -201,33 +201,6 @@ GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
 
 
 /**
- * Update a value in the datastore.
- *
- * @param h handle to the datastore
- * @param uid identifier for the value
- * @param priority how much to increase the priority of the value
- * @param expiration new expiration value should be MAX of existing and this argument
- * @param queue_priority ranking of this request in the priority queue
- * @param max_queue_size at what queue size should this request be dropped
- *        (if other requests of higher priority are in the queue)
- * @param cont continuation to call when done
- * @param cont_cls closure for @a cont
- * @return NULL if the entry was not queued, otherwise a handle that can be used to
- *         cancel; note that even if NULL is returned, the callback will be invoked
- *         (or rather, will already have been invoked)
- */
-struct GNUNET_DATASTORE_QueueEntry *
-GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
-                         uint64_t uid,
-                         uint32_t priority,
-                         struct GNUNET_TIME_Absolute expiration,
-                         unsigned int queue_priority,
-                         unsigned int max_queue_size,
-                         GNUNET_DATASTORE_ContinuationWithStatus cont,
-                         void *cont_cls);
-
-
-/**
  * Explicitly remove some content from the database.  @a cont will be
  * called with status #GNUNET_OK if content was removed, #GNUNET_NO if
  * no matching entry was found and #GNUNET_SYSERR on all other types
@@ -267,6 +240,7 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
  * @param type type of the content
  * @param priority priority of the content
  * @param anonymity anonymity-level for the content
+ * @param replication how often should the content be replicated to other peers?
  * @param expiration expiration time for the content
  * @param uid unique identifier for the datum;
  *        maybe 0 if no unique identifier is available
@@ -279,6 +253,7 @@ typedef void
                                     enum GNUNET_BLOCK_Type type,
                                     uint32_t priority,
                                     uint32_t anonymity,
+                                    uint32_t replication,
                                     struct GNUNET_TIME_Absolute expiration,
                                     uint64_t uid);
 
@@ -288,10 +263,8 @@ typedef void
  * will only be called once.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param type desired type, 0 for any
  * @param queue_priority ranking of this request in the priority queue
@@ -305,7 +278,8 @@ typedef void
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
-                          uint64_t offset,
+                          uint64_t next_uid,
+                          bool random,
                           const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           unsigned int queue_priority,
@@ -316,16 +290,9 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
 
 /**
  * Get a single zero-anonymity value from the datastore.
- * Note that some implementations can ignore the 'offset' and
- * instead return a random zero-anonymity value.  In that case,
- * detecting the wrap-around based on a repeating UID is at best
- * probabilistic.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param queue_priority ranking of this request in the priority queue
  * @param max_queue_size at what queue size should this request be dropped
  *        (if other requests of higher priority are in the queue)
@@ -339,7 +306,7 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
-                                     uint64_t offset,
+                                     uint64_t next_uid,
                                      unsigned int queue_priority,
                                      unsigned int max_queue_size,
                                      enum GNUNET_BLOCK_Type type,
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index a9c7e8944..ac418072e 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -438,23 +438,36 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
 /* ******************** command-line option parsing API *********************** */
 
 /**
- * Command-line option parser function that allows the user
- * to specify one or more '-k' options with keywords.  Each
- * specified keyword will be added to the URI.  A pointer to
- * the URI must be passed as the "scls" argument.
- *
- * @param ctx command line processor context
- * @param scls must be of type "struct GNUNET_FS_Uri **"
- * @param option name of the option (typically 'k')
- * @param value command line argument given
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                               void *scls,
-                               const char *option,
-                               const char *value);
-
+ * Allow user to specify keywords.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] topKeywords set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_KEYWORDS (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_Uri **topKeywords);
+
+/**
+ * Allow user to specify metadata.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] metadata set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_METADATA (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_CONTAINER_MetaData **meta);
 
 /**
  * Command-line option parser function that allows the user to specify
diff --git a/src/include/gnunet_getopt_lib.h b/src/include/gnunet_getopt_lib.h
index b04020a70..f707bb091 100644
--- a/src/include/gnunet_getopt_lib.h
+++ b/src/include/gnunet_getopt_lib.h
@@ -97,6 +97,7 @@ typedef int
                                              const char *option,
                                              const char *value);
 
+
 /**
  * @brief Definition of a command line option.
  */
@@ -130,255 +131,301 @@ struct GNUNET_GETOPT_CommandLineOption
   int require_argument;
 
   /**
+   * Is the presence of this option mandatory?
+   */
+  int option_mandatory;
+
+  /**
    * Handler for the option.
    */
   GNUNET_GETOPT_CommandLineOptionProcessor processor;
 
   /**
+   * Function to call on @e scls to clean up after processing all
+   * the arguments. Can be NULL.
+   */
+  void (*cleaner)(void *cls);
+
+  /**
    * Specific closure to pass to the processor.
    */
   void *scls;
 
 };
 
+
 /**
- * Macro defining the option to print the command line
+ * Defining the option to print the command line
  * help text (-h option).
  *
  * @param about string with brief description of the application
  */
-#define GNUNET_GETOPT_OPTION_HELP(about) \
-  { 'h', "help", (const char *) NULL, gettext_noop("print this help"), 0, &GNUNET_GETOPT_format_help_, (void *) about }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_help (const char *about);
 
 
 /**
- * Macro defining the option to print the version of
+ * Define the option to print the version of
  * the application (-v option)
  *
  * @param version string with the version number
  */
-#define GNUNET_GETOPT_OPTION_VERSION(version) \
-  { 'v', "version", (const char *) NULL, gettext_noop("print the version number"), 0, &GNUNET_GETOPT_print_version_, (void *) version }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_version (const char *version);
+
 
 
 /**
  * Allow user to specify log file name (-l option)
  *
- * @param logfn set to the name of the logfile
+ * @param[out] logfn set to the name of the logfile
  */
-#define GNUNET_GETOPT_OPTION_LOGFILE(logfn)                             \
-  { 'l', "logfile", "LOGFILE", gettext_noop("configure logging to write logs to LOGFILE"), 1, &GNUNET_GETOPT_set_string, (void *) logfn }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_logfile (char **logfn);
 
 
 /**
- * Allow user to specify log level (-L option)
+ * Allow user to specify a string.
  *
- * @param loglev set to the log level
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
  */
-#define GNUNET_GETOPT_OPTION_LOGLEVEL(loglev)                           \
-  { 'L', "log", "LOGLEVEL", gettext_noop("configure logging to use LOGLEVEL"), 1, &GNUNET_GETOPT_set_string, (void *) loglev }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_string (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str);
+
+/**
+ * Allow user to specify a filename (automatically path expanded).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_filename (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               char **str);
 
 
 /**
- * Get number of verbose (-V) flags
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding.
  *
- * @param level where to store the verbosity level (should be an 'int')
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument
+ * @param val_size size of @a val in bytes
  */
-#define GNUNET_GETOPT_OPTION_VERBOSE(level)                             \
-  { 'V', "verbose", (const char *) NULL, gettext_noop("be verbose"), 0, &GNUNET_GETOPT_increment_value, (void *) level }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_base32_fixed_size (char shortName,
+                                            const char *name,
+                                            const char *argumentHelp,
+                                            const char *description,
+                                            void *val,
+                                            size_t val_size);
 
 
 /**
- * Get configuration file name (-c option)
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding where the size of the binary value is
+ * automatically determined from its type.
  *
- * @param fn set to the configuration file name
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument;
+ *             size is determined by type (sizeof (*val)).
  */
-#define GNUNET_GETOPT_OPTION_CFG_FILE(fn)                               \
-  { 'c', "config", "FILENAME", gettext_noop("use configuration file FILENAME"), 1, &GNUNET_GETOPT_set_string, (void *) fn }
+#define GNUNET_GETOPT_option_base32_auto(shortName,name,argumentHelp,description,val) \
+  GNUNET_GETOPT_option_base32_fixed_size(shortName,name,argumentHelp,description,val,sizeof(*val))
 
 
 /**
- * Marker for the end of the list of options.
+ * Allow user to specify a flag (which internally means setting
+ * an integer to 1/#GNUNET_YES/#GNUNET_OK.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
  */
-#define GNUNET_GETOPT_OPTION_END \
-  { '\0', NULL, NULL, NULL, 0, NULL, NULL }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_flag (char shortName,
+                              const char *name,
+                              const char *description,
+                              int *val);
 
 
 /**
- * Parse the command line.
+ * Allow user to specify an `unsigned int`.
  *
- * @param binaryOptions Name of application with option summary
- * @param allOptions defined options and handlers
- * @param argc number of arguments in @a argv
- * @param argv actual arguments
- * @return index into argv with first non-option
- *   argument, or #GNUNET_SYSERR on error
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
  */
-int
-GNUNET_GETOPT_run (const char *binaryOptions,
-                   const struct GNUNET_GETOPT_CommandLineOption *allOptions,
-                   unsigned int argc, char *const *argv);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_uint (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               unsigned int *val);
 
 
 /**
- * Set an option of type 'unsigned long long' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `unsigned long long`.
+ * Allow user to specify an `unsigned long long`.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'unsigned long long')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
  */
-int
-GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                         void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_ulong (char shortName,
+                                const char *name,
+                                const char *argumentHelp,
+                                const char *description,
+                                unsigned long long *val);
 
 
 /**
- * Set an option of type 'struct GNUNET_TIME_Relative' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `struct GNUNET_TIME_Relative`.
+ * Allow user to specify a `struct GNUNET_TIME_Relative`
+ * (using human-readable "fancy" time).
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'struct GNUNET_TIME_Relative')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
  */
-int
-GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                                 void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_relative_time (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Relative *val);
 
 
 /**
- * Set an option of type 'unsigned int' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `unsigned int`.
+ * Allow user to specify a `struct GNUNET_TIME_Absolute`
+ * (using human-readable "fancy" time).
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'unsigned int')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
  */
-int
-GNUNET_GETOPT_set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                        void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_absolute_time (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Absolute *val);
 
 
 /**
- * Set an option of type 'int' from the command line to 1 if the
- * given option is present.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `int`.
+ * Increment @a val each time the option flag is given by one.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the `int`)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_OK (always)
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
  */
-int
-GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                       void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_increment_uint (char shortName,
+                                      const char *name,
+                                      const char *description,
+                                      unsigned int *val);
 
 
 /**
- * Set an option of type 'char *' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `char *`, which will be allocated with the requested string.
+ * Define the '-L' log level option.  Note that we do not check
+ * that the log level is valid here.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the `char *`,
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a string)
- * @return #GNUNET_OK (always)
+ * @param[out] level set to the log level
  */
-int
-GNUNET_GETOPT_set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_loglevel (char **level);
 
 
 /**
- * Set an option of type 'char *' from the command line doing fs expansion.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'char *', which will be allocated with the requested string.
+ * Define the '-V' verbosity option.  Using the option more
+ * than once increments @a level each time.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'char *',
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a string)
- * @return #GNUNET_OK (always)
+ * @param[out] level set to the verbosity level
  */
-int
-GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_verbose (unsigned int *level);
+
 
 /**
- * Set an option of type 'unsigned int' from the command line. Each
- * time the option flag is given, the value is incremented by one.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'int'.
+ * Allow user to specify log file name (-l option)
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'int')
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_OK (always)
+ * @param[out] logfn set to the name of the logfile
  */
-int
-GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_logfile (char **logfn);
 
 
-/* *************** internal prototypes - use macros above! ************* */
+/**
+ * Allow user to specify configuration file name (-c option)
+ *
+ * @param[out] fn set to the name of the configuration file
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_cfgfile (char **fn);
+
 
 /**
- * Print out details on command line options (implements --help).
+ * Make the given option mandatory.
  *
- * @param ctx command line processing context
- * @param scls additional closure (points to about text)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_NO (do not continue, not an error)
+ * @param opt option to modify
+ * @return @a opt with the mandatory flag set.
  */
-int
-GNUNET_GETOPT_format_help_ (struct GNUNET_GETOPT_CommandLineProcessorContext
-                            *ctx, void *scls, const char *option,
-                            const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_mandatory (struct GNUNET_GETOPT_CommandLineOption opt);
+
 
 /**
- * Print out program version (implements --version).
+ * Marker for the end of the list of options.
+ */
+#define GNUNET_GETOPT_OPTION_END \
+  { '\0', NULL, NULL, NULL, 0, 0, NULL, NULL, NULL }
+
+
+/**
+ * Parse the command line.
  *
- * @param ctx command line processing context
- * @param scls additional closure (points to version string)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_NO (do not continue, not an error)
+ * @param binaryOptions Name of application with option summary
+ * @param allOptions defined options and handlers
+ * @param argc number of arguments in @a argv
+ * @param argv actual arguments
+ * @return index into argv with first non-option
+ *   argument, or #GNUNET_SYSERR on error
  */
 int
-GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext
-                              *ctx, void *scls, const char *option,
-                              const char *value);
+GNUNET_GETOPT_run (const char *binaryOptions,
+                   const struct GNUNET_GETOPT_CommandLineOption *allOptions,
+                   unsigned int argc,
+                   char *const *argv);
+
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_gns_service.h b/src/include/gnunet_gns_service.h
index 8a1099444..ccc868c05 100644
--- a/src/include/gnunet_gns_service.h
+++ b/src/include/gnunet_gns_service.h
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2012-2014 GNUnet e.V.
+      Copyright (C) 2012-2014, 2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -90,19 +90,10 @@ GNUNET_GNS_disconnect (struct GNUNET_GNS_Handle *handle);
  * @param rd_count number of records in @a rd
  * @param rd the records in reply
  */
-typedef void (*GNUNET_GNS_LookupResultProcessor) (void *cls,
-                                                  uint32_t rd_count,
-                                                  const struct GNUNET_GNSRECORD_Data *rd);
-
-/**
- * Iterator called on obtained result for a GNS lookup.
- *
- * @param cls closure
- * @param rd_count number of records in @a rd
- * @param rd the records in reply
- */
-typedef void (*GNUNET_GNS_ReverseLookupResultProcessor) (void *cls,
-                                                  const char* name);
+typedef void
+(*GNUNET_GNS_LookupResultProcessor) (void *cls,
+                                     uint32_t rd_count,
+                                     const struct GNUNET_GNSRECORD_Data *rd);
 
 
 /**
@@ -137,13 +128,8 @@ enum GNUNET_GNS_LocalOptions
  * @param zone zone to look in
  * @param type the GNS record type to look for
  * @param options local options for the lookup
- * @param shorten_zone_key the private key of the shorten zone (can be NULL);
- *                    specify to enable automatic shortening (given a PSEU
- *                    record, if a given pseudonym is not yet used in the
- *                    shorten zone, we automatically add the respective zone
- *                    under that name)
  * @param proc function to call on result
- * @param proc_cls closure for processor
+ * @param proc_cls closure for @a proc
  * @return handle to the queued request
  */
 struct GNUNET_GNS_LookupRequest *
@@ -152,27 +138,9 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
                    const struct GNUNET_CRYPTO_EcdsaPublicKey *zone,
                    uint32_t type,
                    enum GNUNET_GNS_LocalOptions options,
-                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *shorten_zone_key,
                    GNUNET_GNS_LookupResultProcessor proc,
                    void *proc_cls);
 
-/**
- * Perform an asynchronous reverse lookup operation on the GNS.
- *
- * @param handle handle to the GNS service
- * @param zone_key zone to find a name for
- * @param root_key our zone
- * @param proc processor to call on result
- * @param proc_cls closure for @a proc
- * @return handle to the request
- */
-struct GNUNET_GNS_ReverseLookupRequest*
-GNUNET_GNS_reverse_lookup (struct GNUNET_GNS_Handle *handle,
-                           const struct GNUNET_CRYPTO_EcdsaPublicKey *zone_key,
-                           const struct GNUNET_CRYPTO_EcdsaPublicKey *root_key,
-                           GNUNET_GNS_ReverseLookupResultProcessor proc,
-                           void *proc_cls);
-
 
 /**
  * Cancel pending lookup request
diff --git a/src/include/gnunet_helper_lib.h b/src/include/gnunet_helper_lib.h
index db0ca98aa..60b3ff681 100644
--- a/src/include/gnunet_helper_lib.h
+++ b/src/include/gnunet_helper_lib.h
@@ -38,7 +38,8 @@
 #define GNUNET_HELPER_LIB_H
 
 #include "gnunet_scheduler_lib.h"
-#include "gnunet_server_lib.h"
+#include "gnunet_mst_lib.h"
+
 
 /**
  * The handle to a helper process.
@@ -52,7 +53,8 @@ struct GNUNET_HELPER_Handle;
  *
  * @param cls the closure from GNUNET_HELPER_start()
  */
-typedef void (*GNUNET_HELPER_ExceptionCallback) (void *cls);
+typedef void
+(*GNUNET_HELPER_ExceptionCallback) (void *cls);
 
 
 /**
@@ -75,7 +77,7 @@ struct GNUNET_HELPER_Handle *
 GNUNET_HELPER_start (int with_control_pipe,
                      const char *binary_name,
                      char *const binary_argv[],
-                     GNUNET_SERVER_MessageTokenizerCallback cb,
+                     GNUNET_MessageTokenizerCallback cb,
                      GNUNET_HELPER_ExceptionCallback exp_cb,
                      void *cb_cls);
 
diff --git a/src/include/gnunet_json_lib.h b/src/include/gnunet_json_lib.h
index 9e5f9e284..ce721d8d8 100644
--- a/src/include/gnunet_json_lib.h
+++ b/src/include/gnunet_json_lib.h
@@ -427,6 +427,26 @@ void
 GNUNET_JSON_post_parser_cleanup (void *con_cls);
 
 
+/* ****************** GETOPT JSON helper ******************* */
+
+
+/**
+ * Allow user to specify a JSON input value.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the JSON specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_JSON_getopt (char shortName,
+                    const char *name,
+                    const char *argumentHelp,
+                    const char *description,
+                    json_t **json);
+
+
 #endif
 
 /* end of gnunet_json_lib.h */
diff --git a/src/include/gnunet_mq_lib.h b/src/include/gnunet_mq_lib.h
index 108ba5d54..ecee1b223 100644
--- a/src/include/gnunet_mq_lib.h
+++ b/src/include/gnunet_mq_lib.h
@@ -32,9 +32,10 @@
  *
  * @{
  */
-#ifndef GNUNET_MQ_H
-#define GNUNET_MQ_H
+#ifndef GNUNET_MQ_LIB_H
+#define GNUNET_MQ_LIB_H
 
+#include "gnunet_scheduler_lib.h"
 
 /**
  * Allocate an envelope, with extra space allocated after the space needed
@@ -269,6 +270,66 @@ typedef void
 
 
 /**
+ * Insert @a env into the envelope DLL starting at @a env_head
+ * Note that @a env must not be in any MQ while this function
+ * is used with DLLs defined outside of the MQ module.  This
+ * is just in case some application needs to also manage a
+ * FIFO of envelopes independent of MQ itself and wants to
+ * re-use the pointers internal to @a env.  Use with caution.
+ *
+ * @param[in|out] env_head of envelope DLL
+ * @param[in|out] env_tail tail of envelope DLL
+ * @param[in|out] env element to insert at the tail
+ */
+void
+GNUNET_MQ_dll_insert_tail (struct GNUNET_MQ_Envelope **env_head,
+                           struct GNUNET_MQ_Envelope **env_tail,
+                           struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * Remove @a env from the envelope DLL starting at @a env_head.
+ * Note that @a env must not be in any MQ while this function
+ * is used with DLLs defined outside of the MQ module. This
+ * is just in case some application needs to also manage a
+ * FIFO of envelopes independent of MQ itself and wants to
+ * re-use the pointers internal to @a env.  Use with caution.
+ *
+ * @param[in|out] env_head of envelope DLL
+ * @param[in|out] env_tail tail of envelope DLL
+ * @param[in|out] env element to remove from the DLL
+ */
+void
+GNUNET_MQ_dll_remove (struct GNUNET_MQ_Envelope **env_head,
+                      struct GNUNET_MQ_Envelope **env_tail,
+                      struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * Copy an array of handlers.
+ *
+ * Useful if the array has been delared in local memory and needs to be
+ * persisted for future use.
+ *
+ * @param handlers Array of handlers to be copied.
+ * @return A newly allocated array of handlers.
+ *         Needs to be freed with #GNUNET_free.
+ */
+struct GNUNET_MQ_MessageHandler *
+GNUNET_MQ_copy_handlers (const struct GNUNET_MQ_MessageHandler *handlers);
+
+
+/**
+ * Count the handlers in a handler array.
+ *
+ * @param handlers Array of handlers to be counted.
+ * @return The number of handlers in the array.
+ */
+unsigned int
+GNUNET_MQ_count_handlers (const struct GNUNET_MQ_MessageHandler *handlers);
+
+
+/**
  * Message handler for a specific message type.
  */
 struct GNUNET_MQ_MessageHandler
@@ -413,7 +474,6 @@ struct GNUNET_MQ_MessageHandler
  */
 struct GNUNET_MQ_Envelope *
 GNUNET_MQ_msg_ (struct GNUNET_MessageHeader **mhp,
-
                 uint16_t size,
                 uint16_t type);
 
@@ -451,6 +511,17 @@ GNUNET_MQ_get_current_envelope (struct GNUNET_MQ_Handle *mq);
 
 
 /**
+ * Function to copy an envelope.  The envelope must not yet
+ * be in any queue or have any options or callbacks set.
+ *
+ * @param env envelope to copy
+ * @return copy of @a env
+ */
+struct GNUNET_MQ_Envelope *
+GNUNET_MQ_env_copy (struct GNUNET_MQ_Envelope *env);
+
+
+/**
  * Function to obtain the last envelope in the queue.
  *
  * @param mq message queue to interrogate
@@ -488,6 +559,17 @@ GNUNET_MQ_env_get_options (struct GNUNET_MQ_Envelope *env,
 
 
 /**
+ * Remove the first envelope that has not yet been sent from the message
+ * queue and return it.
+ *
+ * @param mq queue to remove envelope from
+ * @return NULL if queue is empty (or has no envelope that is not under transmission)
+ */
+struct GNUNET_MQ_Envelope *
+GNUNET_MQ_unsent_head (struct GNUNET_MQ_Handle *mq);
+
+
+/**
  * Set application-specific options for this queue.
  *
  * @param mq message queue to set options for
@@ -717,7 +799,7 @@ GNUNET_MQ_impl_send_continue (struct GNUNET_MQ_Handle *mq);
  * try to send the next message until #gnunet_mq_impl_send_continue
  * is called.
  *
- * only useful for implementing message queues, results in undefined
+ * Only useful for implementing message queues, results in undefined
  * behavior if not used carefully.
  *
  * @param mq message queue to send the next message with
diff --git a/src/include/gnunet_multicast_service.h b/src/include/gnunet_multicast_service.h
index 3829a7040..5645207dd 100644
--- a/src/include/gnunet_multicast_service.h
+++ b/src/include/gnunet_multicast_service.h
@@ -617,6 +617,7 @@ GNUNET_MULTICAST_origin_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
  *         data.  If 0 is returned in @a data_size the transmission is paused,
  *         and can be resumed with GNUNET_MULTICAST_origin_to_all_resume().
  *         #GNUNET_YES if this completes the transmission (all data supplied)
+ * @deprecated should move to MQ-style API!
  */
 typedef int
 (*GNUNET_MULTICAST_OriginTransmitNotify) (void *cls,
@@ -647,6 +648,7 @@ struct GNUNET_MULTICAST_OriginTransmitHandle;
  *        Closure for @a notify.
  *
  * @return NULL on error (i.e. request already pending).
+ * @deprecated should move to MQ-style API!
  */
 struct GNUNET_MULTICAST_OriginTransmitHandle *
 GNUNET_MULTICAST_origin_to_all (struct GNUNET_MULTICAST_Origin *origin,
@@ -854,6 +856,7 @@ GNUNET_MULTICAST_member_part (struct GNUNET_MULTICAST_Member *member,
  *         data.  If 0 is returned in @a data_size the transmission is paused,
  *         and can be resumed with GNUNET_MULTICAST_member_to_origin_resume().
  *         #GNUNET_YES if this completes the transmission (all data supplied)
+ * @deprecated should move to MQ-style API!
  */
 typedef int
 (*GNUNET_MULTICAST_MemberTransmitNotify) (void *cls,
@@ -880,6 +883,7 @@ struct GNUNET_MULTICAST_MemberTransmitHandle;
  *        Closure for @a notify.
  *
  * @return Handle to cancel request, NULL on error (i.e. request already pending).
+ * @deprecated should move to MQ-style API!
  */
 struct GNUNET_MULTICAST_MemberTransmitHandle *
 GNUNET_MULTICAST_member_to_origin (struct GNUNET_MULTICAST_Member *member,
diff --git a/src/include/gnunet_mysql_lib.h b/src/include/gnunet_mysql_lib.h
index fc6f42f86..6a1e5b3ac 100644
--- a/src/include/gnunet_mysql_lib.h
+++ b/src/include/gnunet_mysql_lib.h
@@ -61,10 +61,12 @@ struct GNUNET_MYSQL_StatementHandle;
  * @param cls user-defined argument
  * @param num_values number of elements in values
  * @param values values returned by MySQL
- * @return GNUNET_OK to continue iterating, GNUNET_SYSERR to abort
+ * @return #GNUNET_OK to continue iterating, #GNUNET_SYSERR to abort
  */
-typedef int (*GNUNET_MYSQL_DataProcessor) (void *cls, unsigned int num_values,
-                                           MYSQL_BIND * values);
+typedef int
+(*GNUNET_MYSQL_DataProcessor) (void *cls,
+                               unsigned int num_values,
+                               MYSQL_BIND * values);
 
 
 /**
@@ -102,7 +104,7 @@ GNUNET_MYSQL_statements_invalidate (struct GNUNET_MYSQL_Context *mc);
 /**
  * Get internal handle for a prepared statement.  This function should rarely
  * be used, and if, with caution!  On failures during the interaction with
- * the handle, you must call 'GNUNET_MYSQL_statements_invalidate'!
+ * the handle, you must call #GNUNET_MYSQL_statements_invalidate()!
  *
  * @param sh prepared statement to introspect
  * @return MySQL statement handle, NULL on error
@@ -129,8 +131,8 @@ GNUNET_MYSQL_statement_prepare (struct GNUNET_MYSQL_Context *mc,
  *
  * @param mc mysql context
  * @param sql SQL statement to run
- * @return GNUNET_OK on success
- *         GNUNET_SYSERR if there was a problem
+ * @return #GNUNET_OK on success
+ *         #GNUNET_SYSERR if there was a problem
  */
 int
 GNUNET_MYSQL_statement_run (struct GNUNET_MYSQL_Context *mc,
diff --git a/src/include/gnunet_network_lib.h b/src/include/gnunet_network_lib.h
index beca83807..d9d3d90e7 100644
--- a/src/include/gnunet_network_lib.h
+++ b/src/include/gnunet_network_lib.h
@@ -588,6 +588,18 @@ void
 GNUNET_NETWORK_fdset_destroy (struct GNUNET_NETWORK_FDSet *fds);
 
 
+/**
+ * Test if the given @a port is available.
+ *
+ * @param ipproto transport protocol to test (i.e. IPPROTO_TCP)
+ * @param port port number to test
+ * @return #GNUNET_OK if the port is available, #GNUNET_NO if not
+ */
+int
+GNUNET_NETWORK_test_port_free (int ipproto,
+                               uint16_t port);
+
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif
diff --git a/src/include/gnunet_os_lib.h b/src/include/gnunet_os_lib.h
index ce3e05f13..b3800f286 100644
--- a/src/include/gnunet_os_lib.h
+++ b/src/include/gnunet_os_lib.h
@@ -381,7 +381,8 @@ GNUNET_OS_process_current (void);
  * @return 0 on success, -1 on error
  */
 int
-GNUNET_OS_process_kill (struct GNUNET_OS_Process *proc, int sig);
+GNUNET_OS_process_kill (struct GNUNET_OS_Process *proc,
+                        int sig);
 
 
 /**
diff --git a/src/include/gnunet_peerstore_service.h b/src/include/gnunet_peerstore_service.h
index 3cafe70b8..c636447c2 100644
--- a/src/include/gnunet_peerstore_service.h
+++ b/src/include/gnunet_peerstore_service.h
@@ -86,7 +86,7 @@ struct GNUNET_PEERSTORE_Record
   /**
    * Peer Identity
    */
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity peer;
 
   /**
    * Record key string
@@ -106,7 +106,7 @@ struct GNUNET_PEERSTORE_Record
   /**
    * Expiry time of entry
    */
-  struct GNUNET_TIME_Absolute *expiry;
+  struct GNUNET_TIME_Absolute expiry;
 
   /**
    * Client from which this record originated.
diff --git a/src/include/gnunet_plugin_lib.h b/src/include/gnunet_plugin_lib.h
index b76b9f9cf..9e3e72a21 100644
--- a/src/include/gnunet_plugin_lib.h
+++ b/src/include/gnunet_plugin_lib.h
@@ -87,7 +87,7 @@ GNUNET_PLUGIN_load (const char *library_name,
 
 
 /**
- * Signature of a function called by 'GNUNET_PLUGIN_load_all'.
+ * Signature of a function called by #GNUNET_PLUGIN_load_all().
  *
  * @param cls closure
  * @param library_name full name of the library (to be used with
diff --git a/src/include/gnunet_pq_lib.h b/src/include/gnunet_pq_lib.h
index 33a3e5425..756370b74 100644
--- a/src/include/gnunet_pq_lib.h
+++ b/src/include/gnunet_pq_lib.h
@@ -15,11 +15,11 @@
 */
 /**
  * @file include/gnunet_pq_lib.h
- * @brief helper functions for DB interactions
+ * @brief helper functions for Postgres DB interactions
  * @author Christian Grothoff
  */
-#ifndef GNUNET_PQ_LIB_H_
-#define GNUNET_PQ_LIB_H_
+#ifndef GNUNET_PQ_LIB_H
+#define GNUNET_PQ_LIB_H
 
 #include <libpq-fe.h>
 #include "gnunet_util_lib.h"
@@ -317,20 +317,6 @@ GNUNET_PQ_result_spec_fixed_size (const char *name,
 
 
 /**
- * Variable-size result expected.
- *
- * @param name name of the field in the table
- * @param[out] dst where to store the result, allocated
- * @param[out] sptr where to store the size of @a dst
- * @return array entry for the result specification to use
- */
-struct GNUNET_PQ_ResultSpec
-GNUNET_PQ_result_spec_variable_size (const char *name,
-                                     void **dst,
-                                     size_t *sptr);
-
-
-/**
  * 0-terminated string expected.
  *
  * @param name name of the field in the table
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index b2807e990..1d8049593 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -424,11 +424,6 @@ extern "C"
 #define GNUNET_MESSAGE_TYPE_DATASTORE_PUT 95
 
 /**
- * Message sent by datastore client to update data.
- */
-#define GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE 96
-
-/**
  * Message sent by datastore client to get data.
  */
 #define GNUNET_MESSAGE_TYPE_DATASTORE_GET 97
@@ -1645,6 +1640,12 @@ extern "C"
  * Demand the whole element from the other
  * peer, given only the hash code.
  */
+#define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL 565
+
+/**
+ * Demand the whole element from the other
+ * peer, given only the hash code.
+ */
 #define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND 566
 
 /**
@@ -1800,6 +1801,19 @@ extern "C"
  */
 #define GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT 596
 
+/**
+ * Request all missing elements from the other peer,
+ * based on their sets and the elements we previously sent
+ * with #GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS.
+ */
+#define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE 597
+
+/**
+ * Send a set element, not as response to a demand but because
+ * we're sending the full set.
+ */
+#define GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT 598
+
 
 /*******************************************************************************
  * TESTBED LOGGER message types
@@ -2664,10 +2678,21 @@ extern "C"
 
 /**
  * Hop-by-hop, connection dependent ACK.
+ *
+ * @deprecated
  */
 #define GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK 1005
 
 /**
+ * We do not bother with ACKs for
+ * #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED messages, but we instead
+ * poll for one if we got nothing for a while and start to be worried.
+ *
+ * @deprecated
+ */
+#define GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL 1006
+
+/**
  * Axolotl key exchange.
  */
 #define GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX 1007
@@ -2678,11 +2703,9 @@ extern "C"
 #define GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED 1008
 
 /**
- * We do not bother with ACKs for
- * #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED messages, but we instead
- * poll for one if we got nothing for a while and start to be worried.
+ * Axolotl key exchange response with authentication.
  */
-#define GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL 1006
+#define GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH 1009
 
 
 
@@ -2720,6 +2743,8 @@ extern "C"
 
 /**
  * Reject the creation of a channel
+ *
+ * @deprecated
  */
 #define GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED 1016
 
diff --git a/src/include/gnunet_scheduler_lib.h b/src/include/gnunet_scheduler_lib.h
index 2be1858ce..a7385e31c 100644
--- a/src/include/gnunet_scheduler_lib.h
+++ b/src/include/gnunet_scheduler_lib.h
@@ -97,6 +97,84 @@ enum GNUNET_SCHEDULER_Reason
 
 
 /**
+ * Possible events on FDs, used as a bitmask.
+ * Modelled after GPollFD.
+ */
+enum GNUNET_SCHEDULER_EventType
+{
+
+  /**
+   * No event (useful for timeout).
+   */
+  GNUNET_SCHEDULER_ET_NONE = 0,
+
+  /**
+   * Data available for reading.
+   */
+  GNUNET_SCHEDULER_ET_IN = 1,
+
+  /**
+   * Buffer available for writing.
+   */
+  GNUNET_SCHEDULER_ET_OUT = 2,
+
+  /**
+   *
+   */
+  GNUNET_SCHEDULER_ET_HUP = 4,
+
+  /**
+   *
+   */
+  GNUNET_SCHEDULER_ET_ERR = 8,
+
+  /**
+   *
+   */
+  GNUNET_SCHEDULER_ET_PRI = 16,
+
+  /**
+   *
+   */
+  GNUNET_SCHEDULER_ET_NVAL = 32
+
+};
+
+
+/**
+ * Information about an event relating to a file descriptor/socket.
+ */
+struct GNUNET_SCHEDULER_FdInfo
+{
+
+  /**
+   * GNUnet network socket the event is about, matches @a sock,
+   * NULL if this is about a file handle or if no network
+   * handle was given to the scheduler originally.
+   */
+  struct GNUNET_NETWORK_Handle *fd;
+
+  /**
+   * GNUnet file handle the event is about, matches @a sock,
+   * NULL if this is about a network socket or if no network
+   * handle was given to the scheduler originally.
+   */ 
+  struct GNUNET_DISK_FileHandle *fh;
+
+  /**
+   * Type of the event that was generated related to @e sock.
+   */
+  enum GNUNET_SCHEDULER_EventType et;
+
+  /**
+   * Underlying OS handle the event was about.
+   */
+  int sock;
+
+};
+
+
+/**
  * Context information passed to each scheduler task.
  */
 struct GNUNET_SCHEDULER_TaskContext
@@ -107,16 +185,29 @@ struct GNUNET_SCHEDULER_TaskContext
   enum GNUNET_SCHEDULER_Reason reason;
 
   /**
-   * Set of file descriptors ready for reading;
-   * note that additional bits may be set
-   * that were not in the original request
+   * Length of the following array.
+   */ 
+  unsigned int fds_len;
+
+  /**
+   * Array of length @e fds_len with information about ready FDs.
+   * Note that we use the same format regardless of the internal
+   * event loop that was used.  The given array should only contain
+   * information about file descriptors relevant to the current task.
+   */
+  const struct GNUNET_SCHEDULER_FdInfo *fds;  
+
+  /**
+   * Set of file descriptors ready for reading; note that additional
+   * bits may be set that were not in the original request.
+   * @deprecated
    */
   const struct GNUNET_NETWORK_FDSet *read_ready;
 
   /**
-   * Set of file descriptors ready for writing;
-   * note that additional bits may be set
-   * that were not in the original request.
+   * Set of file descriptors ready for writing; note that additional
+   * bits may be set that were not in the original request.
+   * @deprecated
    */
   const struct GNUNET_NETWORK_FDSet *write_ready;
 
@@ -124,16 +215,155 @@ struct GNUNET_SCHEDULER_TaskContext
 
 
 /**
+ * Function used by event-loop implementations to signal the scheduler
+ * that a particular @a task is ready due to an event of type @a et.
+ *
+ * This function will then queue the task to notify the application
+ * that the task is ready (with the respective priority).
+ *
+ * @param task the task that is ready
+ * @param et information about why the task is ready
+ */
+void
+GNUNET_SCHEDULER_task_ready (struct GNUNET_SCHEDULER_Task *task,
+                             enum GNUNET_SCHEDULER_EventType et);
+
+
+/**
+ * Handle to the scheduler's state to be used by the driver.
+ */
+struct GNUNET_SCHEDULER_Handle;
+
+
+/**
+ * Function called by the driver to tell the scheduler to run some of
+ * the tasks that are ready.  This function may return even though
+ * there are tasks left to run just to give other tasks a chance as
+ * well.  If we return #GNUNET_YES, the driver should call this
+ * function again as soon as possible, while if we return #GNUNET_NO
+ * it must block until the operating system has more work as the
+ * scheduler has no more work to do right now.
+ *
+ * @param sh scheduler handle that was given to the `loop`
+ * @return #GNUNET_OK if there are more tasks that are ready,
+ *          and thus we would like to run more (yield to avoid 
+ *          blocking other activities for too long)
+ *         #GNUNET_NO if we are done running tasks (yield to block)
+ *         #GNUNET_SYSERR on error
+ */
+int
+GNUNET_SCHEDULER_run_from_driver (struct GNUNET_SCHEDULER_Handle *sh);
+
+
+/**
+ * API a driver has to implement for
+ * #GNUNET_SCHEDULER_run_with_driver().
+ */
+struct GNUNET_SCHEDULER_Driver
+{
+
+  /**
+   * Closure to pass to the functions in this struct.
+   */
+  void *cls;
+
+  /**
+   * Add a @a task to be run if the conditions given
+   * in @a fdi are satisfied.
+   *
+   * @param cls closure
+   * @param task task to add
+   * @param fdi conditions to watch for
+   * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+   *   (i.e. @a fdi too high or invalid)
+   */
+  int
+  (*add)(void *cls,
+         struct GNUNET_SCHEDULER_Task *task,
+         struct GNUNET_SCHEDULER_FdInfo *fdi);
+
+  /**
+   * Delete a @a task from the set of tasks to be run.
+   *
+   * @param cls closure
+   * @param task task to delete
+   * @param fdi conditions to watch for (must match @e add call)
+   * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+   *   (i.e. @a task or @a fdi do not match prior @e add call)
+   */
+  int
+  (*del)(void *cls,
+         struct GNUNET_SCHEDULER_Task *task,
+         const struct GNUNET_SCHEDULER_FdInfo *fdi);
+
+  /**
+   * Set time at which we definitively want to get a wakeup call.
+   *
+   * @param cls closure
+   * @param dt time when we want to wake up next
+   */
+  void
+  (*set_wakeup)(void *cls,
+                struct GNUNET_TIME_Absolute dt);
+
+  /**
+   * Event loop's "main" function, to be called from
+   * #GNUNET_SCHEDULER_run_with_driver() to actually
+   * launch the loop.
+   *
+   * @param cls closure
+   * @param sh scheduler handle to pass to
+   *     #GNUNET_SCHEDULER_run_from_driver()
+   * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+   */
+  int
+  (*loop)(void *cls,
+          struct GNUNET_SCHEDULER_Handle *sh);
+  
+};
+
+
+/**
  * Signature of the main function of a task.
  *
  * @param cls closure
- * @param tc context information (why was this task triggered now)
  */
 typedef void
 (*GNUNET_SCHEDULER_TaskCallback) (void *cls);
 
 
 /**
+ * Initialize and run scheduler.  This function will return when all
+ * tasks have completed.  On systems with signals, receiving a SIGTERM
+ * (and other similar signals) will cause #GNUNET_SCHEDULER_shutdown
+ * to be run after the active task is complete.  As a result, SIGTERM
+ * causes all shutdown tasks to be scheduled with reason
+ * #GNUNET_SCHEDULER_REASON_SHUTDOWN.  (However, tasks added
+ * afterwards will execute normally!).  Note that any particular
+ * signal will only shut down one scheduler; applications should
+ * always only create a single scheduler.
+ *
+ * @param driver drive to use for the event loop
+ * @param task task to run first (and immediately)
+ * @param task_cls closure of @a task
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+int
+GNUNET_SCHEDULER_run_with_driver (const struct GNUNET_SCHEDULER_Driver *driver,
+                                  GNUNET_SCHEDULER_TaskCallback task,
+                                  void *task_cls);
+
+
+/**
+ * Obtain the driver for using select() as the event loop.
+ *
+ * @return NULL on error
+ */
+const struct GNUNET_SCHEDULER_Driver *
+GNUNET_SCHEDULER_driver_select (void);
+
+
+/**
  * Signature of the select function used by the scheduler.
  * #GNUNET_NETWORK_socket_select matches it.
  *
@@ -571,7 +801,7 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio,
  * Sets the select function to use in the scheduler (scheduler_select).
  *
  * @param new_select new select function to use (NULL to reset to default)
- * @param new_select_cls closure for 'new_select'
+ * @param new_select_cls closure for @a new_select
  */
 void
 GNUNET_SCHEDULER_set_select (GNUNET_SCHEDULER_select new_select,
diff --git a/src/include/gnunet_server_lib.h b/src/include/gnunet_server_lib.h
deleted file mode 100644
index 5da31bcd7..000000000
--- a/src/include/gnunet_server_lib.h
+++ /dev/null
@@ -1,887 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @author Christian Grothoff
- *
- * @file
- * Library for building GNUnet network servers
-
- * @defgroup server  Server library
- * Library for building GNUnet network servers
- *
- * Provides functions for a server that communicates with clients.
- *
- * @see [Documentation](https://gnunet.org/ipc)
- *
- * @{
- */
-
-#ifndef GNUNET_SERVER_LIB_H
-#define GNUNET_SERVER_LIB_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "gnunet_common.h"
-#include "gnunet_connection_lib.h"
-
-
-/**
- * Largest supported message (to be precise, one byte more
- * than the largest possible message, so tests involving
- * this value should check for messages being smaller than
- * this value).
- */
-#define GNUNET_SERVER_MAX_MESSAGE_SIZE 65536
-
-/**
- * Smallest supported message.
- */
-#define GNUNET_SERVER_MIN_BUFFER_SIZE sizeof (struct GNUNET_MessageHeader)
-
-/**
- * @brief handle for a server
- */
-struct GNUNET_SERVER_Handle;
-
-/**
- * @brief opaque handle for a client of the server
- */
-struct GNUNET_SERVER_Client;
-
-/**
- * @brief opaque handle server returns for aborting transmission to a client.
- */
-struct GNUNET_SERVER_TransmitHandle;
-
-
-/**
- * Functions with this signature are called whenever a message is
- * received.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-typedef void
-(*GNUNET_SERVER_MessageCallback) (void *cls,
-                                  struct GNUNET_SERVER_Client *client,
-                                  const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Message handler.  Each struct specifies how to handle on particular
- * type of message received.
- */
-struct GNUNET_SERVER_MessageHandler
-{
-  /**
-   * Function to call for messages of "type".
-   */
-  GNUNET_SERVER_MessageCallback callback;
-
-  /**
-   * Closure argument for @e callback.
-   */
-  void *callback_cls;
-
-  /**
-   * Type of the message this handler covers.
-   */
-  uint16_t type;
-
-  /**
-   * Expected size of messages of this type.  Use 0 for
-   * variable-size.  If non-zero, messages of the given
-   * type will be discarded (and the connection closed)
-   * if they do not have the right size.
-   */
-  uint16_t expected_size;
-
-};
-
-
-/**
- * Create a new server.
- *
- * @param access_cb function for access control
- * @param access_cb_cls closure for @a access_cb
- * @param lsocks NULL-terminated array of listen sockets
- * @param idle_timeout after how long should we timeout idle connections?
- * @param require_found if #GNUNET_YES, connections sending messages of unknown type
- *        will be closed
- * @return handle for the new server, NULL on error
- *         (typically, "port" already in use)
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVER_create_with_sockets (GNUNET_CONNECTION_AccessCheck access_cb,
-                                   void *access_cb_cls,
-                                   struct GNUNET_NETWORK_Handle **lsocks,
-                                   struct GNUNET_TIME_Relative idle_timeout,
-                                   int require_found);
-
-/**
- * Create a new server.
- *
- * @param access_cb function for access control
- * @param access_cb_cls closure for @a access_cb
- * @param server_addr address toes listen on (including port), NULL terminated array
- * @param socklen lengths of respective @a server_addr
- * @param idle_timeout after how long should we timeout idle connections?
- * @param require_found if #GNUNET_YES, connections sending messages of unknown type
- *        will be closed
- * @return handle for the new server, NULL on error
- *         (typically, "port" already in use)
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVER_create (GNUNET_CONNECTION_AccessCheck access_cb,
-                      void *access_cb_cls,
-                      struct sockaddr *const *server_addr,
-                      const socklen_t *socklen,
-                      struct GNUNET_TIME_Relative idle_timeout,
-                      int require_found);
-
-
-/**
- * Suspend accepting connections from the listen socket temporarily.
- * Resume activity using #GNUNET_SERVER_resume.
- *
- * @param server server to stop accepting connections.
- */
-void
-GNUNET_SERVER_suspend (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Resume accepting connections from the listen socket.
- *
- * @param server server to resume accepting connections.
- */
-void
-GNUNET_SERVER_resume (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Stop the listen socket and get ready to shutdown the server once
- * only clients marked using #GNUNET_SERVER_client_mark_monitor are
- * left.
- *
- * @param server server to stop listening on
- */
-void
-GNUNET_SERVER_stop_listening (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Free resources held by this server.
- *
- * @param server server to destroy
- */
-void
-GNUNET_SERVER_destroy (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Add additional handlers to an existing server.
- *
- * @param server the server to add handlers to
- * @param handlers array of message handlers for
- *        incoming messages; the last entry must
- *        have "NULL" for the "callback"; multiple
- *        entries for the same type are allowed,
- *        they will be called in order of occurence.
- *        These handlers can be removed later;
- *        the handlers array must exist until removed
- *        (or server is destroyed).
- */
-void
-GNUNET_SERVER_add_handlers (struct GNUNET_SERVER_Handle *server,
-                            const struct GNUNET_SERVER_MessageHandler *handlers);
-
-
-/**
- * Notify us when the server has enough space to transmit
- * a message of the given size to the given client.
- *
- * @param client client to transmit message to
- * @param size requested amount of buffer space
- * @param timeout after how long should we give up (and call
- *        notify with buf NULL and size 0)?
- * @param callback function to call when space is available
- * @param callback_cls closure for @a callback
- * @return non-NULL if the notify callback was queued; can be used
- *           to cancel the request using
- *           #GNUNET_SERVER_notify_transmit_ready_cancel.
- *         NULL if we are already going to notify someone else (busy)
- */
-struct GNUNET_SERVER_TransmitHandle *
-GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
-                                     size_t size,
-                                     struct GNUNET_TIME_Relative timeout,
-                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
-                                     void *callback_cls);
-
-
-/**
- * Abort transmission request.
- *
- * @param th request to abort
- */
-void
-GNUNET_SERVER_notify_transmit_ready_cancel (struct GNUNET_SERVER_TransmitHandle *th);
-
-
-/**
- * Set the 'monitor' flag on this client.  Clients which have been
- * marked as 'monitors' won't prevent the server from shutting down
- * once #GNUNET_SERVER_stop_listening has been invoked.  The idea is
- * that for "normal" clients we likely want to allow them to process
- * their requests; however, monitor-clients are likely to 'never'
- * disconnect during shutdown and thus will not be considered when
- * determining if the server should continue to exist after
- * #GNUNET_SERVER_destroy has been called.
- *
- * @param client the client to set the 'monitor' flag on
- */
-void
-GNUNET_SERVER_client_mark_monitor (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Set the persistent flag on this client, used to setup client
- * connection to only be killed when the process of the service it's
- * connected to is actually dead.  This API is used during shutdown
- * signalling within ARM, and it is not expected that typical users
- * of the API would need this function.
- *
- * @param client the client to set the persistent flag on
- */
-void
-GNUNET_SERVER_client_persist_ (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Resume receiving from this client, we are done processing the
- * current request.  This function must be called from within each
- * #GNUNET_SERVER_MessageCallback (or its respective continuations).
- *
- * @param client client we were processing a message of
- * @param success #GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                #GNUNET_NO to close the connection (normal behavior)
- *                #GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_receive_done (struct GNUNET_SERVER_Client *client,
-                            int success);
-
-
-/**
- * Change the timeout for a particular client.  Decreasing the timeout
- * may not go into effect immediately (only after the previous timeout
- * times out or activity happens on the socket).
- *
- * @param client the client to update
- * @param timeout new timeout for activities on the socket
- */
-void
-GNUNET_SERVER_client_set_timeout (struct GNUNET_SERVER_Client *client,
-                                  struct GNUNET_TIME_Relative timeout);
-
-
-/**
- * Return user context associated with the given client.
- * Note: you should probably use the macro (call without the underscore).
- *
- * @param client client to query
- * @param size number of bytes in user context struct (for verification only)
- * @return pointer to user context
- */
-void *
-GNUNET_SERVER_client_get_user_context_ (struct GNUNET_SERVER_Client *client,
-                                        size_t size);
-
-
-/**
- * Set user context to be associated with the given client.
- * Note: you should probably use the macro (call without the underscore).
- *
- * @param client client to query
- * @param ptr pointer to user context
- * @param size number of bytes in user context struct (for verification only)
- */
-void
-GNUNET_SERVER_client_set_user_context_ (struct GNUNET_SERVER_Client *client,
-                                        void *ptr,
-                                        size_t size);
-
-
-/**
- * Return user context associated with the given client.
- *
- * @param client client to query
- * @param type expected return type (i.e. 'struct Foo')
- * @return pointer to user context of type 'type *'.
- */
-#define GNUNET_SERVER_client_get_user_context(client,type)              \
-  (type *) GNUNET_SERVER_client_get_user_context_ (client, sizeof (type))
-
-/**
- * Set user context to be associated with the given client.
- *
- * @param client client to query
- * @param value pointer to user context
- */
-#define GNUNET_SERVER_client_set_user_context(client,value)             \
-  GNUNET_SERVER_client_set_user_context_ (client, value, sizeof (*value))
-
-
-/**
- * Disable the warning the server issues if a message is not acknowledged
- * in a timely fashion.  Use this call if a client is intentionally delayed
- * for a while.  Only applies to the current message.
- *
- * @param client client for which to disable the warning
- */
-void
-GNUNET_SERVER_disable_receive_done_warning (struct GNUNET_SERVER_Client
-                                            *client);
-
-
-/**
- * Inject a message into the server, pretend it came
- * from the specified client.  Delivery of the message
- * will happen instantly (if a handler is installed;
- * otherwise the call does nothing).
- *
- * @param server the server receiving the message
- * @param sender the "pretended" sender of the message
- *        can be NULL!
- * @param message message to transmit
- * @return #GNUNET_OK if the message was OK and the
- *                   connection can stay open
- *         #GNUNET_SYSERR if the connection to the
- *         client should be shut down
- */
-int
-GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
-                      struct GNUNET_SERVER_Client *sender,
-                      const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Add a TCP socket-based connection to the set of handles managed by
- * this server.  Use this function for outgoing (P2P) connections that
- * we initiated (and where this server should process incoming
- * messages).
- *
- * @param server the server to use
- * @param connection the connection to manage (client must
- *        stop using this connection from now on)
- * @return the client handle
- */
-struct GNUNET_SERVER_Client *
-GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
-                              struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Notify the server that the given client handle should
- * be kept (keeps the connection up if possible, increments
- * the internal reference counter).
- *
- * @param client the client to keep
- */
-void
-GNUNET_SERVER_client_keep (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Notify the server that the given client handle is no
- * longer required.  Decrements the reference counter.  If
- * that counter reaches zero an inactive connection maybe
- * closed.
- *
- * @param client the client to drop
- */
-void
-GNUNET_SERVER_client_drop (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Obtain the network address of the other party.
- *
- * @param client the client to get the address for
- * @param addr where to store the address
- * @param addrlen where to store the length of @a addr
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_SERVER_client_get_address (struct GNUNET_SERVER_Client *client,
-                                  void **addr, size_t *addrlen);
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
- */
-typedef void
-(*GNUNET_SERVER_DisconnectCallback) (void *cls,
-                                     struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Functions with this signature are called whenever a client
- * is connected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-typedef void
-(*GNUNET_SERVER_ConnectCallback) (void *cls,
-                                  struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Ask the server to notify us whenever a client disconnects.
- * This function is called whenever the actual network connection
- * is closed; the reference count may be zero or larger than zero
- * at this point.  If the server is destroyed before this
- * notification is explicitly cancelled, the 'callback' will
- * once be called with a 'client' argument of NULL to indicate
- * that the server itself is now gone (and that the callback
- * won't be called anymore and also can no longer be cancelled).
- *
- * @param server the server manageing the clients
- * @param callback function to call on disconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_disconnect_notify (struct GNUNET_SERVER_Handle *server,
-                                 GNUNET_SERVER_DisconnectCallback callback,
-                                 void *callback_cls);
-
-
-/**
- * Ask the server to notify us whenever a client connects.
- * This function is called whenever the actual network connection
- * is opened. If the server is destroyed before this
- * notification is explicitly cancelled, the @a callback will
- * once be called with a 'client' argument of NULL to indicate
- * that the server itself is now gone (and that the callback
- * won't be called anymore and also can no longer be cancelled).
- *
- * @param server the server manageing the clients
- * @param callback function to call on sconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_connect_notify (struct GNUNET_SERVER_Handle *server,
-                              GNUNET_SERVER_ConnectCallback callback,
-                              void *callback_cls);
-
-
-/**
- * Ask the server to stop notifying us whenever a client disconnects.
- * Arguments must match exactly those given to
- * #GNUNET_SERVER_disconnect_notify.  It is not necessary to call this
- * function during shutdown of the server; in fact, most applications
- * will never use this function.
- *
- * @param server the server manageing the clients
- * @param callback function to call on disconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_disconnect_notify_cancel (struct GNUNET_SERVER_Handle *server,
-                                        GNUNET_SERVER_DisconnectCallback callback,
-                                        void *callback_cls);
-
-
-/**
- * Ask the server to stop notifying us whenever a client connects.
- * Arguments must match exactly those given to
- * #GNUNET_SERVER_connect_notify.  It is not necessary to call this
- * function during shutdown of the server; in fact, most applications
- * will never use this function.
- *
- * @param server the server manageing the clients
- * @param callback function to call on connect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_connect_notify_cancel (struct GNUNET_SERVER_Handle *server,
-                                     GNUNET_SERVER_ConnectCallback callback,
-                                     void *callback_cls);
-
-
-/**
- * Ask the server to disconnect from the given client.  This is the
- * same as passing #GNUNET_SYSERR to #GNUNET_SERVER_receive_done,
- * except that it allows dropping of a client even when not handling a
- * message from that client.
- *
- * @param client the client to disconnect from
- */
-void
-GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Disable the "CORK" feature for communication with the given client,
- * forcing the OS to immediately flush the buffer on transmission
- * instead of potentially buffering multiple messages.
- *
- * @param client handle to the client
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_SERVER_client_disable_corking (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * The tansmit context is the key datastructure for a conveniance API
- * used for transmission of complex results to the client followed
- * ONLY by signaling receive_done with success or error
- */
-struct GNUNET_SERVER_TransmitContext;
-
-
-/**
- * Create a new transmission context for the given client.
- *
- * @param client client to create the context for.
- * @return NULL on error
- */
-struct GNUNET_SERVER_TransmitContext *
-GNUNET_SERVER_transmit_context_create (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the #GNUNET_SERVER_transmit_context_run method.
- *
- * @param tc context to use
- * @param data what to append to the result message
- * @param length length of @a data
- * @param type type of the message
- */
-void
-GNUNET_SERVER_transmit_context_append_data (struct GNUNET_SERVER_TransmitContext *tc,
-                                            const void *data,
-                                            size_t length, uint16_t type);
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param msg message to append
- */
-void
-GNUNET_SERVER_transmit_context_append_message (struct GNUNET_SERVER_TransmitContext *tc,
-                                               const struct GNUNET_MessageHeader *msg);
-
-
-/**
- * Execute a transmission context.  If there is an error in the
- * transmission, the receive_done method will be called with an error
- * code (#GNUNET_SYSERR), otherwise with #GNUNET_OK.
- *
- * @param tc transmission context to use
- * @param timeout when to time out and abort the transmission
- */
-void
-GNUNET_SERVER_transmit_context_run (struct GNUNET_SERVER_TransmitContext *tc,
-                                    struct GNUNET_TIME_Relative timeout);
-
-
-/**
- * Destroy a transmission context.  This function must not be called
- * after #GNUNET_SERVER_transmit_context_run.
- *
- * @param tc transmission context to destroy
- * @param success code to give to #GNUNET_SERVER_receive_done  for
- *        the client: #GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                #GNUNET_NO to close the connection (normal behavior)
- *                #GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_transmit_context_destroy (struct GNUNET_SERVER_TransmitContext *tc,
-                                        int success);
-
-
-/**
- * The notification context is the key datastructure for a conveniance
- * API used for transmission of notifications to the client until the
- * client disconnects or is disconnected (or the notification context
- * is destroyed, in which case we disconnect these clients).
- * Essentially, all (notification) messages are queued up until the
- * client is able to read them.
- */
-struct GNUNET_SERVER_NotificationContext;
-
-
-/**
- * Create a new notification context.
- *
- * @param server server for which this function creates the context
- * @param queue_length maximum number of messages to keep in
- *        the notification queue; optional messages are dropped
- *        if the queue gets longer than this number of messages
- * @return handle to the notification context
- */
-struct GNUNET_SERVER_NotificationContext *
-GNUNET_SERVER_notification_context_create (struct GNUNET_SERVER_Handle *server,
-                                           unsigned int queue_length);
-
-
-/**
- * Destroy the context, force disconnect for all clients.
- *
- * @param nc context to destroy.
- */
-void
-GNUNET_SERVER_notification_context_destroy (struct GNUNET_SERVER_NotificationContext *nc);
-
-
-/**
- * Add a client to the notification context.
- *
- * @param nc context to modify
- * @param client client to add
- */
-void
-GNUNET_SERVER_notification_context_add (struct GNUNET_SERVER_NotificationContext *nc,
-                                        struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Send a message to a particular client; must have
- * already been added to the notification context.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-                                            struct GNUNET_SERVER_Client *client,
-                                            const struct GNUNET_MessageHeader *msg,
-                                            int can_drop);
-
-
-/**
- * Send a message to all clients of this context.
- *
- * @param nc context to modify
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_broadcast (struct GNUNET_SERVER_NotificationContext *nc,
-                                              const struct GNUNET_MessageHeader *msg,
-                                              int can_drop);
-
-
-/**
- * Return active number of subscribers in this context.
- *
- * @param nc context to query
- * @return number of current subscribers
- */
-unsigned int
-GNUNET_SERVER_notification_context_get_size (struct GNUNET_SERVER_NotificationContext *nc);
-
-
-/**
- * Create a message queue for a server's client.
- *
- * @param client the client
- * @return the message queue
- */
-struct GNUNET_MQ_Handle *
-GNUNET_MQ_queue_for_server_client (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Handle to a message stream tokenizer.
- */
-struct GNUNET_SERVER_MessageStreamTokenizer;
-
-
-/**
- * Functions with this signature are called whenever a
- * complete message is received by the tokenizer.
- *
- * Do not call #GNUNET_SERVER_mst_destroy from within
- * the scope of this callback.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
- */
-typedef int
-(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
-                                           void *client,
-                                           const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Create a message stream tokenizer.
- *
- * @param cb function to call on completed messages
- * @param cb_cls closure for @a cb
- * @return handle to tokenizer
- */
-struct GNUNET_SERVER_MessageStreamTokenizer *
-GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
-                          void *cb_cls);
-
-
-/**
- * Add incoming data to the receive buffer and call the
- * callback for all complete messages.
- *
- * @param mst tokenizer to use
- * @param client_identity ID of client for which this is a buffer,
- *        can be NULL (will be passed back to 'cb')
- * @param buf input data to add
- * @param size number of bytes in @a buf
- * @param purge should any excess bytes in the buffer be discarded
- *       (i.e. for packet-based services like UDP)
- * @param one_shot only call callback once, keep rest of message in buffer
- * @return #GNUNET_OK if we are done processing (need more data)
- *         #GNUNET_NO if one_shot was set and we have another message ready
- *         #GNUNET_SYSERR if the data stream is corrupt
- */
-int
-GNUNET_SERVER_mst_receive (struct GNUNET_SERVER_MessageStreamTokenizer *mst,
-                           void *client_identity,
-                           const char *buf, size_t size,
-                           int purge, int one_shot);
-
-
-/**
- * Destroys a tokenizer.
- *
- * @param mst tokenizer to destroy
- */
-void
-GNUNET_SERVER_mst_destroy (struct GNUNET_SERVER_MessageStreamTokenizer *mst);
-
-
-/**
- * Signature of a function to create a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param client handle to client the tokenzier will be used for
- * @return handle to custom tokenizer ('mst')
- */
-typedef void*
-(*GNUNET_SERVER_MstCreateCallback) (void *cls,
-                                    struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Signature of a function to destroy a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param mst custom tokenizer handle
- */
-typedef void
-(*GNUNET_SERVER_MstDestroyCallback) (void *cls,
-                                     void *mst);
-
-
-/**
- * Signature of a function to receive data for a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param mst custom tokenizer handle
- * @param client_identity ID of client for which this is a buffer,
- *        can be NULL (will be passed back to 'cb')
- * @param buf input data to add
- * @param size number of bytes in @a buf
- * @param purge should any excess bytes in the buffer be discarded
- *       (i.e. for packet-based services like UDP)
- * @param one_shot only call callback once, keep rest of message in buffer
- * @return #GNUNET_OK if we are done processing (need more data)
- *         #GNUNET_NO if one_shot was set and we have another message ready
- *         #GNUNET_SYSERR if the data stream is corrupt
- */
-typedef int
-(*GNUNET_SERVER_MstReceiveCallback) (void *cls, void *mst,
-                                     struct GNUNET_SERVER_Client *client,
-                                     const char *buf,
-                                     size_t size,
-                                     int purge,
-                                     int one_shot);
-
-
-/**
- * Change functions used by the server to tokenize the message stream.
- * (very rarely used).
- *
- * @param server server to modify
- * @param create new tokenizer initialization function
- * @param destroy new tokenizer destruction function
- * @param receive new tokenizer receive function
- * @param cls closure for @a create, @a receive and @a destroy
- */
-void
-GNUNET_SERVER_set_callbacks (struct GNUNET_SERVER_Handle *server,
-                             GNUNET_SERVER_MstCreateCallback create,
-                             GNUNET_SERVER_MstDestroyCallback destroy,
-                             GNUNET_SERVER_MstReceiveCallback receive,
-                             void *cls);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_SERVER_LIB_H */
-#endif
-
-/** @} */  /* end of group server */
-
-/* end of gnunet_server_lib.h */
diff --git a/src/include/gnunet_service_lib.h b/src/include/gnunet_service_lib.h
index 75b880530..aacafe956 100644
--- a/src/include/gnunet_service_lib.h
+++ b/src/include/gnunet_service_lib.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2013, 2016 GNUnet e.V.
+     Copyright (C) 2009-2013, 2016, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -44,25 +44,10 @@ extern "C"
 #endif
 
 #include "gnunet_configuration_lib.h"
-#include "gnunet_server_lib.h"
 #include "gnunet_mq_lib.h"
 
 
 /**
- * Function called by the service's run
- * method to run service-specific setup code.
- *
- * @param cls closure
- * @param server the initialized server
- * @param cfg configuration to use
- */
-typedef void
-(*GNUNET_SERVICE_Main) (void *cls,
-                        struct GNUNET_SERVER_Handle *server,
-                        const struct GNUNET_CONFIGURATION_Handle *cfg);
-
-
-/**
  * Options for the service (bitmask).
  */
 enum GNUNET_SERVICE_Options
@@ -88,84 +73,6 @@ enum GNUNET_SERVICE_Options
 };
 
 
-/**
- * Run a standard GNUnet service startup sequence (initialize loggers
- * and configuration, parse options).
- *
- * @param argc number of command line arguments in @a argv
- * @param argv command line arguments
- * @param service_name our service name
- * @param options service options
- * @param task main task of the service
- * @param task_cls closure for @a task
- * @return #GNUNET_SYSERR on error, #GNUNET_OK
- *         if we shutdown nicely
- * @deprecated
- */
-int
-GNUNET_SERVICE_run (int argc,
-                    char *const *argv,
-                    const char *service_name,
-                    enum GNUNET_SERVICE_Options options,
-                    GNUNET_SERVICE_Main task,
-                    void *task_cls);
-
-
-/**
- * Opaque handle for a service.
- */
-struct GNUNET_SERVICE_Context;
-
-
-/**
- * Run a service startup sequence within an existing
- * initialized system.
- *
- * @param service_name our service name
- * @param cfg configuration to use
- * @param options service options
- * @return NULL on error, service handle
- * @deprecated
- */
-struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      enum GNUNET_SERVICE_Options options);
-
-
-/**
- * Obtain the server used by a service.  Note that the server must NOT
- * be destroyed by the caller.
- *
- * @param ctx the service context returned from the start function
- * @return handle to the server for this service, NULL if there is none
- * @deprecated
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVICE_get_server (struct GNUNET_SERVICE_Context *ctx);
-
-
-/**
- * Get the NULL-terminated array of listen sockets for this service.
- *
- * @param ctx service context to query
- * @return NULL if there are no listen sockets, otherwise NULL-terminated
- *              array of listen sockets.
- * @deprecated
- */
-struct GNUNET_NETWORK_Handle *const *
-GNUNET_SERVICE_get_listen_sockets (struct GNUNET_SERVICE_Context *ctx);
-
-
-/**
- * Stop a service that was started with #GNUNET_SERVICE_start.
- *
- * @param sctx the service context returned from the start function
- * @deprecated
- */
-void
-GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Context *sctx);
-
 
 /* **************** NEW SERVICE API ********************** */
 
@@ -245,7 +152,7 @@ typedef void
  * dropped. Additionally, clients can be dropped at any time using
  * #GNUNET_SERVICE_client_drop().
  *
- * The service must be stopped using #GNUNET_SERVICE_stoP().
+ * The service must be stopped using #GNUNET_SERVICE_stop().
  *
  * @param service_name name of the service to run
  * @param cfg configuration to use
@@ -258,7 +165,7 @@ typedef void
  * @return NULL on error
  */
 struct GNUNET_SERVICE_Handle *
-GNUNET_SERVICE_starT (const char *service_name,
+GNUNET_SERVICE_start (const char *service_name,
                       const struct GNUNET_CONFIGURATION_Handle *cfg,
                       GNUNET_SERVICE_ConnectHandler connect_cb,
                       GNUNET_SERVICE_DisconnectHandler disconnect_cb,
@@ -267,12 +174,12 @@ GNUNET_SERVICE_starT (const char *service_name,
 
 
 /**
- * Stops a service that was started with #GNUNET_SERVICE_starT().
+ * Stops a service that was started with #GNUNET_SERVICE_start().
  *
  * @param srv service to stop
  */
 void
-GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv);
+GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Handle *srv);
 
 
 /**
@@ -317,7 +224,7 @@ GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv);
  * @return 0 on success, non-zero on error
  */
 int
-GNUNET_SERVICE_ruN_ (int argc,
+GNUNET_SERVICE_run_ (int argc,
                      char *const *argv,
                      const char *service_name,
                      enum GNUNET_SERVICE_Options options,
@@ -393,7 +300,7 @@ GNUNET_SERVICE_ruN_ (int argc,
     struct GNUNET_MQ_MessageHandler mh[] = { \
       __VA_ARGS__ \
     };                        \
-    return GNUNET_SERVICE_ruN_ (argc, \
+    return GNUNET_SERVICE_run_ (argc, \
                                 argv, \
                                 service_name, \
                                 service_options, \
diff --git a/src/include/gnunet_set_service.h b/src/include/gnunet_set_service.h
index 44773f187..6e822d82e 100644
--- a/src/include/gnunet_set_service.h
+++ b/src/include/gnunet_set_service.h
@@ -126,7 +126,7 @@ enum GNUNET_SET_Status
 
   /**
    * Element should be added to the result set
-   * of the remove peer, i.e. the remote peer is
+   * of the remote peer, i.e. the remote peer is
    * missing an element.
    *
    * Only applies to #GNUNET_SET_RESULT_SYMMETRIC
@@ -153,6 +153,7 @@ enum GNUNET_SET_Status
 };
 
 
+
 /**
  * The way results are given to the client.
  */
@@ -181,7 +182,7 @@ enum GNUNET_SET_ResultMode
   GNUNET_SET_RESULT_REMOVED,
 
   /**
-   * Client gets only elements that have been removed from the set.
+   * Client gets only elements that have been added to the set.
    *
    * Only supported for set union.
    */
@@ -212,6 +213,58 @@ struct GNUNET_SET_Element
 
 
 /**
+ * Possible options to pass to a set operation.
+ *
+ * Used as tag for struct #GNUNET_SET_Option.
+ */
+enum GNUNET_SET_OptionType
+{
+  /**
+   * List terminator.
+   */
+  GNUNET_SET_OPTION_END=0,
+  /**
+   * Fail set operations when the other peer shows weird behavior
+   * that might by a Byzantine fault.
+   *
+   * For set union, 'v.num' is a lower bound on elements
+   * that the other peer must have in common with us.
+   */
+  GNUNET_SET_OPTION_BYZANTINE=1,
+  /**
+   * Do not use the optimized set operation, but send full sets.
+   * Might trigger Byzantine fault detection.
+   */
+  GNUNET_SET_OPTION_FORCE_FULL=2,
+  /**
+   * Only use optimized set operations, even though for this
+   * particular set operation they might be much slower.
+   * Might trigger Byzantine fault detection.
+   */
+  GNUNET_SET_OPTION_FORCE_DELTA=4,
+};
+
+
+/**
+ * Option for set operations.
+ */
+struct GNUNET_SET_Option
+{
+  /**
+   * Type of the option.
+   */
+  enum GNUNET_SET_OptionType type;
+
+  /**
+   * Value for the option, only used with some options.
+   */
+  union {
+    uint64_t num;
+  } v;
+};
+
+
+/**
  * Continuation used for some of the set operations
  *
  * @param cls closure
@@ -226,11 +279,13 @@ typedef void
  *
  * @param cls closure
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status see `enum GNUNET_SET_Status`
  */
 typedef void
 (*GNUNET_SET_ResultIterator) (void *cls,
                               const struct GNUNET_SET_Element *element,
+                              uint64_t current_size,
                               enum GNUNET_SET_Status status);
 
 /**
@@ -367,6 +422,7 @@ GNUNET_SET_prepare (const struct GNUNET_PeerIdentity *other_peer,
                     const struct GNUNET_HashCode *app_id,
                     const struct GNUNET_MessageHeader *context_msg,
                     enum GNUNET_SET_ResultMode result_mode,
+                    struct GNUNET_SET_Option options[],
                     GNUNET_SET_ResultIterator result_cb,
                     void *result_cls);
 
@@ -420,6 +476,7 @@ GNUNET_SET_listen_cancel (struct GNUNET_SET_ListenHandle *lh);
 struct GNUNET_SET_OperationHandle *
 GNUNET_SET_accept (struct GNUNET_SET_Request *request,
                    enum GNUNET_SET_ResultMode result_mode,
+                   struct GNUNET_SET_Option options[],
                    GNUNET_SET_ResultIterator result_cb,
                    void *result_cls);
 
diff --git a/src/include/gnunet_sq_lib.h b/src/include/gnunet_sq_lib.h
new file mode 100644
index 000000000..c196d7767
--- /dev/null
+++ b/src/include/gnunet_sq_lib.h
@@ -0,0 +1,451 @@
+/*
+  This file is part of GNUnet
+  Copyright (C) 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file include/gnunet_sq_lib.h
+ * @brief helper functions for Sqlite3 DB interactions
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SQ_LIB_H
+#define GNUNET_SQ_LIB_H
+
+#include <sqlite3.h>
+#include "gnunet_util_lib.h"
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+typedef int
+(*GNUNET_SQ_QueryConverter)(void *cls,
+                            const void *data,
+                            size_t data_len,
+                            sqlite3_stmt *stmt,
+                            unsigned int off);
+
+
+/**
+ * @brief Description of a DB query parameter.
+ */
+struct GNUNET_SQ_QueryParam
+{
+
+  /**
+   * Function for how to handle this type of entry.
+   */
+  GNUNET_SQ_QueryConverter conv;
+
+  /**
+   * Closure for @e conv.
+   */
+  void *conv_cls;
+
+  /**
+   * Data or NULL.
+   */
+  const void *data;
+
+  /**
+   * Size of @e data
+   */
+  size_t size;
+
+  /**
+   * Number of parameters eaten by this operation.
+   */
+  unsigned int num_params;
+};
+
+
+/**
+ * End of query parameter specification.
+ */
+#define GNUNET_SQ_query_param_end { NULL, NULL, NULL, 0, 0 }
+
+
+/**
+ * Generate query parameter for a buffer @a ptr of
+ * @a ptr_size bytes.
+ *
+ * @param ptr pointer to the query parameter to pass
+ * @oaran ptr_size number of bytes in @a ptr
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_fixed_size (const void *ptr,
+                                  size_t ptr_size);
+
+
+
+/**
+ * Generate query parameter for a string.
+ *
+ * @param ptr pointer to the string query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_string (const char *ptr);
+
+
+/**
+ * Generate fixed-size query parameter with size determined
+ * by variable type.
+ *
+ * @param x pointer to the query parameter to pass.
+ */
+#define GNUNET_SQ_query_param_auto_from_type(x) GNUNET_SQ_query_param_fixed_size ((x), sizeof (*(x)))
+
+
+/**
+ * Generate query parameter for an RSA public key.  The
+ * database must contain a BLOB type in the respective position.
+ *
+ * @param x the query parameter to pass.
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_rsa_public_key (const struct GNUNET_CRYPTO_RsaPublicKey *x);
+
+
+/**
+ * Generate query parameter for an RSA signature.  The
+ * database must contain a BLOB type in the respective position.
+ *
+ * @param x the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x);
+
+
+/**
+ * Generate query parameter for an absolute time value.
+ * The database must store a 64-bit integer.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_absolute_time (const struct GNUNET_TIME_Absolute *x);
+
+
+/**
+ * Generate query parameter for an absolute time value.
+ * The database must store a 64-bit integer.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_absolute_time_nbo (const struct GNUNET_TIME_AbsoluteNBO *x);
+
+
+/**
+ * Generate query parameter for an uint16_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint16 (const uint16_t *x);
+
+
+/**
+ * Generate query parameter for an uint32_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint32 (const uint32_t *x);
+
+
+/**
+ * Generate query parameter for an uint16_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint64 (const uint64_t *x);
+
+
+/**
+ * Execute binding operations for a prepared statement.
+ *
+ * @param db_conn database connection
+ * @param params parameters to the statement
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_SQ_bind (sqlite3_stmt *stmt,
+                const struct GNUNET_SQ_QueryParam *params);
+
+
+/**
+ * Reset @a stmt and log error.
+ *
+ * @param dbh database handle
+ * @param stmt statement to reset
+ */
+void
+GNUNET_SQ_reset (sqlite3 *dbh,
+                 sqlite3_stmt *stmt);
+
+
+/**
+ * Extract data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+typedef int
+(*GNUNET_SQ_ResultConverter)(void *cls,
+                             sqlite3_stmt *result,
+                             unsigned int column,
+                             size_t *dst_size,
+                             void *dst);
+
+
+/**
+ * @brief Description of a DB result cell.
+ */
+struct GNUNET_SQ_ResultSpec;
+
+
+/**
+ * Function called to clean up memory allocated
+ * by a #GNUNET_SQ_ResultConverter.
+ *
+ * @param cls closure
+ */
+typedef void
+(*GNUNET_SQ_ResultCleanup)(void *cls);
+
+
+/**
+ * @brief Description of a DB result cell.
+ */
+struct GNUNET_SQ_ResultSpec
+{
+
+  /**
+   * What is the format of the result?
+   */
+  GNUNET_SQ_ResultConverter conv;
+
+  /**
+   * Function to clean up result data, NULL if cleanup is
+   * not necessary.
+   */
+  GNUNET_SQ_ResultCleanup cleaner;
+
+  /**
+   * Closure for @e conv and @e cleaner.
+   */
+  void *cls;
+
+  /**
+   * Destination for the data.
+   */
+  void *dst;
+
+  /**
+   * Allowed size for the data, 0 for variable-size
+   * (in this case, the type of @e dst is a `void **`
+   * and we need to allocate a buffer of the right size).
+   */
+  size_t dst_size;
+
+  /**
+   * Where to store actual size of the result.  If left at
+   * NULL, will be made to point to @e dst_size before
+   * @a conv is called.
+   */
+  size_t *result_size;
+
+  /**
+   * Number of parameters (columns) eaten by this operation.
+   */
+  unsigned int num_params;
+
+};
+
+
+/**
+ * End of result parameter specification.
+ *
+ * @return array last entry for the result specification to use
+ */
+#define GNUNET_SQ_result_spec_end { NULL, NULL, NULL, NULL, 0, NULL }
+
+
+/**
+ * Variable-size result expected.
+ *
+ * @param[out] dst where to store the result, allocated
+ * @param[out] sptr where to store the size of @a dst
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_variable_size (void **dst,
+                                     size_t *sptr);
+
+
+/**
+ * Fixed-size result expected.
+ *
+ * @param[out] dst where to store the result
+ * @param dst_size number of bytes in @a dst
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_fixed_size (void *dst,
+                                  size_t dst_size);
+
+
+/**
+ * We expect a fixed-size result, with size determined by the type of `* dst`
+ *
+ * @param dst point to where to store the result, type fits expected result size
+ * @return array entry for the result specification to use
+ */
+#define GNUNET_SQ_result_spec_auto_from_type(dst) GNUNET_SQ_result_spec_fixed_size ((dst), sizeof (*(dst)))
+
+
+/**
+ * Variable-size result expected.
+ *
+ * @param[out] dst where to store the result, allocated
+ * @param[out] sptr where to store the size of @a dst
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_variable_size (void **dst,
+                                     size_t *sptr);
+
+
+/**
+ * 0-terminated string expected.
+ *
+ * @param[out] dst where to store the result, allocated
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_string (char **dst);
+
+
+/**
+ * RSA public key expected.
+ *
+ * @param[out] rsa where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_rsa_public_key (struct GNUNET_CRYPTO_RsaPublicKey **rsa);
+
+
+/**
+ * RSA signature expected.
+ *
+ * @param[out] sig where to store the result;
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_rsa_signature (struct GNUNET_CRYPTO_RsaSignature **sig);
+
+
+/**
+ * Absolute time expected.
+ *
+ * @param[out] at where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_absolute_time (struct GNUNET_TIME_Absolute *at);
+
+
+/**
+ * Absolute time expected.
+ *
+ * @param[out] at where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_absolute_time_nbo (struct GNUNET_TIME_AbsoluteNBO *at);
+
+
+/**
+ * uint16_t expected.
+ *
+ * @param[out] u16 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint16 (uint16_t *u16);
+
+
+/**
+ * uint32_t expected.
+ *
+ * @param[out] u32 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint32 (uint32_t *u32);
+
+
+/**
+ * uint64_t expected.
+ *
+ * @param[out] u64 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint64 (uint64_t *u64);
+
+
+/**
+ * Extract results from a query result according to the given specification.
+ *
+ * @param result result to process
+ * @param[in,out] rs result specification to extract for
+ * @return
+ *   #GNUNET_OK if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field)
+ */
+int
+GNUNET_SQ_extract_result (sqlite3_stmt *result,
+                          struct GNUNET_SQ_ResultSpec *rs);
+
+
+/**
+ * Free all memory that was allocated in @a rs during
+ * #GNUNET_SQ_extract_result().
+ *
+ * @param rs reult specification to clean up
+ */
+void
+GNUNET_SQ_cleanup_result (struct GNUNET_SQ_ResultSpec *rs);
+
+
+#endif  /* GNUNET_SQ_LIB_H_ */
+
+/* end of include/gnunet_sq_lib.h */
diff --git a/src/include/gnunet_strings_lib.h b/src/include/gnunet_strings_lib.h
index 0328882dd..144780c82 100644
--- a/src/include/gnunet_strings_lib.h
+++ b/src/include/gnunet_strings_lib.h
@@ -360,6 +360,18 @@ GNUNET_STRINGS_base64_decode (const char *data,
 
 
 /**
+ * Convert a peer path to a human-readable string.
+ *
+ * @param pids array of PIDs to convert to a string
+ * @param num_pids length of the @a pids array
+ * @return string representing the array of @a pids
+ */
+char *
+GNUNET_STRINGS_pp2s (const struct GNUNET_PeerIdentity *pids,
+                     unsigned int num_pids);
+
+
+/**
  * Parse a path that might be an URI.
  *
  * @param path path to parse. Must be NULL-terminated.
@@ -477,7 +489,7 @@ GNUNET_STRINGS_to_address_ipv4 (const char *zt_addr,
 
 
 /**
- * Parse an address given as a string into a 
+ * Parse an address given as a string into a
  * `struct sockaddr`.
  *
  * @param addr the address
diff --git a/src/include/gnunet_util_lib.h b/src/include/gnunet_util_lib.h
index 52f5d8ab2..a9bd6c33f 100644
--- a/src/include/gnunet_util_lib.h
+++ b/src/include/gnunet_util_lib.h
@@ -38,10 +38,24 @@ extern "C"
 #endif
 #endif
 
+
+/**
+ * Largest supported message (to be precise, one byte more
+ * than the largest possible message, so tests involving
+ * this value should check for messages being smaller than
+ * this value).
+ */
+#define GNUNET_MAX_MESSAGE_SIZE 65536
+
+/**
+ * Smallest supported message.
+ */
+#define GNUNET_MIN_MESSAGE_SIZE sizeof (struct GNUNET_MessageHeader)
+
+
 #include "gnunet_crypto_lib.h"
 #include "gnunet_bandwidth_lib.h"
 #include "gnunet_bio_lib.h"
-#include "gnunet_connection_lib.h"
 #include "gnunet_client_lib.h"
 #include "gnunet_container_lib.h"
 #include "gnunet_getopt_lib.h"
@@ -55,7 +69,6 @@ extern "C"
 #include "gnunet_plugin_lib.h"
 #include "gnunet_program_lib.h"
 #include "gnunet_protocols.h"
-#include "gnunet_server_lib.h"
 #include "gnunet_service_lib.h"
 #include "gnunet_signal_lib.h"
 #include "gnunet_strings_lib.h"
diff --git a/src/include/platform.h b/src/include/platform.h
index add58821f..6095d0258 100644
--- a/src/include/platform.h
+++ b/src/include/platform.h
@@ -110,6 +110,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <errno.h>
 #include <signal.h>
 #include <libgen.h>
diff --git a/src/integration-tests/.gitignore b/src/integration-tests/.gitignore
index c4d1568f0..46915b2a4 100644
--- a/src/integration-tests/.gitignore
+++ b/src/integration-tests/.gitignore
@@ -1,2 +1,10 @@
 gnunet_testing.py
 gnunet_pyexpect.py
+gnunet_pyexpect.pyc
+gnunet_testing.pyc
+test_integration_bootstrap_and_connect.py
+test_integration_clique.py
+test_integration_disconnect_nat.py
+test_integration_disconnect.py
+test_integration_reconnect_nat.py
+test_integration_reconnect.py
diff --git a/src/integration-tests/Makefile.am b/src/integration-tests/Makefile.am
index 4e96ee173..6fff0b407 100644
--- a/src/integration-tests/Makefile.am
+++ b/src/integration-tests/Makefile.am
@@ -27,7 +27,7 @@ endif
 
 if HAVE_MHD
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 if HAVE_LIBGNURL
 TESTS = \
  $(check_SCRIPTS)
diff --git a/src/integration-tests/confs/test_defaults.conf b/src/integration-tests/confs/test_defaults.conf
index 8d3356cbc..1be582650 100644
--- a/src/integration-tests/confs/test_defaults.conf
+++ b/src/integration-tests/confs/test_defaults.conf
@@ -1,5 +1,5 @@
-@INLINE@ ../../contrib/no_forcestart.conf
-@INLINE@ ../../contrib/no_autostart_above_core.conf
+@INLINE@ ../../../contrib/no_forcestart.conf
+@INLINE@ ../../../contrib/no_autostart_above_core.conf
 
 [fs]
 FORCESTART = YES
diff --git a/src/json/json.c b/src/json/json.c
index a2d1a9608..c182a02f4 100644
--- a/src/json/json.c
+++ b/src/json/json.c
@@ -1,6 +1,6 @@
 /*
   This file is part of GNUnet
-  Copyright (C) 2014, 2015, 2016 GNUnet e.V.
+  Copyright (C) 2014-2017 GNUnet e.V.
 
   GNUnet is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -82,13 +82,78 @@ GNUNET_JSON_parse (const json_t *root,
 void
 GNUNET_JSON_parse_free (struct GNUNET_JSON_Specification *spec)
 {
-  unsigned int i;
-
-  for (i=0;NULL != spec[i].parser;i++)
+  for (unsigned int i=0;NULL != spec[i].parser;i++)
     if (NULL != spec[i].cleaner)
       spec[i].cleaner (spec[i].cls,
                        &spec[i]);
 }
 
 
+/**
+ * Set an option with a JSON value from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the 'json_t *')
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_json (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+          void *scls,
+          const char *option,
+          const char *value)
+{
+  json_t **json = scls;
+  json_error_t error;
+
+  *json = json_loads (value,
+                      JSON_REJECT_DUPLICATES,
+                      &error);
+  if (NULL == *json)
+  {
+    FPRINTF (stderr,
+             _("Failed to parse JSON in option `%s': %s (%s)\n"),
+             option,
+             error.text,
+             error.source);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify a JSON input value.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the JSON specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_JSON_getopt (char shortName,
+                    const char *name,
+                    const char *argumentHelp,
+                    const char *description,
+                    json_t **json)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_json,
+    .scls = (void *) json
+  };
+
+  return clo;
+}
+
+
 /* end of json.c */
diff --git a/src/multicast/.gitignore b/src/multicast/.gitignore
index 0f386fb40..43752ec4b 100644
--- a/src/multicast/.gitignore
+++ b/src/multicast/.gitignore
@@ -1,2 +1,5 @@
 gnunet-service-multicast
 gnunet-multicast
+test_multicast 
+test_multicast_multipeer
+test_multicast_2peers
diff --git a/src/multicast/Makefile.am b/src/multicast/Makefile.am
index b1f422765..1a61abb09 100644
--- a/src/multicast/Makefile.am
+++ b/src/multicast/Makefile.am
@@ -33,7 +33,8 @@ bin_PROGRAMS = \
  gnunet-multicast
 
 libexec_PROGRAMS = \
- gnunet-service-multicast
+ gnunet-service-multicast \
+ $(EXP_LIBEXEC)
 
 gnunet_multicast_SOURCES = \
  gnunet-multicast.c
@@ -49,13 +50,13 @@ gnunet_service_multicast_LDADD = \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(GN_LIBINTL)
 
-
 check_PROGRAMS = \
  test_multicast \
- test_multicast_multipeer
+ test_multicast_2peers
+# test_multicast_multipeer
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@}; export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH; unset XDG_DATA_HOME; unset XDG_CONFIG_HOME; export GNUNET_FORCE_LOG=';;;;INFO';
 TESTS = $(check_PROGRAMS)
 endif
 
@@ -66,9 +67,9 @@ test_multicast_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-test_multicast_multipeer_SOURCE = \
- test_multicast_multipeer.c
-test_multicast_multipeer_LDADD = \
+test_multicast_2peers_SOURCE = \
+ test_multicast_2peers.c
+test_multicast_2peers_LDADD = \
   libgnunetmulticast.la \
   $(top_builddir)/src/testbed/libgnunettestbed.la \
   $(top_builddir)/src/util/libgnunetutil.la
diff --git a/src/multicast/gnunet-service-multicast.c b/src/multicast/gnunet-service-multicast.c
index de65e0ab7..9683efcff 100644
--- a/src/multicast/gnunet-service-multicast.c
+++ b/src/multicast/gnunet-service-multicast.c
@@ -163,6 +163,16 @@ struct Channel
   struct GNUNET_PeerIdentity peer;
 
   /**
+   * Current window size, set by cadet_notify_window_change()
+   */
+  int32_t window_size;
+
+  /**
+   * Is the connection established?
+   */
+  int8_t is_connected;
+
+  /**
    * Is the remote peer admitted to the group?
    * @see enum JoinStatus
    */
@@ -336,6 +346,17 @@ struct ReplayRequestKey
 };
 
 
+static struct Channel *
+cadet_channel_create (struct Group *grp, struct GNUNET_PeerIdentity *peer);
+
+static void
+cadet_channel_destroy (struct Channel *chn);
+
+static void
+client_send_join_decision (struct Member *mem,
+                           const struct MulticastJoinDecisionMessageHeader *hdcsn);
+
+
 /**
  * Task run during shutdown.
  *
@@ -443,6 +464,9 @@ replay_key_hash (uint64_t fragment_id, uint64_t message_id,
 static int
 replay_req_remove_cadet (struct Channel *chn)
 {
+  if (NULL == chn || NULL == chn->group)
+    return GNUNET_SYSERR;
+
   struct GNUNET_CONTAINER_MultiHashMap *
     grp_replay_req = GNUNET_CONTAINER_multihashmap_get (replay_req_cadet,
                                                         &chn->group->pub_key_hash);
@@ -497,7 +521,7 @@ replay_req_remove_client (struct Group *grp, struct GNUNET_SERVICE_Client *clien
   {
     if (c == client)
     {
-      GNUNET_CONTAINER_multihashmap_remove (replay_req_client, &key, client);
+      GNUNET_CONTAINER_multihashmap_remove (grp_replay_req, &key, client);
       GNUNET_CONTAINER_multihashmap_iterator_destroy (it);
       return GNUNET_YES;
     }
@@ -653,6 +677,9 @@ client_send_origin (struct GNUNET_HashCode *pub_key_hash,
 static void
 client_send_ack (struct GNUNET_HashCode *pub_key_hash)
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Sending message ACK to client.\n");
+
   static struct GNUNET_MessageHeader *msg = NULL;
   if (NULL == msg)
   {
@@ -672,36 +699,6 @@ struct CadetTransmitClosure
 
 
 /**
- * CADET is ready to transmit a message.
- */
-size_t
-cadet_notify_transmit_ready (void *cls, size_t buf_size, void *buf)
-{
-  if (0 == buf_size)
-  {
-    /* FIXME: connection closed */
-    return 0;
-  }
-  struct CadetTransmitClosure *tcls = cls;
-  struct Channel *chn = tcls->chn;
-  uint16_t msg_size = ntohs (tcls->msg->size);
-  GNUNET_assert (msg_size <= buf_size);
-  GNUNET_memcpy (buf, tcls->msg, msg_size);
-  GNUNET_free (tcls);
-
-  if (0 == chn->msgs_pending)
-  {
-    GNUNET_break (0);
-  }
-  else if (0 == --chn->msgs_pending)
-  {
-    client_send_ack (&chn->group_pub_hash);
-  }
-  return msg_size;
-}
-
-
-/**
  * Send a message to a CADET channel.
  *
  * @param chn  Channel.
@@ -710,53 +707,22 @@ cadet_notify_transmit_ready (void *cls, size_t buf_size, void *buf)
 static void
 cadet_send_channel (struct Channel *chn, const struct GNUNET_MessageHeader *msg)
 {
-  uint16_t msg_size = ntohs (msg->size);
-  struct GNUNET_MessageHeader *msg_copy = GNUNET_malloc (msg_size);
-  GNUNET_memcpy (msg_copy, msg, msg_size);
-
-  struct CadetTransmitClosure *tcls = GNUNET_malloc (sizeof (*tcls));
-  tcls->chn = chn;
-  tcls->msg = msg_copy;
-
-  chn->msgs_pending++;
-  chn->tmit_handle
-    = GNUNET_CADET_notify_transmit_ready (chn->channel, GNUNET_NO,
-                                          GNUNET_TIME_UNIT_FOREVER_REL,
-                                          msg_size,
-                                          &cadet_notify_transmit_ready,
-                                          tcls);
-  GNUNET_assert (NULL != chn->tmit_handle);
-}
+  struct GNUNET_MQ_Envelope *
+    env = GNUNET_MQ_msg_copy (msg);
 
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (chn->channel), env);
 
-/**
- * Create new outgoing CADET channel.
- *
- * @param peer
- *        Peer to connect to.
- * @param group_pub_key
- *        Public key of group the channel belongs to.
- * @param group_pub_hash
- *        Hash of @a group_pub_key.
- *
- * @return Channel.
- */
-static struct Channel *
-cadet_channel_create (struct Group *grp, struct GNUNET_PeerIdentity *peer)
-{
-  struct Channel *chn = GNUNET_malloc (sizeof (*chn));
-  chn->group = grp;
-  chn->group_pub_key = grp->pub_key;
-  chn->group_pub_hash = grp->pub_key_hash;
-  chn->peer = *peer;
-  chn->direction = DIR_OUTGOING;
-  chn->join_status = JOIN_WAITING;
-  chn->channel = GNUNET_CADET_channel_create (cadet, chn, &chn->peer,
-                                              &grp->cadet_port_hash,
-                                              GNUNET_CADET_OPTION_RELIABLE);
-  GNUNET_CONTAINER_multihashmap_put (channels_out, &chn->group_pub_hash, chn,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-  return chn;
+  if (0 < chn->window_size)
+  {
+    client_send_ack (&chn->group_pub_hash);
+  }
+  else
+  {
+    chn->msgs_pending++;
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "%p Queuing message. Pending messages: %u\n",
+                chn, chn->msgs_pending);
+  }
 }
 
 
@@ -787,7 +753,7 @@ cadet_send_join_decision_cb (void *cls,
   const struct MulticastJoinDecisionMessageHeader *hdcsn = cls;
   struct Channel *chn = channel;
 
-  const struct MulticastJoinDecisionMessage *dcsn = 
+  const struct MulticastJoinDecisionMessage *dcsn =
     (struct MulticastJoinDecisionMessage *) &hdcsn[1];
 
   if (0 == memcmp (&hdcsn->member_pub_key, &chn->member_pub_key, sizeof (chn->member_pub_key))
@@ -870,31 +836,74 @@ cadet_send_parents (struct GNUNET_HashCode *pub_key_hash,
 
 
 /**
- * New incoming CADET channel.
+ * CADET channel connect handler.
+ *
+ * @see GNUNET_CADET_ConnectEventHandler()
  */
 static void *
-cadet_notify_channel_new (void *cls,
-                          struct GNUNET_CADET_Channel *channel,
-                          const struct GNUNET_PeerIdentity *initiator,
-                          const struct GNUNET_HashCode *port,
-                          enum GNUNET_CADET_ChannelOption options)
+cadet_notify_connect (void *cls,
+                      struct GNUNET_CADET_Channel *channel,
+                      const struct GNUNET_PeerIdentity *source)
 {
-  return NULL;
+  struct Channel *chn = GNUNET_malloc (sizeof *chn);
+  chn->group = cls;
+  chn->channel = channel;
+  chn->direction = DIR_INCOMING;
+  chn->join_status = JOIN_NOT_ASKED;
+
+  GNUNET_CONTAINER_multihashmap_put (channels_in, &chn->group_pub_hash, chn,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  return chn;
 }
 
 
 /**
- * CADET channel is being destroyed.
+ * CADET window size change handler.
+ *
+ * @see GNUNET_CADET_WindowSizeEventHandler()
  */
 static void
-cadet_notify_channel_end (void *cls,
-                          const struct GNUNET_CADET_Channel *channel,
-                          void *ctx)
+cadet_notify_window_change (void *cls,
+                            const struct GNUNET_CADET_Channel *channel,
+                            int window_size)
 {
-  if (NULL == ctx)
+  struct Channel *chn = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "%p Window size changed to %d.  Pending messages: %u\n",
+              chn, window_size, chn->msgs_pending);
+
+  chn->is_connected = GNUNET_YES;
+  chn->window_size = (int32_t) window_size;
+
+  for (int i = 0; i < window_size; i++)
+  {
+    if (0 < chn->msgs_pending)
+    {
+      client_send_ack (&chn->group_pub_hash);
+      chn->msgs_pending--;
+    }
+    else
+    {
+      break;
+    }
+  }
+}
+
+
+/**
+ * CADET channel disconnect handler.
+ *
+ * @see GNUNET_CADET_DisconnectEventHandler()
+ */
+static void
+cadet_notify_disconnect (void *cls,
+                         const struct GNUNET_CADET_Channel *channel)
+{
+  if (NULL == cls)
     return;
 
-  struct Channel *chn = ctx;
+  struct Channel *chn = cls;
   if (NULL != chn->group)
   {
     if (GNUNET_NO == chn->group->is_origin)
@@ -905,12 +914,331 @@ cadet_notify_channel_end (void *cls,
     }
   }
 
-  while (GNUNET_YES == replay_req_remove_cadet (chn));
+  int ret;
+  do
+  {
+    ret = replay_req_remove_cadet (chn);
+  }
+  while (GNUNET_YES == ret);
 
   GNUNET_free (chn);
 }
 
 
+static int
+check_cadet_join_request (void *cls,
+                          const struct MulticastJoinRequestMessage *req)
+{
+  struct Channel *chn = cls;
+
+  if (NULL == chn
+      || JOIN_NOT_ASKED != chn->join_status)
+  {
+    return GNUNET_SYSERR;
+  }
+
+  uint16_t size = ntohs (req->header.size);
+  if (size < sizeof (*req))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (ntohl (req->purpose.size) != (size
+                                    - sizeof (req->header)
+                                    - sizeof (req->reserved)
+                                    - sizeof (req->signature)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_REQUEST,
+                                  &req->purpose, &req->signature,
+                                  &req->member_pub_key))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming join request message from CADET.
+ */
+static void
+handle_cadet_join_request (void *cls,
+                           const struct MulticastJoinRequestMessage *req)
+{
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+
+  struct GNUNET_HashCode group_pub_hash;
+  GNUNET_CRYPTO_hash (&req->group_pub_key, sizeof (req->group_pub_key), &group_pub_hash);
+  chn->group_pub_key = req->group_pub_key;
+  chn->group_pub_hash = group_pub_hash;
+  chn->member_pub_key = req->member_pub_key;
+  chn->peer = req->peer;
+  chn->join_status = JOIN_WAITING;
+
+  client_send_all (&group_pub_hash, &req->header);
+}
+
+
+static int
+check_cadet_join_decision (void *cls,
+                           const struct MulticastJoinDecisionMessageHeader *hdcsn)
+{
+  uint16_t size = ntohs (hdcsn->header.size);
+  if (size < sizeof (struct MulticastJoinDecisionMessageHeader) +
+             sizeof (struct MulticastJoinDecisionMessage))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  struct Channel *chn = cls;
+  if (NULL == chn)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (NULL == chn->group || GNUNET_NO != chn->group->is_origin)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  switch (chn->join_status)
+  {
+  case JOIN_REFUSED:
+    return GNUNET_SYSERR;
+
+  case JOIN_ADMITTED:
+    return GNUNET_OK;
+
+  case JOIN_NOT_ASKED:
+  case JOIN_WAITING:
+    break;
+  }
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming join decision message from CADET.
+ */
+static void
+handle_cadet_join_decision (void *cls,
+                            const struct MulticastJoinDecisionMessageHeader *hdcsn)
+{
+  const struct MulticastJoinDecisionMessage *
+    dcsn = (const struct MulticastJoinDecisionMessage *) &hdcsn[1];
+
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+
+  // FIXME: do we need to copy chn->peer or compare it with hdcsn->peer?
+  struct Member *mem = (struct Member *) chn->group;
+  client_send_join_decision (mem, hdcsn);
+  if (GNUNET_YES == ntohl (dcsn->is_admitted))
+  {
+    chn->join_status = JOIN_ADMITTED;
+  }
+  else
+  {
+    chn->join_status = JOIN_REFUSED;
+    cadet_channel_destroy (chn);
+  }
+}
+
+
+static int
+check_cadet_message (void *cls,
+                     const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  uint16_t size = ntohs (msg->header.size);
+  if (size < sizeof (*msg))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  struct Channel *chn = cls;
+  if (NULL == chn)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (ntohl (msg->purpose.size) != (size
+                                    - sizeof (msg->header)
+                                    - sizeof (msg->hop_counter)
+                                    - sizeof (msg->signature)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_MESSAGE,
+                                  &msg->purpose, &msg->signature,
+                                  &chn->group_pub_key))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming multicast message from CADET.
+ */
+static void
+handle_cadet_message (void *cls,
+                      const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+  client_send_all (&chn->group_pub_hash, &msg->header);
+}
+
+
+static int
+check_cadet_request (void *cls,
+                     const struct GNUNET_MULTICAST_RequestHeader *req)
+{
+  uint16_t size = ntohs (req->header.size);
+  if (size < sizeof (*req))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  struct Channel *chn = cls;
+  if (NULL == chn)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (ntohl (req->purpose.size) != (size
+                                    - sizeof (req->header)
+                                    - sizeof (req->member_pub_key)
+                                    - sizeof (req->signature)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_REQUEST,
+                                  &req->purpose, &req->signature,
+                                  &req->member_pub_key))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming multicast request message from CADET.
+ */
+static void
+handle_cadet_request (void *cls,
+                      const struct GNUNET_MULTICAST_RequestHeader *req)
+{
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+  client_send_origin (&chn->group_pub_hash, &req->header);
+}
+
+
+static int
+check_cadet_replay_request (void *cls,
+                            const struct MulticastReplayRequestMessage *req)
+{
+  uint16_t size = ntohs (req->header.size);
+  if (size < sizeof (*req))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  struct Channel *chn = cls;
+  if (NULL == chn)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming multicast replay request from CADET.
+ */
+static void
+handle_cadet_replay_request (void *cls,
+                             const struct MulticastReplayRequestMessage *req)
+{
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+
+  struct MulticastReplayRequestMessage rep = *req;
+  GNUNET_memcpy (&rep.member_pub_key, &chn->member_pub_key, sizeof (chn->member_pub_key));
+
+  struct GNUNET_CONTAINER_MultiHashMap *
+    grp_replay_req = GNUNET_CONTAINER_multihashmap_get (replay_req_cadet,
+                                                        &chn->group->pub_key_hash);
+  if (NULL == grp_replay_req)
+  {
+    grp_replay_req = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
+    GNUNET_CONTAINER_multihashmap_put (replay_req_cadet,
+                                       &chn->group->pub_key_hash, grp_replay_req,
+                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
+  }
+  struct GNUNET_HashCode key_hash;
+  replay_key_hash (rep.fragment_id, rep.message_id, rep.fragment_offset,
+                   rep.flags, &key_hash);
+  GNUNET_CONTAINER_multihashmap_put (grp_replay_req, &key_hash, chn,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+
+  client_send_random (&chn->group_pub_hash, &rep.header);
+}
+
+
+static int
+check_cadet_replay_response (void *cls,
+                             const struct MulticastReplayResponseMessage *res)
+{
+  struct Channel *chn = cls;
+  if (NULL == chn)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Incoming multicast replay response from CADET.
+ */
+static void
+handle_cadet_replay_response (void *cls,
+                              const struct MulticastReplayResponseMessage *res)
+{
+  struct Channel *chn = cls;
+  GNUNET_CADET_receive_done (chn->channel);
+
+  /* @todo FIXME: got replay error response, send request to other members */
+}
+
+
 static void
 group_set_cadet_port_hash (struct Group *grp)
 {
@@ -925,6 +1253,78 @@ group_set_cadet_port_hash (struct Group *grp)
 }
 
 
+
+/**
+ * Create new outgoing CADET channel.
+ *
+ * @param peer
+ *        Peer to connect to.
+ * @param group_pub_key
+ *        Public key of group the channel belongs to.
+ * @param group_pub_hash
+ *        Hash of @a group_pub_key.
+ *
+ * @return Channel.
+ */
+static struct Channel *
+cadet_channel_create (struct Group *grp, struct GNUNET_PeerIdentity *peer)
+{
+  struct Channel *chn = GNUNET_malloc (sizeof (*chn));
+  chn->group = grp;
+  chn->group_pub_key = grp->pub_key;
+  chn->group_pub_hash = grp->pub_key_hash;
+  chn->peer = *peer;
+  chn->direction = DIR_OUTGOING;
+  chn->is_connected = GNUNET_NO;
+  chn->join_status = JOIN_WAITING;
+
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_var_size (cadet_message,
+                           GNUNET_MESSAGE_TYPE_MULTICAST_MESSAGE,
+                           struct GNUNET_MULTICAST_MessageHeader,
+                           chn),
+
+    GNUNET_MQ_hd_var_size (cadet_join_decision,
+                           GNUNET_MESSAGE_TYPE_MULTICAST_JOIN_DECISION,
+                           struct MulticastJoinDecisionMessageHeader,
+                           chn),
+
+    GNUNET_MQ_hd_var_size (cadet_replay_request,
+                           GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_REQUEST,
+                           struct MulticastReplayRequestMessage,
+                           chn),
+
+    GNUNET_MQ_hd_var_size (cadet_replay_response,
+                           GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_RESPONSE,
+                           struct MulticastReplayResponseMessage,
+                           chn),
+
+    GNUNET_MQ_handler_end ()
+  };
+
+  chn->channel = GNUNET_CADET_channel_create (cadet, chn, &chn->peer,
+                                              &grp->cadet_port_hash,
+                                              GNUNET_CADET_OPTION_RELIABLE,
+                                              cadet_notify_window_change,
+                                              cadet_notify_disconnect,
+                                              cadet_handlers);
+  GNUNET_CONTAINER_multihashmap_put (channels_out, &chn->group_pub_hash, chn,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  return chn;
+}
+
+
+/**
+ * Destroy outgoing CADET channel.
+ */
+static void
+cadet_channel_destroy (struct Channel *chn)
+{
+  GNUNET_CADET_channel_destroy (chn->channel);
+  GNUNET_CONTAINER_multihashmap_remove_all (channels_out, &chn->group_pub_hash);
+  GNUNET_free (chn);
+}
+
 /**
  * Handle a connecting client starting an origin.
  */
@@ -961,8 +1361,44 @@ handle_client_origin_start (void *cls,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
 
     group_set_cadet_port_hash (grp);
-    orig->cadet_port = GNUNET_CADET_open_port (cadet, &grp->cadet_port_hash,
-                                               cadet_notify_channel_new, NULL);
+
+    struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+      GNUNET_MQ_hd_var_size (cadet_message,
+                             GNUNET_MESSAGE_TYPE_MULTICAST_MESSAGE,
+                             struct GNUNET_MULTICAST_MessageHeader,
+                             grp),
+
+      GNUNET_MQ_hd_var_size (cadet_request,
+                             GNUNET_MESSAGE_TYPE_MULTICAST_REQUEST,
+                             struct GNUNET_MULTICAST_RequestHeader,
+                             grp),
+
+      GNUNET_MQ_hd_var_size (cadet_join_request,
+                             GNUNET_MESSAGE_TYPE_MULTICAST_JOIN_REQUEST,
+                             struct MulticastJoinRequestMessage,
+                             grp),
+
+      GNUNET_MQ_hd_var_size (cadet_replay_request,
+                             GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_REQUEST,
+                             struct MulticastReplayRequestMessage,
+                             grp),
+
+      GNUNET_MQ_hd_var_size (cadet_replay_response,
+                             GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_RESPONSE,
+                             struct MulticastReplayResponseMessage,
+                             grp),
+
+      GNUNET_MQ_handler_end ()
+    };
+
+
+    orig->cadet_port = GNUNET_CADET_open_port (cadet,
+                                               &grp->cadet_port_hash,
+                                               cadet_notify_connect,
+                                               NULL,
+                                               cadet_notify_window_change,
+                                               cadet_notify_disconnect,
+                                               cadet_handlers);
   }
   else
   {
@@ -1258,10 +1694,8 @@ handle_client_multicast_message (void *cls,
   }
 
   client_send_all (&grp->pub_key_hash, &out->header);
-  if (0 == cadet_send_children (&grp->pub_key_hash, &out->header))
-  {
-    client_send_ack (&grp->pub_key_hash);
-  }
+  cadet_send_children (&grp->pub_key_hash, &out->header);
+  client_send_ack (&grp->pub_key_hash);
   GNUNET_free (out);
 
   GNUNET_SERVICE_client_continue (client);
@@ -1544,278 +1978,6 @@ handle_client_replay_response (void *cls,
 
 
 /**
- * Incoming join request message from CADET.
- */
-int
-cadet_recv_join_request (void *cls,
-                         struct GNUNET_CADET_Channel *channel,
-                         void **ctx,
-                         const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done(channel);
-  const struct MulticastJoinRequestMessage *
-    req = (const struct MulticastJoinRequestMessage *) m;
-  uint16_t size = ntohs (m->size);
-  if (size < sizeof (*req))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL != *ctx)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (ntohl (req->purpose.size) != (size
-                                    - sizeof (req->header)
-                                    - sizeof (req->reserved)
-                                    - sizeof (req->signature)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_REQUEST,
-                                  &req->purpose, &req->signature,
-                                  &req->member_pub_key))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-
-  struct GNUNET_HashCode group_pub_hash;
-  GNUNET_CRYPTO_hash (&req->group_pub_key, sizeof (req->group_pub_key), &group_pub_hash);
-
-  struct Channel *chn = GNUNET_malloc (sizeof *chn);
-  chn->channel = channel;
-  chn->group_pub_key = req->group_pub_key;
-  chn->group_pub_hash = group_pub_hash;
-  chn->member_pub_key = req->member_pub_key;
-  chn->peer = req->peer;
-  chn->join_status = JOIN_WAITING;
-  GNUNET_CONTAINER_multihashmap_put (channels_in, &chn->group_pub_hash, chn,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-  *ctx = chn;
-
-  client_send_all (&group_pub_hash, m);
-  return GNUNET_OK;
-}
-
-
-/**
- * Incoming join decision message from CADET.
- */
-int
-cadet_recv_join_decision (void *cls,
-                          struct GNUNET_CADET_Channel *channel,
-                          void **ctx,
-                          const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done (channel);
-  const struct MulticastJoinDecisionMessageHeader *
-    hdcsn = (const struct MulticastJoinDecisionMessageHeader *) m;
-  const struct MulticastJoinDecisionMessage *
-    dcsn = (const struct MulticastJoinDecisionMessage *) &hdcsn[1];
-  uint16_t size = ntohs (m->size);
-  if (size < sizeof (struct MulticastJoinDecisionMessageHeader) +
-             sizeof (struct MulticastJoinDecisionMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  struct Channel *chn = *ctx;
-  if (NULL == chn)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL == chn->group || GNUNET_NO != chn->group->is_origin)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  switch (chn->join_status)
-  {
-  case JOIN_REFUSED:
-    return GNUNET_SYSERR;
-
-  case JOIN_ADMITTED:
-    return GNUNET_OK;
-
-  case JOIN_NOT_ASKED:
-  case JOIN_WAITING:
-    break;
-  }
-
-  // FIXME: do we need to copy chn->peer or compare it with hdcsn->peer?
-  struct Member *mem = (struct Member *) chn->group;
-  client_send_join_decision (mem, hdcsn);
-  if (GNUNET_YES == ntohl (dcsn->is_admitted))
-  {
-    chn->join_status = JOIN_ADMITTED;
-    return GNUNET_OK;
-  }
-  else
-  {
-    chn->join_status = JOIN_REFUSED;
-    return GNUNET_SYSERR;
-  }
-}
-
-/**
- * Incoming multicast message from CADET.
- */
-int
-cadet_recv_message (void *cls,
-                    struct GNUNET_CADET_Channel *channel,
-                    void **ctx,
-                    const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done(channel);
-  const struct GNUNET_MULTICAST_MessageHeader *
-    msg = (const struct GNUNET_MULTICAST_MessageHeader *) m;
-  uint16_t size = ntohs (m->size);
-  if (size < sizeof (*msg))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  struct Channel *chn = *ctx;
-  if (NULL == chn)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (ntohl (msg->purpose.size) != (size
-                                    - sizeof (msg->header)
-                                    - sizeof (msg->hop_counter)
-                                    - sizeof (msg->signature)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_MESSAGE,
-                                  &msg->purpose, &msg->signature,
-                                  &chn->group_pub_key))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-
-  client_send_all (&chn->group_pub_hash, m);
-  return GNUNET_OK;
-}
-
-
-/**
- * Incoming multicast request message from CADET.
- */
-int
-cadet_recv_request (void *cls,
-                    struct GNUNET_CADET_Channel *channel,
-                    void **ctx,
-                    const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done(channel);
-  const struct GNUNET_MULTICAST_RequestHeader *
-    req = (const struct GNUNET_MULTICAST_RequestHeader *) m;
-  uint16_t size = ntohs (m->size);
-  if (size < sizeof (*req))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  struct Channel *chn = *ctx;
-  if (NULL == chn)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (ntohl (req->purpose.size) != (size
-                                    - sizeof (req->header)
-                                    - sizeof (req->member_pub_key)
-                                    - sizeof (req->signature)))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_MULTICAST_REQUEST,
-                                  &req->purpose, &req->signature,
-                                  &req->member_pub_key))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-
-  client_send_origin (&chn->group_pub_hash, m);
-  return GNUNET_OK;
-}
-
-
-/**
- * Incoming multicast replay request from CADET.
- */
-int
-cadet_recv_replay_request (void *cls,
-                           struct GNUNET_CADET_Channel *channel,
-                           void **ctx,
-                           const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done(channel);
-  struct MulticastReplayRequestMessage rep;
-  uint16_t size = ntohs (m->size);
-  if (size < sizeof (rep))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  struct Channel *chn = *ctx;
-
-  GNUNET_memcpy (&rep, m, sizeof (rep));
-  GNUNET_memcpy (&rep.member_pub_key, &chn->member_pub_key, sizeof (chn->member_pub_key));
-
-  struct GNUNET_CONTAINER_MultiHashMap *
-    grp_replay_req = GNUNET_CONTAINER_multihashmap_get (replay_req_cadet,
-                                                        &chn->group->pub_key_hash);
-  if (NULL == grp_replay_req)
-  {
-    grp_replay_req = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
-    GNUNET_CONTAINER_multihashmap_put (replay_req_cadet,
-                                       &chn->group->pub_key_hash, grp_replay_req,
-                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
-  }
-  struct GNUNET_HashCode key_hash;
-  replay_key_hash (rep.fragment_id, rep.message_id, rep.fragment_offset,
-                   rep.flags, &key_hash);
-  GNUNET_CONTAINER_multihashmap_put (grp_replay_req, &key_hash, chn,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-  client_send_random (&chn->group_pub_hash, &rep.header);
-  return GNUNET_OK;
-}
-
-
-/**
- * Incoming multicast replay response from CADET.
- */
-int
-cadet_recv_replay_response (void *cls,
-                            struct GNUNET_CADET_Channel *channel,
-                            void **ctx,
-                            const struct GNUNET_MessageHeader *m)
-{
-  GNUNET_CADET_receive_done(channel);
-  //struct Channel *chn = *ctx;
-
-  /* @todo FIXME: got replay error response, send request to other members */
-
-  return GNUNET_OK;
-}
-
-
-/**
  * A new client connected.
  *
  * @param cls NULL
@@ -1899,32 +2061,6 @@ client_notify_disconnect (void *cls,
 
 
 /**
- * Message handlers for CADET.
- */
-static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-  { cadet_recv_join_request,
-    GNUNET_MESSAGE_TYPE_MULTICAST_JOIN_REQUEST, 0 },
-
-  { cadet_recv_join_decision,
-    GNUNET_MESSAGE_TYPE_MULTICAST_JOIN_DECISION, 0 },
-
-  { cadet_recv_message,
-    GNUNET_MESSAGE_TYPE_MULTICAST_MESSAGE, 0 },
-
-  { cadet_recv_request,
-    GNUNET_MESSAGE_TYPE_MULTICAST_REQUEST, 0 },
-
-  { cadet_recv_replay_request,
-    GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_REQUEST, 0 },
-
-  { cadet_recv_replay_response,
-    GNUNET_MESSAGE_TYPE_MULTICAST_REPLAY_RESPONSE, 0 },
-
-  { NULL, 0, 0 }
-};
-
-
-/**
  * Service started.
  *
  * @param cls closure
@@ -1949,9 +2085,8 @@ run (void *cls,
   replay_req_cadet = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
   replay_req_client = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
 
-  cadet = GNUNET_CADET_connect (cfg, NULL,
-                                cadet_notify_channel_end,
-                                cadet_handlers);
+  cadet = GNUNET_CADET_connect (cfg);
+
   GNUNET_assert (NULL != cadet);
 
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
diff --git a/src/multicast/test_multicast.c b/src/multicast/test_multicast.c
index 1e3a4922b..7e9b51e23 100644
--- a/src/multicast/test_multicast.c
+++ b/src/multicast/test_multicast.c
@@ -484,6 +484,10 @@ member_recv_message (void *cls,
 
   switch (test)
   {
+  case TEST_ORIGIN_TO_ALL:
+    test = TEST_ORIGIN_TO_ALL_RECV;
+    break;
+
   case TEST_ORIGIN_TO_ALL_RECV:
     // Test 6 starts here
     member_to_origin ();
@@ -523,6 +527,11 @@ origin_recv_message (void *cls,
     test = TEST_ORIGIN_TO_ALL_RECV;
     break;
 
+  case TEST_ORIGIN_TO_ALL_RECV:
+    // Test 6 starts here
+    member_to_origin ();
+    break;
+
   default:
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Invalid test #%d in origin_recv_message()\n", test);
diff --git a/src/multicast/test_multicast.conf b/src/multicast/test_multicast.conf
index 675776bbc..3081aeecc 100644
--- a/src/multicast/test_multicast.conf
+++ b/src/multicast/test_multicast.conf
@@ -1,4 +1,12 @@
+[testbed]
+HOSTNAME = localhost
+
 [arm]
 GLOBAL_POSTFIX=-L ERROR
 
-#PREFIX = sakura -t test-multicast -e cgdb --args
+[multicast]
+#PREFIX = xterm -T peer -e gdb --args
+UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-multicast.sock
+
+[vpn]
+AUTOSTART = NO
diff --git a/src/multicast/test_multicast_2peers.c b/src/multicast/test_multicast_2peers.c
new file mode 100644
index 000000000..058533e50
--- /dev/null
+++ b/src/multicast/test_multicast_2peers.c
@@ -0,0 +1,511 @@
+/*
+ * This file is part of GNUnet
+ * Copyright (C) 2013 GNUnet e.V.
+ *
+ * GNUnet is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 3, or (at your
+ * option) any later version.
+ *
+ * GNUnet is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GNUnet; see the file COPYING.  If not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file multicast/test_multicast_2peers.c
+ * @brief Tests for the Multicast API with two peers doing the ping
+ *        pong test.
+ * @author xrs
+ */
+
+#include <inttypes.h>
+
+#include "platform.h"
+#include "gnunet_crypto_lib.h"
+#include "gnunet_common.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testbed_service.h"
+#include "gnunet_multicast_service.h"
+
+#define NUM_PEERS 2
+
+static struct GNUNET_TESTBED_Operation *op0;
+static struct GNUNET_TESTBED_Operation *op1;
+static struct GNUNET_TESTBED_Operation *pi_op0;
+static struct GNUNET_TESTBED_Operation *pi_op1;
+
+static struct GNUNET_TESTBED_Peer **peers;
+const struct GNUNET_PeerIdentity *peer_id[2];
+
+static struct GNUNET_SCHEDULER_Task *timeout_tid;
+
+static struct GNUNET_MULTICAST_Origin *origin;
+static struct GNUNET_MULTICAST_Member *member;
+
+struct GNUNET_CRYPTO_EddsaPrivateKey *group_key;
+struct GNUNET_CRYPTO_EddsaPublicKey group_pub_key;
+
+struct GNUNET_CRYPTO_EcdsaPrivateKey *member_key;
+struct GNUNET_CRYPTO_EcdsaPublicKey member_pub_key;
+
+/**
+ * Global result for testcase.
+ */
+static int result;
+
+
+/**
+ * Function run on CTRL-C or shutdown (i.e. success/timeout/etc.).
+ * Cleans up.
+ */
+static void
+shutdown_task (void *cls)
+{
+  if (NULL != op0)
+  {
+    GNUNET_TESTBED_operation_done (op0);
+    op0 = NULL;
+  }
+  if (NULL != op1)
+  {
+    GNUNET_TESTBED_operation_done (op1);
+    op1 = NULL;
+  }
+  if (NULL != pi_op0)
+  {
+    GNUNET_TESTBED_operation_done (pi_op0);
+    pi_op0 = NULL;
+  }
+  if (NULL != pi_op1)
+  {
+    GNUNET_TESTBED_operation_done (pi_op1);
+    pi_op1 = NULL;
+  }
+  if (NULL != timeout_tid)
+    {
+      GNUNET_SCHEDULER_cancel (timeout_tid);
+      timeout_tid = NULL;
+    }
+}
+
+
+static void
+timeout_task (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Timeout!\n");
+  result = GNUNET_SYSERR;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+member_join_request (void *cls,
+                     const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                     const struct GNUNET_MessageHeader *join_msg,
+                     struct GNUNET_MULTICAST_JoinHandle *jh)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member sent a join request.\n");
+
+}
+
+
+static int
+notify (void *cls,
+        size_t *data_size,
+        void *data)
+{
+
+  char text[] = "ping";
+  *data_size = strlen(text)+1;
+  GNUNET_memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member sents message to origin: %s\n", text);
+
+  return GNUNET_YES;
+}
+
+
+static void
+member_join_decision (void *cls,
+                      int is_admitted,
+                      const struct GNUNET_PeerIdentity *peer,
+                      uint16_t relay_count,
+                      const struct GNUNET_PeerIdentity *relays,
+                      const struct GNUNET_MessageHeader *join_msg)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member received a decision from origin: %s\n",
+              (GNUNET_YES == is_admitted)
+              ? "accepted"
+              : "rejected");
+
+  if (GNUNET_YES == is_admitted)
+  {
+    struct GNUNET_MULTICAST_MemberTransmitHandle *req;
+
+    // FIXME: move to MQ-style API!
+    req = GNUNET_MULTICAST_member_to_origin (member,
+                                             0,
+                                             &notify,
+                                             NULL);
+  }
+}
+
+
+static void
+member_message (void *cls,
+                const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  if (0 != strncmp ("pong", (char *)&msg[1], 4))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "member did not receive pong\n");
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "member receives: %s\n", (char *)&msg[1]);
+
+  // Testcase ends here.
+  result = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+static void
+origin_join_request (void *cls,
+                 const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                 const struct GNUNET_MessageHeader *join_msg,
+                 struct GNUNET_MULTICAST_JoinHandle *jh)
+{
+  struct GNUNET_MessageHeader *join_resp;
+
+  uint8_t data_size = ntohs (join_msg->size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin got a join request...\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin receives: '%s'\n", (char *)&join_msg[1]);
+
+  const char data[] = "Come in!";
+  data_size = strlen (data) + 1;
+  join_resp = GNUNET_malloc (sizeof (join_resp) + data_size);
+  join_resp->size = htons (sizeof (join_resp) + data_size);
+  join_resp->type = htons (123);
+  GNUNET_memcpy (&join_resp[1], data, data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin sends: '%s'\n", data);
+
+  GNUNET_MULTICAST_join_decision (jh,
+                                  GNUNET_YES,
+                                  0,
+                                  NULL,
+                                  join_resp);
+  GNUNET_free (join_resp);
+  result = GNUNET_OK;
+}
+
+int
+origin_notify (void *cls,
+               size_t *data_size,
+               void *data)
+{
+  char text[] = "pong";
+  *data_size = strlen(text)+1;
+  memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin sends (to all): %s\n", text);
+
+  return GNUNET_YES;
+}
+
+
+static void
+origin_request (void *cls,
+                const struct GNUNET_MULTICAST_RequestHeader *req)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin receives: %s\n", (char *)&req[1]);
+
+  if (0 != strncmp ("ping", (char *)&req[1], 4))
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "origin didn't reveice a correct request");
+
+  GNUNET_MULTICAST_origin_to_all (origin,
+                                  0,
+                                  0,
+                                  origin_notify,
+                                  NULL);
+}
+
+static void
+origin_message (void *cls,
+                const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin message msg\n");
+}
+
+
+static void
+service_connect1 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
+{
+  member = ca_result;
+
+  if (NULL != member)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Connected to multicast service of member\n");
+  }
+  else
+  {
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+}
+
+static void
+multicast_da1 (void *cls,
+               void * op_result)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member parting from multicast group\n");
+
+  GNUNET_MULTICAST_member_part (member, NULL, NULL);
+}
+
+
+static void *
+multicast_ca1 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  struct GNUNET_MessageHeader *join_msg;
+  void *ret;
+
+  // Get members keys
+  member_key = GNUNET_CRYPTO_ecdsa_key_create ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (member_key, &member_pub_key);
+
+  char data[] = "Hi, can I enter?";
+  uint8_t data_size = strlen (data) + 1;
+  join_msg = GNUNET_malloc (sizeof (join_msg) + data_size);
+  join_msg->size = htons (sizeof (join_msg) + data_size);
+  join_msg->type = htons (123);
+  GNUNET_memcpy (&join_msg[1], data, data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Members tries to join multicast group\n");
+
+  ret = GNUNET_MULTICAST_member_join (cfg,
+                                       &group_pub_key,
+                                       member_key,
+                                       peer_id[0],
+                                       0,
+                                       NULL,
+                                       join_msg, /* join message */
+                                       member_join_request,
+                                       member_join_decision,
+                                       NULL, /* no test for member_replay_frag */
+                                       NULL, /* no test for member_replay_msg */
+                                       member_message,
+                                       NULL);
+  GNUNET_free (join_msg);
+  return ret;
+}
+
+
+static void
+peer_information_cb (void *cls,
+                     struct GNUNET_TESTBED_Operation *op,
+                     const struct GNUNET_TESTBED_PeerInformation *pinfo,
+                     const char *emsg)
+{
+  int i = (int) (long) cls;
+
+  if (NULL == pinfo)
+  {
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+
+  peer_id[i] = pinfo->result.id;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Got peer information of %s (%s)\n", (0==i)?"origin":"member" ,GNUNET_i2s(pinfo->result.id));
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create member peer\n");
+
+  if (0 == i)
+  {
+    /* connect to multicast service of member */
+    op1 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                          peers[1],                /* The peer whose service to connect to */
+                                          "multicast",             /* The name of the service */
+                                          service_connect1,   /* callback to call after a handle to service
+                                                                 is opened */
+                                          NULL,                    /* closure for the above callback */
+                                          multicast_ca1,      /* callback to call with peer's configuration;
+                                                                 this should open the needed service connection */
+                                          multicast_da1,     /* callback to be called when closing the
+                                                                opened service connection */
+                                          NULL);                   /* closure for the above two callbacks */
+    }
+}
+
+/**
+ * Test logic of peer "0" being origin starts here.
+ *
+ * @param cls closure, for the example: NULL
+ * @param op should be equal to "dht_op"
+ * @param ca_result result of the connect operation, the
+ *        connection to the DHT service
+ * @param emsg error message, if testbed somehow failed to
+ *        connect to the DHT.
+ */
+static void
+service_connect0 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
+{
+  origin = ca_result;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to multicast service of origin\n");
+
+  // Get GNUnet identity of origin
+  pi_op0 = GNUNET_TESTBED_peer_get_information (peers[0],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 0);
+  // Get GNUnet identity of member
+  pi_op1 = GNUNET_TESTBED_peer_get_information (peers[1],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 1);
+
+  /* Connection to service successful. Here we'd usually do something with
+   * the service. */
+  result = GNUNET_OK;
+  //GNUNET_SCHEDULER_shutdown (); /* Also kills the testbed */
+}
+
+
+
+/**
+ * Function run when service multicast has started and is providing us
+ * with a configuration file.
+ */
+static void *
+multicast_ca0 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  group_key = GNUNET_CRYPTO_eddsa_key_create ();
+  GNUNET_CRYPTO_eddsa_key_get_public (group_key, &group_pub_key);
+
+  return GNUNET_MULTICAST_origin_start (cfg,
+                                        group_key,
+                                        0,
+                                        origin_join_request,
+                                        NULL, /* no test for origin_replay_frag */
+                                        NULL, /* no test for origin_replay_msg */
+                                        origin_request,
+                                        origin_message,
+                                        NULL);
+}
+
+static void
+multicast_da0 (void *cls,
+               void *op_result)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Origin closes multicast group\n");
+
+  GNUNET_MULTICAST_origin_stop (origin, NULL, NULL);
+}
+
+
+/**
+ * Main function inovked from TESTBED once all of the
+ * peers are up and running.  This one then connects
+ * just to the multicast service of peer 0 and 1.
+ * Peer 0 is going to be origin.
+ * Peer 1 is going to be one member.
+ * Origin will start a multicast group and the member will try to join it.
+ * After that we execute some multicast test.
+ *
+ * @param cls closure
+ * @param h the run handle
+ * @param peers started peers for the test
+ * @param num_peers size of the 'peers' array
+ * @param links_succeeded number of links between peers that were created
+ * @param links_failed number of links testbed was unable to establish
+ */
+static void
+testbed_master (void *cls,
+     struct GNUNET_TESTBED_RunHandle *h,
+     unsigned int num_peers,
+     struct GNUNET_TESTBED_Peer **p,
+     unsigned int links_succeeded,
+     unsigned int links_failed)
+{
+  /* Testbed is ready with peers running and connected in a pre-defined overlay
+     topology (FIXME)  */
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to testbed_master()\n");
+
+  peers = p;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create origin peer\n");
+  op0 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                        peers[0],                /* The peer whose service to connect to */
+                                        "multicast",             /* The name of the service */
+                                        service_connect0,   /* callback to call after a handle to service
+                                                               is opened */
+                                        NULL,                    /* closure for the above callback */
+                                        multicast_ca0,      /* callback to call with peer's configuration;
+                                                               this should open the needed service connection */
+                                        multicast_da0,     /* callback to be called when closing the
+                                                              opened service connection */
+                                        NULL);                   /* closure for the above two callbacks */
+
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); /* Schedule a new task on shutdown */
+
+  /* Schedule the shutdown task with a delay of a few Seconds */
+  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 50),
+                                              &timeout_task, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int ret;
+
+  result = GNUNET_SYSERR;
+  ret = GNUNET_TESTBED_test_run
+      ("test-multicast-multipeer",  /* test case name */
+       "test_multicast.conf", /* template configuration */
+       NUM_PEERS,       /* number of peers to start */
+       0LL, /* Event mask - set to 0 for no event notifications */
+       NULL, /* Controller event callback */
+       NULL, /* Closure for controller event callback */
+       testbed_master, /* continuation callback to be called when testbed setup is complete */
+       NULL); /* Closure for the test_master callback */
+  if ( (GNUNET_OK != ret) || (GNUNET_OK != result) )
+    return 1;
+  return 0;
+}
+
+/* end of test_multicast_multipeer.c */
diff --git a/src/multicast/test_multicast_multipeer.c b/src/multicast/test_multicast_multipeer.c
index 3edf0153e..58e43d4ba 100644
--- a/src/multicast/test_multicast_multipeer.c
+++ b/src/multicast/test_multicast_multipeer.c
@@ -35,16 +35,24 @@
 
 #define NUM_PEERS 2
 
-static struct GNUNET_TESTBED_Operation *peer0;
-static struct GNUNET_TESTBED_Operation *peer1;
+static struct GNUNET_TESTBED_Operation *op0;
+static struct GNUNET_TESTBED_Operation *op1;
+static struct GNUNET_TESTBED_Operation *pi_op0;
+static struct GNUNET_TESTBED_Operation *pi_op1;
+
+static struct GNUNET_TESTBED_Peer **peers;
+const struct GNUNET_PeerIdentity *peer_id[2];
 
 static struct GNUNET_SCHEDULER_Task *timeout_tid;
 
+static struct GNUNET_MULTICAST_Origin *origin;
+static struct GNUNET_MULTICAST_Member *member;
+
 struct GNUNET_CRYPTO_EddsaPrivateKey *group_key;
 struct GNUNET_CRYPTO_EddsaPublicKey group_pub_key;
 
-struct GNUNET_CRYPTO_EcdsaPrivateKey *member1_key;
-struct GNUNET_CRYPTO_EcdsaPublicKey member1_pub_key;
+struct GNUNET_CRYPTO_EcdsaPrivateKey *member_key;
+struct GNUNET_CRYPTO_EcdsaPublicKey member_pub_key;
 
 
 enum
@@ -68,10 +76,25 @@ static int result;
 static void
 shutdown_task (void *cls)
 {
-  if (NULL != peer0)
+  if (NULL != op0)
+  {
+    GNUNET_TESTBED_operation_done (op0);
+    op0 = NULL;
+  }
+  if (NULL != op1)
+  {
+    GNUNET_TESTBED_operation_done (op1);
+    op1 = NULL;
+  }
+  if (NULL != pi_op0)
+  {
+    GNUNET_TESTBED_operation_done (pi_op0);
+    pi_op0 = NULL;
+  }
+  if (NULL != pi_op1)
   {
-    GNUNET_TESTBED_operation_done (peer0);
-    peer0 = NULL;
+    GNUNET_TESTBED_operation_done (pi_op1);
+    pi_op1 = NULL;
   }
   if (NULL != timeout_tid)
     {
@@ -84,7 +107,6 @@ shutdown_task (void *cls)
 static void
 timeout_task (void *cls)
 {
-  timeout_tid = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
               "Timeout!\n");
   result = GNUNET_SYSERR;
@@ -93,112 +115,275 @@ timeout_task (void *cls)
 
 
 static void
-origin_recv_replay_msg (void *cls,
-                        const struct GNUNET_CRYPTO_EcdsaPublicKey *member_key,
-                        uint64_t message_id,
-                        uint64_t fragment_offset,
-                        uint64_t flags,
-                        struct GNUNET_MULTICAST_ReplayHandle *rh)
+member_join_request (void *cls,
+                     const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                     const struct GNUNET_MessageHeader *join_msg,
+                     struct GNUNET_MULTICAST_JoinHandle *jh)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: origin_recv_replay_msg()\n", test);
-  GNUNET_assert (0);
+              "Member sent a join request.\n");
+
+}
+
+int notify (void *cls,
+            size_t *data_size,
+            void *data)
+{
+
+  char text[] = "ping";
+  *data_size = strlen(text)+1;
+  GNUNET_memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "Member sents message to origin: %s\n", text);
+
+  return GNUNET_YES;
 }
 
 
 static void
-member_recv_replay_msg (void *cls,
-                        const struct GNUNET_CRYPTO_EcdsaPublicKey *member_key,
-                        uint64_t message_id,
-                        uint64_t fragment_offset,
-                        uint64_t flags,
-                        struct GNUNET_MULTICAST_ReplayHandle *rh)
+member_join_decision (void *cls,
+                      int is_admitted,
+                      const struct GNUNET_PeerIdentity *peer,
+                      uint16_t relay_count,
+                      const struct GNUNET_PeerIdentity *relays,
+                      const struct GNUNET_MessageHeader *join_msg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: member_recv_replay_msg()\n", test);
-  GNUNET_assert (0);
+  struct GNUNET_MULTICAST_MemberTransmitHandle *req;
+  
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "Member received a decision from origin: %s\n", (GNUNET_YES == is_admitted)?"accepted":"rejected");
+  
+  if (GNUNET_YES == is_admitted)
+  {
+    req = GNUNET_MULTICAST_member_to_origin (member,
+                                             0,
+                                             notify,
+                                             NULL);
+    
+  }
 }
 
-
 static void
-origin_recv_replay_frag (void *cls,
-                         const struct GNUNET_CRYPTO_EcdsaPublicKey *member_key,
-                         uint64_t fragment_id,
-                         uint64_t flags,
-                         struct GNUNET_MULTICAST_ReplayHandle *rh)
+member_replay_frag ()
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: origin_recv_replay_frag()"
-              " - fragment_id=%" PRIu64 " flags=%" PRIu64 "\n",
-              test, fragment_id, flags);
+              "member replay frag...\n");
 }
 
+static void
+member_replay_msg ()
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "member replay msg...\n");
+}
 
-/**
- * Test: origin receives join request
- */
 static void
-origin_recv_join_request (void *cls,
-                          const struct GNUNET_CRYPTO_EcdsaPublicKey *mem_key,
-                          const struct GNUNET_MessageHeader *join_msg,
-                          struct GNUNET_MULTICAST_JoinHandle *jh)
+member_message ()
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: origin_recv_join_request()\n", test);
+              "member message...\n");
+
+  // FIXME: not finished here
+  result = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
 }
 
+static void
+origin_join_request (void *cls,
+                 const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                 const struct GNUNET_MessageHeader *join_msg,
+                 struct GNUNET_MULTICAST_JoinHandle *jh)
+{
+  struct GNUNET_MessageHeader *join_resp;
+
+  uint8_t data_size = ntohs (join_msg->size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin got a join request...\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin receives: '%s'\n", (char *)&join_msg[1]);
+
+  char data[] = "Come in!";
+  data_size = strlen (data) + 1;
+  join_resp = GNUNET_malloc (sizeof (join_resp) + data_size);
+  join_resp->size = htons (sizeof (join_resp) + data_size);
+  join_resp->type = htons (123);
+  GNUNET_memcpy (&join_resp[1], data, data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin sends: '%s'\n", data);
+
+  GNUNET_MULTICAST_join_decision (jh,
+                                  GNUNET_YES,
+                                  0,
+                                  NULL,
+                                  join_resp);
+
+  result = GNUNET_OK;
+}
 
 static void
-origin_recv_request (void *cls,
-                     const struct GNUNET_MULTICAST_RequestHeader *req)
+origin_replay_frag (void *cls,
+                    const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                    uint64_t fragment_id,
+                    uint64_t flags,
+                    struct GNUNET_MULTICAST_ReplayHandle *rh)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: origin_recv_request()\n",
-              test);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin replay fraq msg\n");
+}
+
+static void
+origin_replay_msg (void *cls,
+                   const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                   uint64_t message_id,
+                   uint64_t fragment_offset,
+                   uint64_t flags,
+                   struct GNUNET_MULTICAST_ReplayHandle *rh) 
+{
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin replay msg\n");
+}
+
+
+int
+origin_notify (void *cls, 
+               size_t *data_size, 
+               void *data)
+{
+  char text[] = "pong";
+  *data_size = strlen(text)+1;
+  memcpy(data, text, *data_size); 
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin sends (to all): %s\n", text);
+
+  return GNUNET_YES; 
 }
 
 
 static void
-origin_recv_message (void *cls,
-                     const struct GNUNET_MULTICAST_MessageHeader *msg)
+origin_request (void *cls,
+                const struct GNUNET_MULTICAST_RequestHeader *req)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Test #%u: origin_recv_message()\n",
-              test);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin receives: %s\n", (char *)&req[1]);
+  
+  if (0 != strncmp ("ping", (char *)&req[1], 4)) {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "origin didn't reveice a correct request");
+  }
+
+  GNUNET_MULTICAST_origin_to_all (origin,
+                                  0,
+                                  0,
+                                  origin_notify,
+                                  NULL);
 }
 
+static void
+origin_message (void *cls,
+                const struct GNUNET_MULTICAST_MessageHeader *msg) 
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin message msg\n");
+}
+
+
+static void
+service_connect1 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
+{
+  member = ca_result;
+
+  if (NULL != member)
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Connected to multicast service of member\n");
+  else
+    result = GNUNET_SYSERR;
+}
 
 static void
-service_close_peer0 (void *cls,
-                     void *op_result)
+multicast_da1 (void *cls,
+               void * op_result)
 {
-  struct GNUNET_MULTICAST_Origin *orig = op_result;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member parting from multicast group\n");
 
-  GNUNET_MULTICAST_origin_stop (orig, NULL, NULL);
+  GNUNET_MULTICAST_member_part (member, NULL, NULL);
 }
 
 
-/**
- * Function run when service multicast has started and is providing us
- * with a configuration file.
- */
 static void *
-service_conf_peer0 (void *cls,
-                    const struct GNUNET_CONFIGURATION_Handle *cfg)
+multicast_ca1 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
-  group_key = GNUNET_CRYPTO_eddsa_key_create ();
-  GNUNET_CRYPTO_eddsa_key_get_public (group_key, &group_pub_key);
+  struct GNUNET_MessageHeader *join_msg;
+
+  // Get members keys
+  member_key = GNUNET_CRYPTO_ecdsa_key_create ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (member_key, &member_pub_key);
+  
+  char data[] = "Hi, can I enter?";
+  uint8_t data_size = strlen (data) + 1;
+  join_msg = GNUNET_malloc (sizeof (join_msg) + data_size);
+  join_msg->size = htons (sizeof (join_msg) + data_size);
+  join_msg->type = htons (123);
+  GNUNET_memcpy (&join_msg[1], data, data_size);
 
-  return GNUNET_MULTICAST_origin_start (cfg, group_key, 0,
-                                        origin_recv_join_request,
-                                        origin_recv_replay_frag,
-                                        origin_recv_replay_msg,
-                                        origin_recv_request,
-                                        origin_recv_message,
-                                        NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Members tries to join multicast group\n");
+
+  return GNUNET_MULTICAST_member_join (cfg,
+                                       &group_pub_key,
+                                       member_key,
+                                       peer_id[0],
+                                       0,
+                                       NULL,
+                                       join_msg, /* join message */
+                                       member_join_request,
+                                       member_join_decision,
+                                       member_replay_frag,
+                                       member_replay_msg,
+                                       member_message,
+                                       NULL);
 }
 
 
+static void
+peer_information_cb (void *cls,
+                     struct GNUNET_TESTBED_Operation *op,
+                     const struct GNUNET_TESTBED_PeerInformation *pinfo,
+                     const char *emsg)
+{
+  int i = (int) (long) cls;
+
+  if (NULL == pinfo) {
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+
+  peer_id[i] = pinfo->result.id;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Got peer information of %s (%s)\n", (0==i)?"origin":"member" ,GNUNET_i2s(pinfo->result.id));
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create member peer\n");
+
+  if (0 == i) {
+  /* connect to multicast service of member */
+  op1 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                        peers[1],                /* The peer whose service to connect to */
+                                        "multicast",             /* The name of the service */
+                                        service_connect1,   /* callback to call after a handle to service
+                                                               is opened */
+                                        NULL,                    /* closure for the above callback */
+                                        multicast_ca1,      /* callback to call with peer's configuration;
+                                                               this should open the needed service connection */
+                                        multicast_da1,     /* callback to be called when closing the
+                                                              opened service connection */
+                                        NULL);                   /* closure for the above two callbacks */
+  }
+}
+
 /**
  * Test logic of peer "0" being origin starts here.
  *
@@ -210,17 +395,65 @@ service_conf_peer0 (void *cls,
  *        connect to the DHT.
  */
 static void
-service_connect_peer0 (void *cls,
-                       struct GNUNET_TESTBED_Operation *op,
-                       void *ca_result,
-                       const char *emsg)
+service_connect0 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
 {
-  struct GNUNET_MULTICAST_Origin *orig = ca_result;
+  origin = ca_result;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to multicast service of origin\n");
+
+  // Get GNUnet identity of origin
+  pi_op0 = GNUNET_TESTBED_peer_get_information (peers[0],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 0);
+  // Get GNUnet identity of member
+  pi_op1 = GNUNET_TESTBED_peer_get_information (peers[1],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 1);
 
   /* Connection to service successful. Here we'd usually do something with
    * the service. */
   result = GNUNET_OK;
-  GNUNET_SCHEDULER_shutdown (); /* Also kills the testbed */
+  //GNUNET_SCHEDULER_shutdown (); /* Also kills the testbed */
+}
+
+
+
+/**
+ * Function run when service multicast has started and is providing us
+ * with a configuration file.
+ */
+static void *
+multicast_ca0 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  group_key = GNUNET_CRYPTO_eddsa_key_create ();
+  GNUNET_CRYPTO_eddsa_key_get_public (group_key, &group_pub_key);
+
+  return GNUNET_MULTICAST_origin_start (cfg,
+                                        group_key,
+                                        0,
+                                        origin_join_request,
+                                        origin_replay_frag,
+                                        origin_replay_msg,
+                                        origin_request,
+                                        origin_message,
+                                        NULL);
+}
+
+static void
+multicast_da0 (void *cls,
+               void *op_result)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Origin closes multicast group\n");
+
+  GNUNET_MULTICAST_origin_stop (origin, NULL, NULL);
 }
 
 
@@ -241,31 +474,38 @@ service_connect_peer0 (void *cls,
  * @param links_failed number of links testbed was unable to establish
  */
 static void
-run (void *cls,
+testbed_master (void *cls,
      struct GNUNET_TESTBED_RunHandle *h,
      unsigned int num_peers,
-     struct GNUNET_TESTBED_Peer **peers,
+     struct GNUNET_TESTBED_Peer **p,
      unsigned int links_succeeded,
      unsigned int links_failed)
 {
   /* Testbed is ready with peers running and connected in a pre-defined overlay
      topology (FIXME)  */
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to testbed_master()\n");
+
+  peers = p;
 
-  /* connect to a peers service */
-  peer0 = GNUNET_TESTBED_service_connect
-      (NULL,                    /* Closure for operation */
-       peers[0],                /* The peer whose service to connect to */
-       "multicast",             /* The name of the service */
-       &service_connect_peer0,   /* callback to call after a handle to service
-                                   is opened */
-       NULL,                    /* closure for the above callback */
-       &service_conf_peer0,      /* callback to call with peer's configuration;
-                                   this should open the needed service connection */
-       &service_close_peer0,     /* callback to be called when closing the
-                                   opened service connection */
-       NULL);                   /* closure for the above two callbacks */
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create origin peer\n");
+  op0 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                        peers[0],                /* The peer whose service to connect to */
+                                        "multicast",             /* The name of the service */
+                                        service_connect0,   /* callback to call after a handle to service
+                                                               is opened */
+                                        NULL,                    /* closure for the above callback */
+                                        multicast_ca0,      /* callback to call with peer's configuration;
+                                                               this should open the needed service connection */
+                                        multicast_da0,     /* callback to be called when closing the
+                                                              opened service connection */
+                                        NULL);                   /* closure for the above two callbacks */
+
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); /* Schedule a new task on shutdown */
+
+  /* Schedule the shutdown task with a delay of a few Seconds */
+  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 48),
                                               &timeout_task, NULL);
 }
 
@@ -283,7 +523,7 @@ main (int argc, char *argv[])
        0LL, /* Event mask - set to 0 for no event notifications */
        NULL, /* Controller event callback */
        NULL, /* Closure for controller event callback */
-       run, /* continuation callback to be called when testbed setup is complete */
+       testbed_master, /* continuation callback to be called when testbed setup is complete */
        NULL); /* Closure for the test_master callback */
   if ( (GNUNET_OK != ret) || (GNUNET_OK != result) )
     return 1;
diff --git a/src/namecache/.gitignore b/src/namecache/.gitignore
index 6d2d8488a..2abc07dfb 100644
--- a/src/namecache/.gitignore
+++ b/src/namecache/.gitignore
@@ -4,3 +4,4 @@ test_namecache_api_cache_block
 test_plugin_namecache_postgres
 test_plugin_namecache_sqlite
 zonefiles
+test_plugin_namecache_flat
diff --git a/src/namecache/Makefile.am b/src/namecache/Makefile.am
index cb4559552..d379b2602 100644
--- a/src/namecache/Makefile.am
+++ b/src/namecache/Makefile.am
@@ -56,7 +56,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
   $(check_PROGRAMS)
 endif
@@ -121,6 +121,7 @@ libgnunet_plugin_namecache_sqlite_la_SOURCES = \
   plugin_namecache_sqlite.c
 libgnunet_plugin_namecache_sqlite_la_LIBADD = \
   libgnunetnamecache.la  \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
@@ -172,4 +173,3 @@ EXTRA_DIST = \
   test_plugin_namecache_sqlite.conf \
   test_plugin_namecache_postgres.conf \
         test_plugin_namecache_flat.conf
-
diff --git a/src/namecache/gnunet-namecache.c b/src/namecache/gnunet-namecache.c
index 490197b1e..d867138e4 100644
--- a/src/namecache/gnunet-namecache.c
+++ b/src/namecache/gnunet-namecache.c
@@ -225,13 +225,19 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "name", "NAME",
-     gettext_noop ("name of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'z', "zone", "PKEY",
-     gettext_noop ("spezifies the public key of the zone to look in"), 1,
-     &GNUNET_GETOPT_set_string, &pkey},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_string ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("name of the record to add/delete/display"),
+                                 &name),
+
+    GNUNET_GETOPT_option_string ('z',
+                                 "zone",
+                                 "PKEY",
+                                 gettext_noop ("spezifies the public key of the zone to look in"),
+                                 &pkey),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index fdce899fa..2f7b2a981 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -25,6 +25,7 @@
  */
 
 #include "platform.h"
+#include "gnunet_sq_lib.h"
 #include "gnunet_namecache_plugin.h"
 #include "gnunet_namecache_service.h"
 #include "gnunet_gnsrecord_lib.h"
@@ -241,25 +242,28 @@ database_setup (struct Plugin *plugin)
   sqlite3_finalize (stmt);
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "INSERT INTO ns096blocks (query,block,expiration_time) VALUES (?, ?, ?)",
-        &plugin->cache_block) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns096blocks WHERE expiration_time<?",
-        &plugin->expire_blocks) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns096blocks WHERE query=? AND expiration_time<=?",
-        &plugin->delete_block) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT block FROM ns096blocks WHERE query=? ORDER BY expiration_time DESC LIMIT 1",
-        &plugin->lookup_block) != SQLITE_OK)
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ns096blocks (query,block,expiration_time) VALUES (?, ?, ?)",
+                    &plugin->cache_block)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns096blocks WHERE expiration_time<?",
+                    &plugin->expire_blocks)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns096blocks WHERE query=? AND expiration_time<=?",
+                    &plugin->delete_block)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT block FROM ns096blocks WHERE query=? "
+                    "ORDER BY expiration_time DESC LIMIT 1",
+                    &plugin->lookup_block) )
       )
   {
-    LOG_SQLITE (plugin,GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -319,35 +323,41 @@ static void
 namecache_sqlite_expire_blocks (struct Plugin *plugin)
 {
   struct GNUNET_TIME_Absolute now;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
   int n;
 
   now = GNUNET_TIME_absolute_get ();
-  if (SQLITE_OK != sqlite3_bind_int64 (plugin->expire_blocks,
-                                       1, now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->expire_blocks,
+                      params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->expire_blocks))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->expire_blocks);
     return;
   }
   n = sqlite3_step (plugin->expire_blocks);
-  if (SQLITE_OK != sqlite3_reset (plugin->expire_blocks))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->expire_blocks);
   switch (n)
   {
   case SQLITE_DONE:
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Records expired\n");
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
+                     "Records expired\n");
     return;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return;
   }
@@ -368,8 +378,21 @@ namecache_sqlite_cache_block (void *cls,
   struct Plugin *plugin = cls;
   struct GNUNET_HashCode query;
   struct GNUNET_TIME_Absolute expiration;
-  int64_t dval;
-  size_t block_size;
+  size_t block_size = ntohl (block->purpose.size) +
+    sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
+    sizeof (struct GNUNET_CRYPTO_EcdsaSignature);
+  struct GNUNET_SQ_QueryParam del_params[] = {
+    GNUNET_SQ_query_param_auto_from_type (&query),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam ins_params[] = {
+    GNUNET_SQ_query_param_auto_from_type (&query),
+    GNUNET_SQ_query_param_fixed_size (block,
+                                      block_size),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_end
+  };
   int n;
 
   namecache_sqlite_expire_blocks (plugin);
@@ -377,12 +400,6 @@ namecache_sqlite_cache_block (void *cls,
                       sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
                       &query);
   expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
-  dval = (int64_t) expiration.abs_value_us;
-  if (dval < 0)
-    dval = INT64_MAX;
-  block_size = ntohl (block->purpose.size) +
-    sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
-    sizeof (struct GNUNET_CRYPTO_EcdsaSignature);
   if (block_size > 64 * 65536)
   {
     GNUNET_break (0);
@@ -390,61 +407,48 @@ namecache_sqlite_cache_block (void *cls,
   }
 
   /* delete old version of the block */
-  if ( (SQLITE_OK !=
-        sqlite3_bind_blob (plugin->delete_block, 1,
-                           &query, sizeof (struct GNUNET_HashCode),
-                           SQLITE_STATIC)) ||
-       (SQLITE_OK !=
-        sqlite3_bind_int64 (plugin->delete_block,
-                            2, dval)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->delete_block,
+                      del_params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->delete_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delete_block);
     return GNUNET_SYSERR;
   }
   n = sqlite3_step (plugin->delete_block);
   switch (n)
   {
   case SQLITE_DONE:
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Old block deleted\n");
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
+                     "Old block deleted\n");
     break;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     break;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     break;
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->delete_block))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->delete_block);
 
   /* insert new version of the block */
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (plugin->cache_block, 1,
-                          &query, sizeof (struct GNUNET_HashCode),
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (plugin->cache_block, 2,
-                          block, block_size,
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_int64 (plugin->cache_block, 3,
-                           dval)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->cache_block,
+                      ins_params))
   {
     LOG_SQLITE (plugin,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->cache_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->cache_block);
     return GNUNET_SYSERR;
 
   }
@@ -452,9 +456,8 @@ namecache_sqlite_cache_block (void *cls,
               "Caching block under derived key `%s'\n",
               GNUNET_h2s_full (&query));
   n = sqlite3_step (plugin->cache_block);
-  if (SQLITE_OK != sqlite3_reset (plugin->cache_block))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->cache_block);
   switch (n)
   {
   case SQLITE_DONE:
@@ -462,11 +465,13 @@ namecache_sqlite_cache_block (void *cls,
          "Record stored\n");
     return GNUNET_OK;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_NO;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_SYSERR;
   }
@@ -486,45 +491,63 @@ namecache_sqlite_cache_block (void *cls,
 static int
 namecache_sqlite_lookup_block (void *cls,
                                const struct GNUNET_HashCode *query,
-                               GNUNET_NAMECACHE_BlockCallback iter, void *iter_cls)
+                               GNUNET_NAMECACHE_BlockCallback iter,
+                               void *iter_cls)
 {
   struct Plugin *plugin = cls;
   int ret;
   int sret;
   size_t block_size;
   const struct GNUNET_GNSRECORD_Block *block;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (query),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size ((void **) &block,
+                                         &block_size),
+    GNUNET_SQ_result_spec_end
+  };
 
-  if (SQLITE_OK != sqlite3_bind_blob (plugin->lookup_block, 1,
-                                      query, sizeof (struct GNUNET_HashCode),
-                                      SQLITE_STATIC))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->lookup_block,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->lookup_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->lookup_block);
     return GNUNET_SYSERR;
   }
   ret = GNUNET_NO;
-  if (SQLITE_ROW == (sret = sqlite3_step (plugin->lookup_block)))
+  if (SQLITE_ROW ==
+      (sret = sqlite3_step (plugin->lookup_block)))
   {
-    block = sqlite3_column_blob (plugin->lookup_block, 0);
-    block_size = sqlite3_column_bytes (plugin->lookup_block, 0);
-    if ( (block_size < sizeof (struct GNUNET_GNSRECORD_Block)) ||
-         (ntohl (block->purpose.size) +
-          sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
-          sizeof (struct GNUNET_CRYPTO_EcdsaSignature) != block_size) )
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->lookup_block,
+                                  rs))
     {
       GNUNET_break (0);
       ret = GNUNET_SYSERR;
     }
+    else if ( (block_size < sizeof (struct GNUNET_GNSRECORD_Block)) ||
+              (ntohl (block->purpose.size) +
+               sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
+               sizeof (struct GNUNET_CRYPTO_EcdsaSignature) != block_size) )
+    {
+      GNUNET_break (0);
+      GNUNET_SQ_cleanup_result (rs);
+      ret = GNUNET_SYSERR;
+    }
     else
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Found block under derived key `%s'\n",
                   GNUNET_h2s_full (query));
-      iter (iter_cls, block);
+      iter (iter_cls,
+            block);
+      GNUNET_SQ_cleanup_result (rs);
       ret = GNUNET_YES;
     }
   }
@@ -532,7 +555,9 @@ namecache_sqlite_lookup_block (void *cls,
   {
     if (SQLITE_DONE != sret)
     {
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+      LOG_SQLITE (plugin,
+                  GNUNET_ERROR_TYPE_ERROR,
+                  "sqlite_step");
       ret = GNUNET_SYSERR;
     }
     else
@@ -542,10 +567,8 @@ namecache_sqlite_lookup_block (void *cls,
                   GNUNET_h2s_full (query));
     }
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->lookup_block))
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->lookup_block);
   return ret;
 }
 
diff --git a/src/namestore/.gitignore b/src/namestore/.gitignore
index a1153c6f9..4995a9e36 100644
--- a/src/namestore/.gitignore
+++ b/src/namestore/.gitignore
@@ -1,20 +1,21 @@
 gnunet-service-namestore
 gnunet-namestore
 gnunet-namestore-fcfsd
-test_namestore_api_lookup_nick
-test_namestore_api_lookup_private
-test_namestore_api_lookup_public
-test_namestore_api_lookup_shadow
-test_namestore_api_lookup_shadow_filter
-test_namestore_api_monitoring
-test_namestore_api_monitoring_existing
-test_namestore_api_remove
-test_namestore_api_remove_not_existing_record
-test_namestore_api_store
-test_namestore_api_store_update
-test_namestore_api_zone_iteration
-test_namestore_api_zone_iteration_nick
-test_namestore_api_zone_iteration_specific_zone
-test_namestore_api_zone_iteration_stop
+test_namestore_api_lookup_nick.nc
+test_namestore_api_lookup_private.nc
+test_namestore_api_lookup_public.nc
+test_namestore_api_lookup_shadow.nc
+test_namestore_api_lookup_shadow_filter.nc
+test_namestore_api_monitoring.nc
+test_namestore_api_monitoring_existing.nc
+test_namestore_api_remove.nc
+test_namestore_api_remove_not_existing_record.nc
+test_namestore_api_store.nc
+test_namestore_api_store_update.nc
+test_namestore_api_zone_iteration.nc
+test_namestore_api_zone_iteration_nick.nc
+test_namestore_api_zone_iteration_specific_zone.nc
+test_namestore_api_zone_iteration_stop.nc
 test_plugin_namestore_postgres
 test_plugin_namestore_sqlite
+test_plugin_namestore_flat
diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am
index 23ce477dd..de46e9c89 100644
--- a/src/namestore/Makefile.am
+++ b/src/namestore/Makefile.am
@@ -45,24 +45,43 @@ endif
 # testcases do not even build yet; thus: experimental!
 if HAVE_TESTING
 TESTING_TESTS = \
- test_namestore_api_store \
- test_namestore_api_store_update \
- test_namestore_api_lookup_public \
- test_namestore_api_lookup_private \
- test_namestore_api_lookup_nick \
- test_namestore_api_lookup_shadow \
- test_namestore_api_lookup_shadow_filter \
- test_namestore_api_remove \
- test_namestore_api_remove_not_existing_record \
- test_namestore_api_zone_iteration \
- test_namestore_api_zone_iteration_nick \
- test_namestore_api_zone_iteration_specific_zone \
- test_namestore_api_zone_iteration_stop \
- test_namestore_api_monitoring \
- test_namestore_api_monitoring_existing
+ test_namestore_api_store.nc \
+ test_namestore_api_store_update.nc \
+ test_namestore_api_lookup_public.nc \
+ test_namestore_api_lookup_private.nc \
+ test_namestore_api_lookup_nick.nc \
+ test_namestore_api_lookup_shadow.nc \
+ test_namestore_api_lookup_shadow_filter.nc \
+ test_namestore_api_remove.nc \
+ test_namestore_api_remove_not_existing_record.nc \
+ test_namestore_api_zone_iteration.nc \
+ test_namestore_api_zone_iteration_nick.nc \
+ test_namestore_api_zone_iteration_specific_zone.nc \
+ test_namestore_api_zone_iteration_stop.nc \
+ test_namestore_api_monitoring.nc \
+ test_namestore_api_monitoring_existing.nc
 # test_namestore_api_zone_to_name
 endif
 
+# Declare .nc (NO-CONCURRENCY) as a test extension so that we can impart
+# sequential execution order for them
+TEST_EXTENSIONS = .nc
+test_namestore_api_store.log: test_namestore_api_store_update.log
+test_namestore_api_store_update.log: test_namestore_api_lookup_public.log
+test_namestore_api_lookup_public.log: test_namestore_api_lookup_private.log
+test_namestore_api_lookup_private.log: test_namestore_api_lookup_nick.log
+test_namestore_api_lookup_nick.log: test_namestore_api_lookup_shadow.log
+test_namestore_api_lookup_shadow.log: test_namestore_api_lookup_shadow_filter.log
+test_namestore_api_lookup_shadow_filter.log: test_namestore_api_remove.log
+test_namestore_api_remove.log: test_namestore_api_remove_not_existing_record.log
+test_namestore_api_remove_not_existing_record.log: test_namestore_api_zone_iteration.log
+test_namestore_api_zone_iteration.log: test_namestore_api_zone_iteration_nick.log
+test_namestore_api_zone_iteration_nick.log: test_namestore_api_zone_iteration_specific_zone.log
+test_namestore_api_zone_iteration_specific_zone.log: test_namestore_api_zone_iteration_stop.log
+test_namestore_api_zone_iteration_stop.log: test_namestore_api_monitoring.log
+test_namestore_api_monitoring.log: test_namestore_api_monitoring_existing.log
+
+
 if HAVE_SQLITE
 check_PROGRAMS = \
  $(SQLITE_TESTS) \
@@ -78,7 +97,7 @@ endif
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
   $(check_PROGRAMS) \
   $(check_SCRIPTS)
@@ -170,6 +189,7 @@ libgnunet_plugin_namestore_sqlite_la_SOURCES = \
   plugin_namestore_sqlite.c
 libgnunet_plugin_namestore_sqlite_la_LIBADD = \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la  \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
@@ -202,79 +222,79 @@ libgnunet_plugin_rest_namestore_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
 
 
-test_namestore_api_store_SOURCES = \
+test_namestore_api_store_nc_SOURCES = \
  test_namestore_api_store.c
-test_namestore_api_store_LDADD = \
+test_namestore_api_store_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   libgnunetnamestore.la
 
-test_namestore_api_store_update_SOURCES = \
+test_namestore_api_store_update_nc_SOURCES = \
  test_namestore_api_store_update.c
-test_namestore_api_store_update_LDADD = \
+test_namestore_api_store_update_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_lookup_public_SOURCES = \
+test_namestore_api_lookup_public_nc_SOURCES = \
  test_namestore_api_lookup_public.c
-test_namestore_api_lookup_public_LDADD = \
+test_namestore_api_lookup_public_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_lookup_nick_SOURCES = \
+test_namestore_api_lookup_nick_nc_SOURCES = \
  test_namestore_api_lookup_nick.c
-test_namestore_api_lookup_nick_LDADD = \
+test_namestore_api_lookup_nick_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_lookup_private_SOURCES = \
+test_namestore_api_lookup_private_nc_SOURCES = \
  test_namestore_api_lookup_private.c
-test_namestore_api_lookup_private_LDADD = \
+test_namestore_api_lookup_private_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_lookup_shadow_SOURCES = \
+test_namestore_api_lookup_shadow_nc_SOURCES = \
  test_namestore_api_lookup_shadow.c
-test_namestore_api_lookup_shadow_LDADD = \
+test_namestore_api_lookup_shadow_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_lookup_shadow_filter_SOURCES = \
+test_namestore_api_lookup_shadow_filter_nc_SOURCES = \
  test_namestore_api_lookup_shadow_filter.c
-test_namestore_api_lookup_shadow_filter_LDADD = \
+test_namestore_api_lookup_shadow_filter_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/namecache/libgnunetnamecache.la \
   libgnunetnamestore.la
 
-test_namestore_api_remove_SOURCES = \
+test_namestore_api_remove_nc_SOURCES = \
  test_namestore_api_remove.c
-test_namestore_api_remove_LDADD = \
+test_namestore_api_remove_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   libgnunetnamestore.la
 
-test_namestore_api_remove_not_existing_record_SOURCES = \
+test_namestore_api_remove_not_existing_record_nc_SOURCES = \
  test_namestore_api_remove_not_existing_record.c
-test_namestore_api_remove_not_existing_record_LDADD = \
+test_namestore_api_remove_not_existing_record_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
@@ -287,50 +307,50 @@ test_namestore_api_remove_not_existing_record_LDADD = \
 #  $(top_builddir)/src/util/libgnunetutil.la \
 #  libgnunetnamestore.la
 
-test_namestore_api_monitoring_SOURCES = \
+test_namestore_api_monitoring_nc_SOURCES = \
  test_namestore_api_monitoring.c
-test_namestore_api_monitoring_LDADD = \
+test_namestore_api_monitoring_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   libgnunetnamestore.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-test_namestore_api_monitoring_existing_SOURCES = \
+test_namestore_api_monitoring_existing_nc_SOURCES = \
  test_namestore_api_monitoring_existing.c
-test_namestore_api_monitoring_existing_LDADD = \
+test_namestore_api_monitoring_existing_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   libgnunetnamestore.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-test_namestore_api_zone_iteration_SOURCES = \
+test_namestore_api_zone_iteration_nc_SOURCES = \
  test_namestore_api_zone_iteration.c
-test_namestore_api_zone_iteration_LDADD = \
+test_namestore_api_zone_iteration_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   libgnunetnamestore.la
 
-test_namestore_api_zone_iteration_nick_SOURCES = \
+test_namestore_api_zone_iteration_nick_nc_SOURCES = \
  test_namestore_api_zone_iteration_nick.c
-test_namestore_api_zone_iteration_nick_LDADD = \
+test_namestore_api_zone_iteration_nick_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   libgnunetnamestore.la
 
 
-test_namestore_api_zone_iteration_specific_zone_SOURCES = \
+test_namestore_api_zone_iteration_specific_zone_nc_SOURCES = \
  test_namestore_api_zone_iteration_specific_zone.c
-test_namestore_api_zone_iteration_specific_zone_LDADD = \
+test_namestore_api_zone_iteration_specific_zone_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
   libgnunetnamestore.la
 
-test_namestore_api_zone_iteration_stop_SOURCES = \
+test_namestore_api_zone_iteration_stop_nc_SOURCES = \
  test_namestore_api_zone_iteration_stop.c
-test_namestore_api_zone_iteration_stop_LDADD = \
+test_namestore_api_zone_iteration_stop_nc_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \
@@ -363,7 +383,7 @@ EXTRA_DIST = \
   test_namestore_api.conf \
   test_plugin_namestore_sqlite.conf \
   test_plugin_namestore_postgres.conf \
-        test_plugin_namestore_flat.conf \
+  test_plugin_namestore_flat.conf \
   test_hostkey \
   zonefiles/S5I9DSGQVAB5FVV16T3B3CC5H1B2JGL3Q412JBKURME8EKU0600G.zkey \
   zonefiles/AQ835GVL939H4O8QJQ7GBLPTQC0QAAO91BN7QK01BA63MDSK6I4G.zkey \
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 457e77022..dcb9dd678 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -1130,49 +1130,88 @@ main (int argc,
   is_public = -1;
   is_shadow = -1;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "add", NULL,
-     gettext_noop ("add record"), 0,
-     &GNUNET_GETOPT_set_one, &add},
-    {'d', "delete", NULL,
-     gettext_noop ("delete record"), 0,
-     &GNUNET_GETOPT_set_one, &del},
-    {'D', "display", NULL,
-     gettext_noop ("display records"), 0,
-     &GNUNET_GETOPT_set_one, &list},
-    {'e', "expiration", "TIME",
-     gettext_noop ("expiration time for record to use (for adding only), \"never\" is possible"), 1,
-     &GNUNET_GETOPT_set_string, &expirationstring},
-    {'i', "nick", "NICKNAME",
-     gettext_noop ("set the desired nick name for the zone"), 1,
-     &GNUNET_GETOPT_set_string, &nickstring},
-    {'m', "monitor", NULL,
-     gettext_noop ("monitor changes in the namestore"), 0,
-     &GNUNET_GETOPT_set_one, &monitor},
-    {'n', "name", "NAME",
-     gettext_noop ("name of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'r', "reverse", "PKEY",
-     gettext_noop ("determine our name for the given PKEY"), 1,
-     &GNUNET_GETOPT_set_string, &reverse_pkey},
-    {'t', "type", "TYPE",
-     gettext_noop ("type of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &typestring},
-    {'u', "uri", "URI",
-     gettext_noop ("URI to import into our zone"), 1,
-     &GNUNET_GETOPT_set_string, &uri},
-    {'V', "value", "VALUE",
-     gettext_noop ("value of the record to add/delete"), 1,
-     &GNUNET_GETOPT_set_string, &value},
-    {'p', "public", NULL,
-     gettext_noop ("create or list public record"), 0,
-     &GNUNET_GETOPT_set_one, &is_public},
-    {'s', "shadow", NULL,
-     gettext_noop ("create shadow record (only valid if all other records of the same type have expired"), 0,
-     &GNUNET_GETOPT_set_one, &is_shadow},
-    {'z', "zone", "EGO",
-     gettext_noop ("name of the ego controlling the zone"), 1,
-     &GNUNET_GETOPT_set_string, &ego_name},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_flag ('a',
+                                  "add",
+                                  gettext_noop ("add record"),
+                                  &add),
+
+    GNUNET_GETOPT_option_flag ('d',
+                                  "delete",
+                                  gettext_noop ("delete record"),
+                                  &del),
+
+    GNUNET_GETOPT_option_flag ('D',
+                                  "display",
+                                  gettext_noop ("display records"),
+                                  &list), 
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "expiration",
+                                 "TIME",
+                                 gettext_noop ("expiration time for record to use (for adding only), \"never\" is possible"),
+                                 &expirationstring),
+
+    GNUNET_GETOPT_option_string ('i',
+                                 "nick",
+                                 "NICKNAME",
+                                 gettext_noop ("set the desired nick name for the zone"),
+                                 &nickstring), 
+
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor changes in the namestore"),
+                                  &monitor),
+
+    GNUNET_GETOPT_option_string ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("name of the record to add/delete/display"),
+                                 &name),
+
+    GNUNET_GETOPT_option_string ('r',
+                                 "reverse",
+                                 "PKEY",
+                                 gettext_noop ("determine our name for the given PKEY"),
+                                 &reverse_pkey),
+
+
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("type of the record to add/delete/display"),
+                                 &typestring),
+
+    GNUNET_GETOPT_option_string ('u',
+                                 "uri",
+                                 "URI",
+                                 gettext_noop ("URI to import into our zone"),
+                                 &uri),
+
+    GNUNET_GETOPT_option_string ('V',
+                                 "value",
+                                 "VALUE",
+                                 gettext_noop ("value of the record to add/delete"),
+                                 &value),
+
+    GNUNET_GETOPT_option_flag ('p',
+                                  "public",
+                                  gettext_noop ("create or list public record"),
+                                  &is_public),
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "shadow",
+                                  gettext_noop ("create shadow record (only valid if all other records of the same type have expired"),
+                                  &is_shadow),
+
+    GNUNET_GETOPT_option_string ('z',
+                                 "zone",
+                                 "EGO",
+                                 gettext_noop ("name of the ego controlling the zone"),
+                                 &ego_name), 
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index ae65802b0..6cb4290a0 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -1136,7 +1136,7 @@ handle_zone_to_name_it (void *cls,
   name_len = (NULL == name) ? 0 : strlen (name) + 1;
   rd_ser_len = GNUNET_GNSRECORD_records_get_size (rd_count, rd);
   msg_size = sizeof (struct ZoneToNameResponseMessage) + name_len + rd_ser_len;
-  if (msg_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msg_size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     ztn_ctx->success = GNUNET_SYSERR;
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index d64ce10a8..5c3533506 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -1,6 +1,6 @@
  /*
   * This file is part of GNUnet
-  * Copyright (C) 2009-2013 GNUnet e.V.
+  * Copyright (C) 2009-2017 GNUnet e.V.
   *
   * GNUnet is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published
@@ -28,6 +28,7 @@
 #include "gnunet_namestore_plugin.h"
 #include "gnunet_namestore_service.h"
 #include "gnunet_gnsrecord_lib.h"
+#include "gnunet_sq_lib.h"
 #include "namestore.h"
 #include <sqlite3.h>
 
@@ -113,16 +114,24 @@ struct Plugin
  * @return 0 on success
  */
 static int
-sq_prepare (sqlite3 * dbh, const char *zSql, sqlite3_stmt ** ppStmt)
+sq_prepare (sqlite3 *dbh,
+            const char *zSql,
+            sqlite3_stmt **ppStmt)
 {
   char *dummy;
   int result;
 
   result =
-      sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
+      sqlite3_prepare_v2 (dbh,
+                          zSql,
+                          strlen (zSql),
+                          ppStmt,
                           (const char **) &dummy);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
+       "Prepared `%s' / %p: %d\n",
+       zSql,
+       *ppStmt,
+       result);
   return result;
 }
 
@@ -275,38 +284,41 @@ database_setup (struct Plugin *plugin)
 
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
-        " VALUES (?, ?, ?, ?, ?, ?)",
-        &plugin->store_records) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns097records WHERE zone_private_key=? AND label=?",
-        &plugin->delete_records) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label"
-        " FROM ns097records WHERE zone_private_key=? AND pkey=?",
-        &plugin->zone_to_name) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label"
-        " FROM ns097records WHERE zone_private_key=? ORDER BY rvalue LIMIT 1 OFFSET ?",
-        &plugin->iterate_zone) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label,zone_private_key"
-        " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
-        &plugin->iterate_all_zones) != SQLITE_OK)  ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label,zone_private_key"
-        " FROM ns097records WHERE zone_private_key=? AND label=?",
-        &plugin->lookup_label) != SQLITE_OK)
-      )
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
+                    " VALUES (?, ?, ?, ?, ?, ?)",
+                    &plugin->store_records)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns097records WHERE zone_private_key=? AND label=?",
+                    &plugin->delete_records)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label"
+                    " FROM ns097records WHERE zone_private_key=? AND pkey=?",
+                    &plugin->zone_to_name)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label"
+                    " FROM ns097records WHERE zone_private_key=?"
+                    " ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    &plugin->iterate_zone)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label,zone_private_key"
+                    " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    &plugin->iterate_all_zones))  ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label,zone_private_key"
+                    " FROM ns097records WHERE zone_private_key=? AND label=?",
+                    &plugin->lookup_label))
+       )
   {
-    LOG_SQLITE (plugin,GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -341,21 +353,30 @@ database_shutdown (struct Plugin *plugin)
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Tried to close sqlite without finalizing all prepared statements.\n"));
-    stmt = sqlite3_next_stmt (plugin->dbh, NULL);
-    while (stmt != NULL)
+    stmt = sqlite3_next_stmt (plugin->dbh,
+                              NULL);
+    while (NULL != stmt)
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                       "Closing statement %p\n", stmt);
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Closing statement %p\n",
+                       stmt);
       result = sqlite3_finalize (stmt);
       if (result != SQLITE_OK)
-        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                         "Failed to close statement %p: %d\n", stmt, result);
-      stmt = sqlite3_next_stmt (plugin->dbh, NULL);
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sqlite",
+                         "Failed to close statement %p: %d\n",
+                         stmt,
+                         result);
+      stmt = sqlite3_next_stmt (plugin->dbh,
+                                NULL);
     }
     result = sqlite3_close (plugin->dbh);
   }
   if (SQLITE_OK != result)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
 
   GNUNET_free_non_null (plugin->fn);
 }
@@ -407,7 +428,13 @@ namestore_sqlite_store_records (void *cls,
     return GNUNET_SYSERR;
   }
   {
+    /* First delete 'old' records */
     char data[data_size];
+    struct GNUNET_SQ_QueryParam dparams[] = {
+      GNUNET_SQ_query_param_auto_from_type (zone_key),
+      GNUNET_SQ_query_param_string (label),
+      GNUNET_SQ_query_param_end
+    };
 
     if (data_size !=
         GNUNET_GNSRECORD_records_serialize (rd_count,
@@ -418,92 +445,71 @@ namestore_sqlite_store_records (void *cls,
       GNUNET_break (0);
       return GNUNET_SYSERR;
     }
-
-    /* First delete 'old' records */
-    if ((SQLITE_OK !=
-         sqlite3_bind_blob (plugin->delete_records,
-                            1,
-                            zone_key,
-                            sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (plugin->delete_records,
-                            2,
-                            label,
-                            -1,
-                            SQLITE_STATIC)))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->delete_records,
+                        dparams))
     {
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_bind_XXXX");
-      if (SQLITE_OK != sqlite3_reset (plugin->delete_records))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->delete_records);
       return GNUNET_SYSERR;
 
     }
     n = sqlite3_step (plugin->delete_records);
-    if (SQLITE_OK != sqlite3_reset (plugin->delete_records))
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delete_records);
 
     if (0 != rd_count)
     {
-      if ((SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records,
-                              1,
-                              zone_key,
-                              sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records,
-                              2,
-                              &pkey,
-                              sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_int64 (plugin->store_records, 3, rvalue)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_int (plugin->store_records, 4, rd_count)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records, 5,
-                              data, data_size,
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_text (plugin->store_records, 6,
-                              label, -1,
-                              SQLITE_STATIC)))
+      uint32_t rd_count32 = (uint32_t) rd_count;
+      struct GNUNET_SQ_QueryParam sparams[] = {
+        GNUNET_SQ_query_param_auto_from_type (zone_key),
+        GNUNET_SQ_query_param_auto_from_type (&rvalue),
+        GNUNET_SQ_query_param_uint64 (&rvalue),
+        GNUNET_SQ_query_param_uint32 (&rd_count32),
+        GNUNET_SQ_query_param_fixed_size (data, data_size),
+        GNUNET_SQ_query_param_string (label),
+        GNUNET_SQ_query_param_end
+      };
+
+      if (GNUNET_OK !=
+          GNUNET_SQ_bind (plugin->store_records,
+                          sparams))
       {
         LOG_SQLITE (plugin,
                     GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                     "sqlite3_bind_XXXX");
-        if (SQLITE_OK != sqlite3_reset (plugin->store_records))
-          LOG_SQLITE (plugin,
-                      GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                      "sqlite3_reset");
+        GNUNET_SQ_reset (plugin->dbh,
+                         plugin->store_records);
         return GNUNET_SYSERR;
       }
       n = sqlite3_step (plugin->store_records);
-      if (SQLITE_OK != sqlite3_reset (plugin->store_records))
-        LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->store_records);
     }
   }
   switch (n)
   {
   case SQLITE_DONE:
     if (0 != rd_count)
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Record stored\n");
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Record stored\n");
     else
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Record deleted\n");
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Record deleted\n");
     return GNUNET_OK;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_NO;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_SYSERR;
   }
@@ -526,37 +532,47 @@ static int
 get_record_and_call_iterator (struct Plugin *plugin,
                               sqlite3_stmt *stmt,
                               const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
-                              GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                              GNUNET_NAMESTORE_RecordIterator iter,
+                              void *iter_cls)
 {
-  unsigned int record_count;
+  uint32_t record_count;
   size_t data_size;
-  const char *data;
-  const char *label;
+  void *data;
+  char *label;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
   int ret;
   int sret;
 
   ret = GNUNET_NO;
   if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
   {
-    record_count = sqlite3_column_int (stmt, 0);
-    data_size = sqlite3_column_bytes (stmt, 1);
-    data = sqlite3_column_blob (stmt, 1);
-    label = (const char*) sqlite3_column_text (stmt, 2);
+    struct GNUNET_SQ_ResultSpec rs[] = {
+      GNUNET_SQ_result_spec_uint32 (&record_count),
+      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+      GNUNET_SQ_result_spec_string (&label),
+      GNUNET_SQ_result_spec_end
+    };
+    struct GNUNET_SQ_ResultSpec rsx[] = {
+      GNUNET_SQ_result_spec_uint32 (&record_count),
+      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+      GNUNET_SQ_result_spec_string (&label),
+      GNUNET_SQ_result_spec_auto_from_type (&zk),
+      GNUNET_SQ_result_spec_end
+    };
+
     if (NULL == zone_key)
     {
-      /* must be "iterate_all_zones", got one extra return value */
-      if (sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey) !=
-          sqlite3_column_bytes (stmt, 3))
-      {
-        GNUNET_break (0);
-        ret = GNUNET_SYSERR;
-      }
-      else
-      {
-        zone_key = sqlite3_column_blob (stmt, 3);
-      }
+      zone_key = &zk;
+      ret = GNUNET_SQ_extract_result (stmt,
+                                      rsx);
+    }
+    else
+    {
+      ret = GNUNET_SQ_extract_result (stmt,
+                                      rs);
     }
-    if (record_count > 64 * 1024)
+    if ( (GNUNET_OK != ret) ||
+         (record_count > 64 * 1024) )
     {
       /* sanity check, don't stack allocate far too much just
          because database might contain a large value here */
@@ -568,32 +584,40 @@ get_record_and_call_iterator (struct Plugin *plugin,
       struct GNUNET_GNSRECORD_Data rd[record_count];
 
       if (GNUNET_OK !=
-          GNUNET_GNSRECORD_records_deserialize (data_size, data,
-                                                record_count, rd))
+          GNUNET_GNSRECORD_records_deserialize (data_size,
+                                                data,
+                                                record_count,
+                                                rd))
       {
         GNUNET_break (0);
         ret = GNUNET_SYSERR;
       }
-      else if (NULL != zone_key)
+      else
       {
         if (NULL != iter)
-                iter (iter_cls, zone_key, label, record_count, rd);
+          iter (iter_cls,
+                zone_key,
+                label,
+                record_count,
+                rd);
         ret = GNUNET_YES;
       }
     }
+    GNUNET_SQ_cleanup_result (rs);
   }
   else
   {
     if (SQLITE_DONE != sret)
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+      LOG_SQLITE (plugin,
+                  GNUNET_ERROR_TYPE_ERROR,
+                  "sqlite_step");
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   return ret;
 }
 
+
 /**
  * Lookup records in the datastore for which we are the authority.
  *
@@ -606,37 +630,35 @@ get_record_and_call_iterator (struct Plugin *plugin,
  */
 static int
 namestore_sqlite_lookup_records (void *cls,
-    const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, const char *label,
-    GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
+                                 const char *label,
+                                 GNUNET_NAMESTORE_RecordIterator iter,
+                                 void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
-  int err;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (zone),
+    GNUNET_SQ_query_param_string (label),
+    GNUNET_SQ_query_param_end
+  };
 
   if (NULL == zone)
-  {
     return GNUNET_SYSERR;
-  }
-  else
-  {
-    stmt = plugin->lookup_label;
-    err = ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                             zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                             SQLITE_STATIC)) ||
-            (SQLITE_OK != sqlite3_bind_text (stmt, 2,
-                                              label, -1, SQLITE_STATIC)) );
-  }
-  if (err)
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->lookup_label,
+                      params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->lookup_label);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       plugin->lookup_label,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
@@ -655,7 +677,8 @@ static int
 namestore_sqlite_iterate_records (void *cls,
                                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
                                   uint64_t offset,
-                                  GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                                  GNUNET_NAMESTORE_RecordIterator iter,
+                                  void *iter_cls)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
@@ -663,30 +686,41 @@ namestore_sqlite_iterate_records (void *cls,
 
   if (NULL == zone)
   {
+    struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_uint64 (&offset),
+      GNUNET_SQ_query_param_end
+    };
+
     stmt = plugin->iterate_all_zones;
-    err = (SQLITE_OK != sqlite3_bind_int64 (stmt, 1,
-                                            offset));
+    err = GNUNET_SQ_bind (stmt,
+                          params);
   }
   else
   {
+    struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_auto_from_type (zone),
+      GNUNET_SQ_query_param_uint64 (&offset),
+      GNUNET_SQ_query_param_end
+    };
+
     stmt = plugin->iterate_zone;
-    err = ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                             zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                             SQLITE_STATIC)) ||
-            (SQLITE_OK != sqlite3_bind_int64 (stmt, 2,
-                                              offset)) );
+    err = GNUNET_SQ_bind (stmt,
+                          params);
   }
-  if (err)
+  if (GNUNET_OK != err)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       stmt,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
@@ -708,29 +742,31 @@ namestore_sqlite_zone_to_name (void *cls,
                                GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (zone),
+    GNUNET_SQ_query_param_auto_from_type (value_zone),
+    GNUNET_SQ_query_param_end
+  };
 
-  stmt = plugin->zone_to_name;
-  if ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                        zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                        SQLITE_STATIC)) ||
-       (SQLITE_OK != sqlite3_bind_blob (stmt, 2,
-                                        value_zone, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
-                                        SQLITE_STATIC)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->zone_to_name,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->zone_to_name);
     return GNUNET_SYSERR;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Performing reverse lookup for `%s'\n",
        GNUNET_GNSRECORD_z2s (value_zone));
-
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       plugin->zone_to_name,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
diff --git a/src/namestore/plugin_rest_namestore.c b/src/namestore/plugin_rest_namestore.c
index 0d36cf445..50957a5b4 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -899,10 +899,11 @@ namestore_zkey_cont (struct GNUNET_REST_RequestHandle *con,
   }
   handle->zkey_str = GNUNET_CONTAINER_multihashmap_get (handle->rest_handle->url_param_map,
                                                         &key);
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_ecdsa_public_key_from_string (handle->zkey_str,
+  if ((NULL == handle->zkey_str) ||
+      (GNUNET_OK !=
+       GNUNET_CRYPTO_ecdsa_public_key_from_string (handle->zkey_str,
                                                   strlen (handle->zkey_str),
-                                                  &pubkey))
+                                                   &pubkey)))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Zkey invalid %s\n", handle->zkey_str);
@@ -1126,8 +1127,8 @@ rest_identity_process_request(struct GNUNET_REST_RequestHandle *rest_handle,
   {
     type = GNUNET_CONTAINER_multihashmap_get (handle->rest_handle->url_param_map,
                                               &key);
-
-    handle->type = GNUNET_GNSRECORD_typename_to_number (type);
+    if (NULL != type)
+      handle->type = GNUNET_GNSRECORD_typename_to_number (type);
   }
   name = get_name_from_url (handle->url);
   if (NULL != ego)
diff --git a/src/namestore/test_namestore_api_remove.c b/src/namestore/test_namestore_api_remove.c
index c6c439e86..2d670c1ee 100644
--- a/src/namestore/test_namestore_api_remove.c
+++ b/src/namestore/test_namestore_api_remove.c
@@ -98,19 +98,22 @@ remove_cont (void *cls,
              int32_t success,
              const char *emsg)
 {
+  nsqe = NULL;
   if (GNUNET_YES != success)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-              _("Records could not be removed: `%s'\n"), emsg);
-    if (endbadly_task != NULL)
+                _("Records could not be removed: `%s'\n"),
+                emsg);
+    if (NULL != endbadly_task)
       GNUNET_SCHEDULER_cancel (endbadly_task);
-    endbadly_task =  GNUNET_SCHEDULER_add_now (&endbadly, NULL);
+    endbadly_task =  GNUNET_SCHEDULER_add_now (&endbadly,
+                                               NULL);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Records were removed, perform lookup\n");
   removed = GNUNET_YES;
-  if (endbadly_task != NULL)
+  if (NULL != endbadly_task)
     GNUNET_SCHEDULER_cancel (endbadly_task);
   GNUNET_SCHEDULER_add_now (&end, NULL);
 }
@@ -139,8 +142,11 @@ put_cont (void *cls, int32_t success,
               "Name store added record for `%s': %s\n",
               name,
               (success == GNUNET_OK) ? "SUCCESS" : "FAIL");
-  nsqe = GNUNET_NAMESTORE_records_store (nsh, privkey, name,
-                                         0, NULL, &remove_cont, (void *) name);
+  nsqe = GNUNET_NAMESTORE_records_store (nsh,
+                                         privkey,
+                                         name,
+                                         0, NULL,
+                                         &remove_cont, (void *) name);
 }
 
 
diff --git a/src/nat-auto/Makefile.am b/src/nat-auto/Makefile.am
index 19695fabd..14f3f2330 100644
--- a/src/nat-auto/Makefile.am
+++ b/src/nat-auto/Makefile.am
@@ -37,11 +37,11 @@ lib_LTLIBRARIES = \
 
 libgnunetnatauto_la_SOURCES = \
   nat_auto_api.c \
-  nat_auto_api_test.c 
+  nat_auto_api_test.c
 libgnunetnatauto_la_LIBADD = \
   $(top_builddir)/src/nat/libgnunetnatnew.la \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(GN_LIBINTL) @EXT_LIBS@ 
+  $(GN_LIBINTL) @EXT_LIBS@
 libgnunetnatauto_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS) $(WINFLAGS) \
   -version-info 0:0:0
@@ -55,4 +55,3 @@ gnunet_service_nat_auto_LDADD = \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
   $(GN_LIBINTL)
-
diff --git a/src/nat-auto/gnunet-nat-auto.c b/src/nat-auto/gnunet-nat-auto.c
index 9ba81eb5f..7d09253b8 100644
--- a/src/nat-auto/gnunet-nat-auto.c
+++ b/src/nat-auto/gnunet-nat-auto.c
@@ -63,7 +63,7 @@ static char *section_name;
 /**
  * Should we run autoconfiguration?
  */
-static unsigned int do_auto;
+static int do_auto;
 
 /**
  * Handle to a NAT test operation.
@@ -174,6 +174,9 @@ auto_config_cb (void *cls,
               GNUNET_NAT_AUTO_status2string (result),
               nat_type);
 
+  if (NULL == diff)
+    return;
+
   /* Shortcut: if there are no changes suggested, bail out early. */
   if (GNUNET_NO ==
       GNUNET_CONFIGURATION_is_dirty (diff))
@@ -186,20 +189,16 @@ auto_config_cb (void *cls,
      to the user */
   new_cfg = write_cfg ? GNUNET_CONFIGURATION_dup (cfg) : NULL;
 
-  if (NULL != diff)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-                _("Suggested configuration changes:\n"));
-    GNUNET_CONFIGURATION_iterate_section_values (diff,
-                                                 "nat",
-                                                 &auto_conf_iter,
-                                                 new_cfg);
-  }
+  GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+              _("Suggested configuration changes:\n"));
+  GNUNET_CONFIGURATION_iterate_section_values (diff,
+                                               "nat",
+                                               &auto_conf_iter,
+                                               new_cfg);
 
   /* If desired, write configuration to file; we write only the
      changes to the defaults to keep things compact. */
-  if ( (write_cfg) &&
-       (NULL != diff) )
+  if (write_cfg)
   {
     struct GNUNET_CONFIGURATION_Handle *def_cfg;
 
@@ -298,8 +297,8 @@ run (void *cls,
   if (do_auto)
   {
     ah = GNUNET_NAT_AUTO_autoconfig_start (c,
-                                      &auto_config_cb,
-                                      NULL);
+                                           &auto_config_cb,
+                                           NULL);
   }
 
   if (use_tcp && use_udp)
@@ -340,22 +339,32 @@ int
 main (int argc,
       char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "auto", NULL,
-     gettext_noop ("run autoconfiguration"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &do_auto },
-    {'S', "section", "NAME",
-     gettext_noop ("section name providing the configuration for the adapter"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &section_name },
-    {'t', "tcp", NULL,
-     gettext_noop ("use TCP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_tcp },
-    {'u', "udp", NULL,
-     gettext_noop ("use UDP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_udp },
-    {'w', "write", NULL,
-     gettext_noop ("write configuration file (for autoconfiguration)"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &write_cfg },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('a',
+                                  "auto",
+                                  gettext_noop ("run autoconfiguration"),
+                                  &do_auto),
+
+    GNUNET_GETOPT_option_string ('S',
+                                 "section",
+                                 "NAME",
+                                 gettext_noop ("section name providing the configuration for the adapter"),
+                                 &section_name),
+
+    GNUNET_GETOPT_option_flag ('t',
+                                   "tcp",
+                                   gettext_noop ("use TCP"),
+                                   &use_tcp),
+
+    GNUNET_GETOPT_option_flag ('u',
+                                   "udp",
+                                   gettext_noop ("use UDP"),
+                                   &use_udp),
+
+    GNUNET_GETOPT_option_flag ('w',
+                                   "write",
+                                   gettext_noop ("write configuration file (for autoconfiguration)"),
+                                   &write_cfg),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/nat-auto/gnunet-nat-server.c b/src/nat-auto/gnunet-nat-server.c
index dd08f8d36..590fad4d6 100644
--- a/src/nat-auto/gnunet-nat-server.c
+++ b/src/nat-auto/gnunet-nat-server.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
+     Copyright (C) 2011, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -31,9 +31,21 @@
 
 
 /**
- * Our server.
+ * Information we track per client.
  */
-static struct GNUNET_SERVER_Handle *server;
+struct ClientData
+{
+  /**
+   * Timeout task.
+   */
+  struct GNUNET_SCHEDULER_Task *tt;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+};
+
 
 /**
  * Our configuration.
@@ -56,6 +68,7 @@ try_anat (uint32_t dst_ipv4,
   struct GNUNET_NAT_Handle *h;
   struct sockaddr_in lsa;
   struct sockaddr_in rsa;
+  const struct sockaddr *sa;
   socklen_t sa_len;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -77,11 +90,12 @@ try_anat (uint32_t dst_ipv4,
   rsa.sin_addr.s_addr = dst_ipv4;
   rsa.sin_port = htons (dport);
   sa_len = sizeof (lsa);
+  sa = (const struct sockaddr *) &lsa;
   h = GNUNET_NAT_register (cfg,
                            "none",
                            is_tcp ? IPPROTO_TCP : IPPROTO_UDP,
                            1,
-                           (const struct sockaddr **) &lsa,
+                           &sa,
                            &sa_len,
                            NULL, NULL, NULL);
   GNUNET_NAT_request_reversal (h,
@@ -246,21 +260,18 @@ try_send_udp (uint32_t dst_ipv4,
  * We've received a request to probe a NAT
  * traversal. Do it.
  *
- * @param cls unused
- * @param client handle to client (we always close)
+ * @param cls handle to client (we always close)
  * @param msg message with details about what to test
  */
 static void
-test (void *cls,
-      struct GNUNET_SERVER_Client *client,
-      const struct GNUNET_MessageHeader *msg)
+handle_test (void *cls,
+             const struct GNUNET_NAT_AUTO_TestMessage *tm)
 {
-  const struct GNUNET_NAT_AUTO_TestMessage *tm;
+  struct ClientData *cd = cls;
   uint16_t dport;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received test request\n");
-  tm = (const struct GNUNET_NAT_AUTO_TestMessage *) msg;
   dport = ntohs (tm->dport);
   if (0 == dport)
     try_anat (tm->dst_ipv4,
@@ -274,126 +285,119 @@ test (void *cls,
     try_send_udp (tm->dst_ipv4,
                   dport,
                   tm->data);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_NO);
+  GNUNET_SERVICE_client_drop (cd->client);
 }
 
 
 /**
- * Task run during shutdown.
+ * Main function that will be run.
  *
- * @param cls unused
+ * @param cls closure
+ * @param c configuration
+ * @param srv service handle
  */
 static void
-shutdown_task (void *cls)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *srv)
 {
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
+  cfg = c;
 }
 
 
 /**
- * Main function that will be run.
+ * Forcefully drops client after 1s.
  *
- * @param cls closure
- * @param args remaining command-line arguments
- * @param cfgfile name of the configuration file used (for saving, can be NULL!)
- * @param c configuration
+ * @param cls our `struct ClientData` of a client to drop
  */
 static void
-run (void *cls,
-     char *const *args,
-     const char *cfgfile,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+force_timeout (void *cls)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {&test, NULL, GNUNET_MESSAGE_TYPE_NAT_TEST,
-     sizeof (struct GNUNET_NAT_AUTO_TestMessage)},
-    {NULL, NULL, 0, 0}
-  };
-  unsigned int port;
-  struct sockaddr_in in4;
-  struct sockaddr_in6 in6;
-
-  socklen_t slen[] = {
-    sizeof (in4),
-    sizeof (in6),
-    0
-  };
-  struct sockaddr *sa[] = {
-    (struct sockaddr *) &in4,
-    (struct sockaddr *) &in6,
-    NULL
-  };
+  struct ClientData *cd = cls;
 
-  cfg = c;
-  if ( (NULL == args[0]) ||
-       (1 != SSCANF (args[0], "%u", &port)) ||
-       (0 == port) ||
-       (65536 <= port) )
-  {
-    FPRINTF (stderr,
-             _("Please pass valid port number as the first argument! (got `%s')\n"),
-             args[0]);
-    return;
-  }
-  memset (&in4, 0, sizeof (in4));
-  memset (&in6, 0, sizeof (in6));
-  in4.sin_family = AF_INET;
-  in4.sin_port = htons ((uint16_t) port);
-  in6.sin6_family = AF_INET6;
-  in6.sin6_port = htons ((uint16_t) port);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  in4.sin_len = sizeof (in4);
-  in6.sin6_len = sizeof (in6);
-#endif
-  server = GNUNET_SERVER_create (NULL,
-                                 NULL,
-                                 (struct sockaddr * const *) sa,
-                                 slen,
-                                 GNUNET_TIME_UNIT_SECONDS,
-                                 GNUNET_YES);
-  GNUNET_SERVER_add_handlers (server,
-                              handlers);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
+  cd->tt = NULL;
+  GNUNET_SERVICE_client_drop (cd->client);
 }
 
 
+
 /**
- * Main function of gnunet-nat-server.
+ * Callback called when a client connects to the service.
  *
- * @param argc number of command-line arguments
- * @param argv command line
- * @return 0 on success, -1 on error
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return our `struct ClientData`
  */
-int
-main (int argc, char *const argv[])
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    GNUNET_GETOPT_OPTION_END
-  };
-
-  if (GNUNET_OK !=
-      GNUNET_STRINGS_get_utf8_args (argc, argv,
-                                    &argc, &argv))
-    return 2;
-
-  if (GNUNET_OK !=
-      GNUNET_PROGRAM_run (argc,
-                          argv,
-                          "gnunet-nat-server [options] PORT",
-                          _("GNUnet NAT traversal test helper daemon"),
-                          options,
-                          &run,
-                          NULL))
-  {
-    GNUNET_free ((void*) argv);
-    return 1;
-  }
-  GNUNET_free ((void*) argv);
-  return 0;
+  struct ClientData *cd;
+
+  cd = GNUNET_new (struct ClientData);
+  cd->client = c;
+  cd->tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
+                                         &force_timeout,
+                                         cd);
+  return cd;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls our `struct ClientData`
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
+{
+  struct ClientData *cd = internal_cls;
+
+  if (NULL != cd->tt)
+    GNUNET_SCHEDULER_cancel (cd->tt);
+  GNUNET_free (cd);
 }
 
 
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("nat-server",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (test,
+                          GNUNET_MESSAGE_TYPE_NAT_TEST,
+                          struct GNUNET_NAT_AUTO_TestMessage,
+                          NULL),
+ GNUNET_MQ_handler_end ());
+
+
+#if defined(LINUX) && defined(__GLIBC__)
+#include <malloc.h>
+
+/**
+ * MINIMIZE heap size (way below 128k) since this process doesn't need much.
+ */
+void __attribute__ ((constructor))
+GNUNET_ARM_memory_init ()
+{
+  mallopt (M_TRIM_THRESHOLD, 4 * 1024);
+  mallopt (M_TOP_PAD, 1 * 1024);
+  malloc_trim (0);
+}
+#endif
+
+
+
+
 /* end of gnunet-nat-server.c */
diff --git a/src/nat-auto/nat_auto_api.c b/src/nat-auto/nat_auto_api.c
index a5b41ac49..8a7eaf264 100644
--- a/src/nat-auto/nat_auto_api.c
+++ b/src/nat-auto/nat_auto_api.c
@@ -42,7 +42,7 @@ struct GNUNET_NAT_AUTO_AutoHandle
    * Configuration we use.
    */
   const struct GNUNET_CONFIGURATION_Handle *cfg;
-  
+
   /**
    * Message queue for communicating with the NAT service.
    */
@@ -206,8 +206,8 @@ ah_error_handler (void *cls,
  */
 struct GNUNET_NAT_AUTO_AutoHandle *
 GNUNET_NAT_AUTO_autoconfig_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                             GNUNET_NAT_AUTO_AutoResultCallback cb,
-                             void *cb_cls)
+                                  GNUNET_NAT_AUTO_AutoResultCallback cb,
+                                  void *cb_cls)
 {
   struct GNUNET_NAT_AUTO_AutoHandle *ah = GNUNET_new (struct GNUNET_NAT_AUTO_AutoHandle);
   struct GNUNET_MQ_MessageHandler handlers[] = {
@@ -224,7 +224,7 @@ GNUNET_NAT_AUTO_autoconfig_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   buf = GNUNET_CONFIGURATION_serialize (cfg,
                                         &size);
-  if (size > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*req))
+  if (size > GNUNET_MAX_MESSAGE_SIZE - sizeof (*req))
   {
     GNUNET_break (0);
     GNUNET_free (buf);
diff --git a/src/nat-auto/nat_auto_api_test.c b/src/nat-auto/nat_auto_api_test.c
index fb2bcd679..c72b611bf 100644
--- a/src/nat-auto/nat_auto_api_test.c
+++ b/src/nat-auto/nat_auto_api_test.c
@@ -28,7 +28,7 @@
 #include "gnunet_nat_auto_service.h"
 #include "nat-auto.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "nat", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "nat-auto", __VA_ARGS__)
 
 #define NAT_SERVER_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
 
@@ -508,19 +508,21 @@ GNUNET_NAT_AUTO_test_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
   {
     nh->lsock
       = GNUNET_NETWORK_socket_create (AF_INET,
-                                      proto,
-                                      0);
+                                      (IPPROTO_UDP == proto)
+                                      ? SOCK_DGRAM
+                                      : SOCK_STREAM,
+                                      proto);
     if ( (NULL == nh->lsock) ||
          (GNUNET_OK !=
           GNUNET_NETWORK_socket_bind (nh->lsock,
                                       (const struct sockaddr *) &sa,
                                       sizeof (sa))))
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Failed to create listen socket bound to `%s' for NAT test: %s\n"),
-                  GNUNET_a2s ((const struct sockaddr *) &sa,
-                              sizeof (sa)),
-                  STRERROR (errno));
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Failed to create socket bound to `%s' for NAT test: %s\n"),
+           GNUNET_a2s ((const struct sockaddr *) &sa,
+                       sizeof (sa)),
+           STRERROR (errno));
       if (NULL != nh->lsock)
       {
         GNUNET_NETWORK_socket_close (nh->lsock);
@@ -551,7 +553,7 @@ GNUNET_NAT_AUTO_test_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                                          nh);
     }
     LOG (GNUNET_ERROR_TYPE_INFO,
-         "NAT test listens on port %u (%s)\n",
+         "NAT test listens on port %llu (%s)\n",
          bnd_port,
          (IPPROTO_TCP == proto) ? "tcp" : "udp");
     nh->nat = GNUNET_NAT_register (cfg,
diff --git a/src/nat/Makefile.am b/src/nat/Makefile.am
index 3dc001dd7..f0d5639a1 100644
--- a/src/nat/Makefile.am
+++ b/src/nat/Makefile.am
@@ -93,7 +93,7 @@ gnunet_service_nat_LDADD = \
 #  test_stun
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/nat/gnunet-helper-nat-client-windows.c b/src/nat/gnunet-helper-nat-client-windows.c
index 89dad9e7b..622f8c961 100644
--- a/src/nat/gnunet-helper-nat-client-windows.c
+++ b/src/nat/gnunet-helper-nat-client-windows.c
@@ -42,6 +42,8 @@
  * - Nathan Evans
  */
 #define _GNU_SOURCE
+/* Instead of including gnunet_common.h */
+#define GNUNET_memcpy(dst,src,n) do { if (0 != n) { (void) memcpy (dst,src,n); } } while (0)
 
 #define FD_SETSIZE 1024
 #include <winsock2.h>
diff --git a/src/nat/gnunet-helper-nat-server-windows.c b/src/nat/gnunet-helper-nat-server-windows.c
index c8e1193e4..09bd02538 100644
--- a/src/nat/gnunet-helper-nat-server-windows.c
+++ b/src/nat/gnunet-helper-nat-server-windows.c
@@ -41,6 +41,8 @@
  * - Christian Grothoff
  */
 #define _GNU_SOURCE
+/* Instead of including gnunet_common.h */
+#define GNUNET_memcpy(dst,src,n) do { if (0 != n) { (void) memcpy (dst,src,n); } } while (0)
 
 #define FD_SETSIZE 1024
 #include <winsock2.h>
diff --git a/src/nat/gnunet-nat.c b/src/nat/gnunet-nat.c
index f198adc0a..b3cf2e946 100644
--- a/src/nat/gnunet-nat.c
+++ b/src/nat/gnunet-nat.c
@@ -34,9 +34,9 @@
 static int global_ret;
 
 /**
- * Name of section in configuration file to use for 
+ * Name of section in configuration file to use for
  * additional options.
- */ 
+ */
 static char *section_name;
 
 /**
@@ -72,7 +72,7 @@ static char *remote_addr;
 /**
  * Should we actually bind to #bind_addr and receive and process STUN requests?
  */
-static unsigned int do_stun;
+static int do_stun;
 
 /**
  * Handle to NAT operation.
@@ -81,7 +81,7 @@ static struct GNUNET_NAT_Handle *nh;
 
 /**
  * Listen socket for STUN processing.
- */ 
+ */
 static struct GNUNET_NETWORK_Handle *ls;
 
 /**
@@ -110,7 +110,7 @@ test_finished ()
  * a function to call whenever our set of 'valid' addresses changes.
  *
  * @param cls closure, NULL
- * @param add_remove #GNUNET_YES to add a new public IP address, 
+ * @param add_remove #GNUNET_YES to add a new public IP address,
  *                   #GNUNET_NO to remove a previous (now invalid) one
  * @param ac address class the address belongs to
  * @param addr either the previous or the new public IP address
@@ -123,12 +123,12 @@ address_cb (void *cls,
             const struct sockaddr *addr,
             socklen_t addrlen)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-              "%s %s (%d)\n",
-              add_remove ? "+" : "-",
-              GNUNET_a2s (addr,
-                          addrlen),
-              (int) ac);
+  fprintf (stdout,
+           "%s %s (%d)\n",
+           add_remove ? "+" : "-",
+           GNUNET_a2s (addr,
+                       addrlen),
+           (int) ac);
 }
 
 
@@ -186,7 +186,7 @@ static void
 stun_read_task (void *cls)
 {
   ssize_t size;
-  
+
   rtask = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
                                          ls,
                                          &stun_read_task,
@@ -204,7 +204,7 @@ stun_read_task (void *cls)
     struct sockaddr_storage sa;
     socklen_t salen = sizeof (sa);
     ssize_t ret;
-    
+
     ret = GNUNET_NETWORK_socket_recvfrom (ls,
                                           buf,
                                           size + 1,
@@ -269,6 +269,10 @@ run (void *cls,
     global_ret = 1;
     return;
   }
+  local_len = 0;
+  local_sa = NULL;
+  remote_len = 0;
+  remote_sa = NULL;
   if (NULL != local_addr)
   {
     local_len = (socklen_t) GNUNET_STRINGS_parse_socket_addr (local_addr,
@@ -279,10 +283,10 @@ run (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Invalid socket address `%s'\n",
                   local_addr);
-      global_ret = 1;
-      return;
+      goto fail_and_shutdown;
     }
   }
+
   if (NULL != remote_addr)
   {
     remote_len = GNUNET_STRINGS_parse_socket_addr (remote_addr,
@@ -293,8 +297,7 @@ run (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Invalid socket address `%s'\n",
                   remote_addr);
-      global_ret = 1;
-      return;
+      goto fail_and_shutdown;
     }
   }
 
@@ -315,32 +318,26 @@ run (void *cls,
   else if (listen_reversal)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-                "Use of `-W` only effective in combination with `-i`\n");    
-    global_ret = 1;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+                "Use of `-W` only effective in combination with `-i`\n");
+    goto fail_and_shutdown;
   }
 
   if (NULL != remote_addr)
   {
     int ret;
-    
+
     if ( (NULL == nh) ||
          (sizeof (struct sockaddr_in) != local_len) )
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require IPv4 local address to initiate connection reversal\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     if (sizeof (struct sockaddr_in) != remote_len)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require IPv4 reversal target address\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     GNUNET_assert (AF_INET == local_sa->sa_family);
     GNUNET_assert (AF_INET == remote_sa->sa_family);
@@ -362,24 +359,20 @@ run (void *cls,
       break;
     }
   }
-  
+
   if (do_stun)
   {
     if (NULL == local_addr)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require local address to support STUN requests\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     if (IPPROTO_UDP != proto)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "STUN only supported over UDP\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     ls = GNUNET_NETWORK_socket_create (af,
                                        SOCK_DGRAM,
@@ -394,17 +387,22 @@ run (void *cls,
                   GNUNET_a2s (local_sa,
                               local_len),
                   STRERROR (errno));
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     rtask = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
                                            ls,
                                            &stun_read_task,
                                            NULL);
   }
-
+  GNUNET_free_non_null (remote_sa);
+  GNUNET_free_non_null (local_sa);
   test_finished ();
+  return;
+ fail_and_shutdown:
+  global_ret = 1;
+  GNUNET_SCHEDULER_shutdown ();
+  GNUNET_free_non_null (remote_sa);
+  GNUNET_free_non_null (local_sa);
 }
 
 
@@ -419,29 +417,46 @@ int
 main (int argc,
       char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "in", "ADDRESS",
-     gettext_noop ("which IP and port are we locally using to bind/listen to"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &local_addr },
-    {'r', "remote", "ADDRESS",
-     gettext_noop ("which remote IP and port should be asked for connection reversal"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &remote_addr },
-    {'S', "section", NULL,
-     gettext_noop ("name of configuration section to find additional options, such as manual host punching data"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &section_name },
-    {'s', "stun", NULL,
-     gettext_noop ("enable STUN processing"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &do_stun },
-    {'t', "tcp", NULL,
-     gettext_noop ("use TCP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_tcp },
-    {'u', "udp", NULL,
-     gettext_noop ("use UDP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_udp },
-    {'W', "watch", NULL,
-     gettext_noop ("watch for connection reversal requests"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &listen_reversal },
-   GNUNET_GETOPT_OPTION_END
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('i',
+                                 "in",
+                                 "ADDRESS",
+                                 gettext_noop ("which IP and port are we locally using to bind/listen to"),
+                                 &local_addr),
+
+    GNUNET_GETOPT_option_string ('r',
+                                 "remote",
+                                 "ADDRESS",
+                                 gettext_noop ("which remote IP and port should be asked for connection reversal"),
+                                 &remote_addr),
+
+    GNUNET_GETOPT_option_string ('S',
+                                 "section",
+                                 NULL,
+                                 gettext_noop ("name of configuration section to find additional options, such as manual host punching data"),
+                                 &section_name),
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "stun",
+                                  gettext_noop ("enable STUN processing"),
+                                  &do_stun),
+
+    GNUNET_GETOPT_option_flag ('t',
+                                  "tcp",
+                                  gettext_noop ("use TCP"),
+                                  &use_tcp),
+
+    GNUNET_GETOPT_option_flag ('u',
+                                  "udp",
+                                  gettext_noop ("use UDP"),
+                                  &use_udp),
+
+    GNUNET_GETOPT_option_flag ('W',
+                                  "watch",
+                                  gettext_noop ("watch for connection reversal requests"),
+                                  &listen_reversal),
+    GNUNET_GETOPT_OPTION_END
   };
 
   if (GNUNET_OK !=
diff --git a/src/nat/gnunet-service-nat.c b/src/nat/gnunet-service-nat.c
index bfe212308..0695c7ac7 100644
--- a/src/nat/gnunet-service-nat.c
+++ b/src/nat/gnunet-service-nat.c
@@ -662,6 +662,11 @@ notify_client (enum GNUNET_NAT_AddressClass ac,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_NAT_AddressChangeNotificationMessage *msg;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Notifying client about %s of IP %s\n",
+              add ? "addition" : "removal",
+              GNUNET_a2s (addr,
+                          addr_len));
   env = GNUNET_MQ_msg_extra (msg,
                              addr_len,
                              GNUNET_MESSAGE_TYPE_NAT_ADDRESS_CHANGE);
@@ -693,7 +698,11 @@ check_notify_client (struct LocalAddressList *delta,
   struct sockaddr_in6 v6;
 
   if (0 == (ch->flags & GNUNET_NAT_RF_ADDRESSES))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Not notifying client as it does not care about addresses\n");
     return;
+  }
   switch (delta->af)
   {
   case AF_INET:
@@ -715,25 +724,24 @@ check_notify_client (struct LocalAddressList *delta,
            (! match_ipv4 ("127.0.0.1", &v4.sin_addr, 8)) )
         continue; /* bound to loopback, but this is not loopback */
       if ( (! match_ipv4 ("127.0.0.1", &c4->sin_addr, 8) ) &&
-           (0 != c4->sin_addr.s_addr) &&
            match_ipv4 ("127.0.0.1", &v4.sin_addr, 8) )
         continue; /* bound to non-loopback, but this is loopback */
-      if ( (0 != (ch->flags & GNUNET_NAT_AC_EXTERN)) &&
-           (0 != c4->sin_addr.s_addr) &&
-           (! is_nat_v4 (&v4.sin_addr)) )
-        continue; /* based on external-IP, but this IP is not
-                     from private address range. */
+      if ( (0 != (delta->ac & GNUNET_NAT_AC_EXTERN)) &&
+           (0 != c4->sin_addr.s_addr) &&
+           (! is_nat_v4 (&v4.sin_addr)) )
+       continue; /* based on external-IP, but this IP is not
+                    from private address range. */
       if ( (0 != memcmp (&v4.sin_addr,
-                         &c4->sin_addr,
-                         sizeof (struct in_addr))) &&
-           (0 != c4->sin_addr.s_addr) &&
-           ( (! is_nat_v4 (&c4->sin_addr)) ||
-             (0 == (ch->flags & GNUNET_NAT_AC_EXTERN))) )
+                         &c4->sin_addr,
+                         sizeof (struct in_addr))) &&
+           (0 != c4->sin_addr.s_addr) &&
+           (! is_nat_v4 (&c4->sin_addr)) )
         continue; /* this IP is not from private address range,
                      and IP does not match. */
 
       /* OK, IP seems relevant, notify client */
-      v4.sin_port = c4->sin_port;
+      if (0 == htons (v4.sin_port))
+        v4.sin_port = c4->sin_port;
       notify_client (delta->ac,
                      ch,
                      add,
@@ -760,13 +768,10 @@ check_notify_client (struct LocalAddressList *delta,
            (! match_ipv6 ("::1", &v6.sin6_addr, 128)) )
         continue; /* bound to loopback, but this is not loopback */
       if ( (! match_ipv6 ("::1", &c6->sin6_addr, 128) ) &&
-           (0 != memcmp (&c6->sin6_addr,
-                         &in6addr_any,
-                         sizeof (struct in6_addr))) &&
            match_ipv6 ("::1", &v6.sin6_addr, 128) )
         continue; /* bound to non-loopback, but this is loopback */
-      if ( (0 != (ch->flags & GNUNET_NAT_AC_EXTERN)) &&
-           (0 != memcmp (&c6->sin6_addr,
+      if ( (0 != (delta->ac & GNUNET_NAT_AC_EXTERN)) &&
+           (0 != memcmp (&c6->sin6_addr,
                          &in6addr_any,
                          sizeof (struct in6_addr))) &&
            (! is_nat_v6 (&v6.sin6_addr)) )
@@ -793,7 +798,8 @@ check_notify_client (struct LocalAddressList *delta,
                      does not match and is not an external IP */
 
       /* OK, IP seems relevant, notify client */
-      v6.sin6_port = c6->sin6_port;
+      if (0 == htons (v6.sin6_port))
+        v6.sin6_port = c6->sin6_port;
       notify_client (delta->ac,
                      ch,
                      add,
@@ -853,6 +859,10 @@ notify_client_external_ipv4_change (void *cls,
     struct LocalAddressList lal;
     struct sockaddr_in *s4;
 
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Detected eternal IP, can now back-fill AUTO:%u in hole punching specification of `%s'\n",
+                (unsigned int) ch->ext_dns_port,
+                ch->section_name);
     memset (&lal, 0, sizeof (lal));
     s4 = (struct sockaddr_in *) &lal.addr;
     s4->sin_family = AF_INET;
@@ -868,8 +878,6 @@ notify_client_external_ipv4_change (void *cls,
   /* (1) check if client cares. */
   if (! ch->natted_address)
     return;
-  if (0 == (GNUNET_NAT_RF_ADDRESSES & ch->flags))
-    return;
   have_v4 = GNUNET_NO;
   for (unsigned int i=0;i<ch->num_caddrs;i++)
   {
@@ -891,6 +899,10 @@ notify_client_external_ipv4_change (void *cls,
   sa.sin_addr = *v4;
   sa.sin_port = htons (0);
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Detected eternal IP %s, notifying client of external IP (without port)\n",
+              GNUNET_a2s ((const struct sockaddr *) &sa,
+                          sizeof (sa)));
   /* (3) notify client of change */
   notify_client (is_nat_v4 (v4)
                  ? GNUNET_NAT_AC_EXTERN | GNUNET_NAT_AC_LAN
@@ -1055,7 +1067,8 @@ run_scan (void *cls)
 
         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                     "Found NATed local address %s, starting NAT server\n",
-                    GNUNET_a2s ((void *) &pos->addr, sizeof (*s4)));
+                    GNUNET_a2s ((const struct sockaddr *) &pos->addr,
+                                sizeof (*s4)));
         pos->hc = GN_start_gnunet_nat_server_ (&s4->sin_addr,
                                                &reversal_callback,
                                                pos);
@@ -1280,6 +1293,11 @@ dyndns_lookup (void *cls)
   struct ClientHandle *ch = cls;
   struct LocalAddressList *lal;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Performing DNS lookup for punched hole given for `%s' as `%s:%u'\n",
+              ch->section_name,
+              ch->hole_external,
+              (unsigned int) ch->ext_dns_port);
   for (lal = ch->ext_addr_head; NULL != lal; lal = lal->next)
     lal->old = GNUNET_YES;
   ch->ext_dns_task = NULL;
@@ -1374,6 +1392,11 @@ lookup_hole_external (struct ClientHandle *ch)
                       ch->hole_external,
                       &s4->sin_addr))
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "IPv4 punched hole given for `%s' via `%s:%u'\n",
+                ch->section_name,
+                ch->hole_external,
+                (unsigned int) ch->ext_dns_port);
     s4->sin_port = htons (ch->ext_dns_port);
     lal->af = AF_INET;
     lal->ac = GNUNET_NAT_AC_GLOBAL | GNUNET_NAT_AC_MANUAL;
@@ -1423,8 +1446,6 @@ handle_register (void *cls,
     GNUNET_SERVICE_client_drop (ch->client);
     return;
   }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received REGISTER message from client\n");
   ch->flags = message->flags;
   ch->proto = message->proto;
   ch->num_caddrs = ntohs (message->num_addrs);
@@ -1512,6 +1533,9 @@ handle_register (void *cls,
   ch->section_name
     = GNUNET_strndup (off,
                       ntohs (message->str_len));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received REGISTER message from client for subsystem `%s'\n",
+              ch->section_name);
   if (GNUNET_OK ==
       GNUNET_CONFIGURATION_get_value_string (cfg,
                                              ch->section_name,
diff --git a/src/nat/gnunet-service-nat_externalip.c b/src/nat/gnunet-service-nat_externalip.c
index 979d2f0f5..f79ff4070 100644
--- a/src/nat/gnunet-service-nat_externalip.c
+++ b/src/nat/gnunet-service-nat_externalip.c
@@ -23,15 +23,12 @@
  *
  * This can be implemented using different methods, and we allow
  * the main service to be notified about changes to what we believe
- * is our external IPv4 address.  
+ * is our external IPv4 address.
  *
  * Note that this is explicitly only about NATed systems; if one
  * of our network interfaces has a global IP address this does
  * not count as "external".
  *
- * TODO:
- * - implement NEW logic for external IP detection based on traceroute!
- *
  * @file nat/gnunet-service-nat_externalip.c
  * @brief Functions for monitoring external IPv4 addresses
  * @author Christian Grothoff
@@ -86,7 +83,7 @@ struct GN_ExternalIPMonitor
    * Kept in DLL.
    */
   struct GN_ExternalIPMonitor *prev;
-  
+
   /**
    * Function to call when we believe our external IPv4 address changed.
    */
@@ -131,7 +128,7 @@ static struct in_addr mini_external_ipv4;
 /**
  * Tell relevant clients about a change in our external
  * IPv4 address.
- * 
+ *
  * @param add #GNUNET_YES to add, #GNUNET_NO to remove
  * @param v4 the external address that changed
  */
@@ -162,7 +159,7 @@ run_external_ip (void *cls);
  * We learn our current external IP address.  If it changed,
  * notify all of our applicable clients. Also re-schedule
  * #run_external_ip with an appropriate timeout.
- * 
+ *
  * @param cls NULL
  * @param addr the address, NULL on errors
  * @param result #GNUNET_NAT_ERROR_SUCCESS on success, otherwise the specific error code
@@ -173,7 +170,7 @@ handle_external_ip (void *cls,
                     enum GNUNET_NAT_StatusCode result)
 {
   char buf[INET_ADDRSTRLEN];
-  
+
   probe_external_ip_op = NULL;
   GNUNET_SCHEDULER_cancel (probe_external_ip_task);
   probe_external_ip_task
@@ -185,6 +182,7 @@ handle_external_ip (void *cls,
   switch (result)
   {
   case GNUNET_NAT_ERROR_SUCCESS:
+    GNUNET_assert (NULL != addr);
     if (addr->s_addr == mini_external_ipv4.s_addr)
       return; /* not change */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -204,7 +202,7 @@ handle_external_ip (void *cls,
     if (0 != mini_external_ipv4.s_addr)
       notify_monitors_external_ipv4_change (GNUNET_NO,
                                             &mini_external_ipv4);
-    mini_external_ipv4.s_addr = 0; 
+    mini_external_ipv4.s_addr = 0;
     break;
   }
 }
@@ -254,7 +252,7 @@ GN_nat_status_changed (int have_nat)
                                   NULL);
     return;
   }
-  if (GNUNET_NO == have_nat) 
+  if (GNUNET_NO == have_nat)
   {
     if (NULL != probe_external_ip_task)
     {
diff --git a/src/nat/gnunet-service-nat_helper.c b/src/nat/gnunet-service-nat_helper.c
index e91f63beb..de6531fa8 100644
--- a/src/nat/gnunet-service-nat_helper.c
+++ b/src/nat/gnunet-service-nat_helper.c
@@ -221,6 +221,26 @@ restart_nat_server (void *cls)
   char ia[INET_ADDRSTRLEN];
 
   h->server_read_task = NULL;
+  GNUNET_assert (NULL !=
+                 inet_ntop (AF_INET,
+                            &h->internal_address,
+                            ia,
+                            sizeof (ia)));
+  /* Start the server process */
+  binary
+    = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-nat-server");
+  if (GNUNET_YES !=
+      GNUNET_OS_check_helper_binary (binary,
+                                     GNUNET_YES,
+                                     ia))
+  {
+    /* move instantly to max delay, as this is unlikely to be fixed */
+    h->server_retry_delay
+      = GNUNET_TIME_STD_EXPONENTIAL_BACKOFF_THRESHOLD;
+    GNUNET_free (binary);
+    try_again (h);
+    return;
+  }
   h->server_stdout
     = GNUNET_DISK_pipe (GNUNET_YES, GNUNET_YES,
                         GNUNET_NO, GNUNET_YES);
@@ -228,21 +248,14 @@ restart_nat_server (void *cls)
   {
     GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
                          "pipe");
+    GNUNET_free (binary);
     try_again (h);
     return;
   }
-  GNUNET_assert (NULL !=
-                 inet_ntop (AF_INET,
-                            &h->internal_address,
-                            ia,
-                            sizeof (ia)));
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Starting `%s' at `%s'\n",
               "gnunet-helper-nat-server",
               ia);
-  /* Start the server process */
-  binary
-    = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-nat-server");
   h->server_proc
     = GNUNET_OS_start_process (GNUNET_NO,
                                0,
diff --git a/src/nat/gnunet-service-nat_mini.c b/src/nat/gnunet-service-nat_mini.c
index e5b9d021b..8f1229be1 100644
--- a/src/nat/gnunet-service-nat_mini.c
+++ b/src/nat/gnunet-service-nat_mini.c
@@ -123,7 +123,7 @@ read_external_ipv4 (void *cls)
   {
     /* try to read more */
     eh->off += ret;
-    eh->task 
+    eh->task
       = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
                                         eh->r,
                                         &read_external_ipv4,
@@ -233,7 +233,7 @@ GNUNET_NAT_mini_get_external_ipv4_ (GNUNET_NAT_IPCallback cb,
                               GNUNET_DISK_PIPE_END_WRITE);
   eh->r = GNUNET_DISK_pipe_handle (eh->opipe,
                                    GNUNET_DISK_PIPE_END_READ);
-  eh->task 
+  eh->task
     = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
                                       eh->r,
                                       &read_external_ipv4,
@@ -254,6 +254,8 @@ GNUNET_NAT_mini_get_external_ipv4_cancel_ (struct GNUNET_NAT_ExternalHandle *eh)
   {
     (void) GNUNET_OS_process_kill (eh->eip,
                                    SIGKILL);
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_OS_process_wait (eh->eip));
     GNUNET_OS_process_destroy (eh->eip);
   }
   if (NULL != eh->opipe)
@@ -372,7 +374,7 @@ run_upnpc_r (struct GNUNET_NAT_MiniHandle *mini)
                    sizeof (pstr),
                    "%u",
                    (unsigned int) mini->port);
-  mini->map_cmd 
+  mini->map_cmd
     = GNUNET_OS_command_run (&process_map_output,
                              mini,
                              MAP_TIMEOUT,
@@ -516,7 +518,7 @@ do_refresh (void *cls)
   struct GNUNET_NAT_MiniHandle *mini = cls;
   int ac;
 
-  mini->refresh_task 
+  mini->refresh_task
     = GNUNET_SCHEDULER_add_delayed (MAP_REFRESH_FREQ,
                                     &do_refresh,
                                     mini);
@@ -538,7 +540,7 @@ do_refresh (void *cls)
     mini->refresh_cmd = NULL;
     ac = GNUNET_YES;
   }
-  mini->refresh_cmd 
+  mini->refresh_cmd
     = GNUNET_OS_command_run (&process_refresh_output,
                              mini,
                              MAP_TIMEOUT,
@@ -582,7 +584,7 @@ process_map_output (void *cls,
                 0,
                 GNUNET_NAT_ERROR_UPNPC_PORTMAP_FAILED);
     if (NULL == mini->refresh_task)
-      mini->refresh_task 
+      mini->refresh_task
         = GNUNET_SCHEDULER_add_delayed (MAP_REFRESH_FREQ,
                                         &do_refresh,
                                         mini);
@@ -747,7 +749,7 @@ GNUNET_NAT_mini_map_stop (struct GNUNET_NAT_MiniHandle *mini)
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Unmapping port %u with UPnP\n",
        ntohs (mini->current_addr.sin_port));
-  mini->unmap_cmd 
+  mini->unmap_cmd
     = GNUNET_OS_command_run (&process_unmap_output,
                              mini,
                              UNMAP_TIMEOUT,
diff --git a/src/nat/nat_api.c b/src/nat/nat_api.c
index eec5d3968..69612584e 100644
--- a/src/nat/nat_api.c
+++ b/src/nat/nat_api.c
@@ -52,7 +52,7 @@ struct AddrEntry
    * Address class of the address.
    */
   enum GNUNET_NAT_AddressClass ac;
-  
+
   /**
    * Number of bytes that follow.
    */
@@ -70,7 +70,7 @@ struct GNUNET_NAT_Handle
    * Configuration we use.
    */
   const struct GNUNET_CONFIGURATION_Handle *cfg;
-  
+
   /**
    * Message queue for communicating with the NAT service.
    */
@@ -80,7 +80,7 @@ struct GNUNET_NAT_Handle
    * Our registration message.
    */
   struct GNUNET_MessageHeader *reg;
-  
+
   /**
    * Head of address DLL.
    */
@@ -95,12 +95,12 @@ struct GNUNET_NAT_Handle
    * Function to call when our addresses change.
    */
   GNUNET_NAT_AddressCallback address_callback;
-  
+
   /**
    * Function to call when another peer requests connection reversal.
    */
   GNUNET_NAT_ReversalCallback reversal_callback;
-  
+
   /**
    * Closure for the various callbacks.
    */
@@ -136,7 +136,7 @@ static void
 reconnect (struct GNUNET_NAT_Handle *nh)
 {
   struct AddrEntry *ae;
-  
+
   if (NULL != nh->mq)
   {
     GNUNET_MQ_destroy (nh->mq);
@@ -184,7 +184,7 @@ check_connection_reversal_request (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handle connection reversal request.
  *
@@ -247,7 +247,7 @@ check_address_change_notification (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handle connection reversal request.
  *
@@ -264,6 +264,8 @@ handle_address_change_notification (void *cls,
   enum GNUNET_NAT_AddressClass ac;
   struct AddrEntry *ae;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received address change notification\n");
   ac = (enum GNUNET_NAT_AddressClass) ntohl (acn->addr_class);
   if (GNUNET_YES == ntohl (acn->add_remove))
   {
@@ -395,13 +397,13 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
   size_t len;
   size_t str_len;
   char *off;
-  
+
   len = 0;
   for (unsigned int i=0;i<num_addrs;i++)
     len += addrlens[i];
   str_len = strlen (config_section) + 1;
   len += str_len;
-  if ( (len > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*rm)) ||
+  if ( (len > GNUNET_MAX_MESSAGE_SIZE - sizeof (*rm)) ||
        (num_addrs > UINT16_MAX) )
   {
     GNUNET_break (0);
@@ -427,6 +429,7 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_in) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
@@ -434,6 +437,7 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_in6) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
@@ -442,12 +446,14 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_un) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
 #endif
     default:
       GNUNET_break (0);
+      GNUNET_free (rm);
       return NULL;
     }
     GNUNET_memcpy (off,
@@ -569,7 +575,7 @@ test_stun_packet (const void *data,
  *
  * The function does some basic sanity checks on packet size and
  * content, try to extract a bit of information.
- * 
+ *
  * At the moment this only processes BIND requests, and returns the
  * externally visible address of the request to the rest of the
  * NAT logic.
@@ -663,7 +669,7 @@ GNUNET_NAT_test_address (struct GNUNET_NAT_Handle *nh,
  * @param nh handle (used for configuration)
  * @param local_sa our local address of the peer (IPv4-only)
  * @param remote_sa the remote address of the peer (IPv4-only)
- * @return #GNUNET_SYSERR on error, 
+ * @return #GNUNET_SYSERR on error,
  *         #GNUNET_NO if connection reversal is unavailable,
  *         #GNUNET_OK otherwise (presumably in progress)
  */
diff --git a/src/nat/nat_stun.h b/src/nat/nat_stun.h
index 3e4228875..8d1c2720f 100644
--- a/src/nat/nat_stun.h
+++ b/src/nat/nat_stun.h
@@ -70,7 +70,7 @@ struct stun_addr
    * Port number.
    */
   uint16_t port;
-  
+
   /**
    * IPv4 address. Should this be "struct in_addr"?
    */
@@ -79,7 +79,7 @@ struct stun_addr
 
 
 /**
- * STUN message classes 
+ * STUN message classes
  */
 enum StunClasses {
   INVALID_CLASS = 0,
@@ -186,18 +186,19 @@ stun_msg2str (int msg)
   static char result[64];
   const char *msg_class = NULL;
   const char *method = NULL;
-  int value;
+  enum StunClasses cvalue;
+  enum StunMethods mvalue;
 
-  value = decode_class (msg);
+  cvalue = decode_class (msg);
   for (unsigned int i = 0; classes[i].name; i++)
-    if (classes[i].value == value)
+    if (classes[i].value == cvalue)
     {
       msg_class = classes[i].name;
       break;
     }
-  value = decode_method (msg);
+  mvalue = decode_method (msg);
   for (unsigned int i = 0; methods[i].name; i++)
-    if (methods[i].value == value)
+    if (methods[i].value == mvalue)
     {
       method = methods[i].name;
       break;
diff --git a/src/nse/Makefile.am b/src/nse/Makefile.am
index 9a163b160..618489780 100644
--- a/src/nse/Makefile.am
+++ b/src/nse/Makefile.am
@@ -82,7 +82,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/nse/gnunet-nse-profiler.c b/src/nse/gnunet-nse-profiler.c
index 48afd9298..4a10022e3 100644
--- a/src/nse/gnunet-nse-profiler.c
+++ b/src/nse/gnunet-nse-profiler.c
@@ -129,7 +129,7 @@ static int ok;
 /**
  * Be verbose (configuration option)
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Name of the file with the hosts to run the test over (configuration option)
@@ -835,28 +835,47 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "connections", "COUNT",
-     gettext_noop ("limit to the number of connections to NSE services, 0 for none"),
-     1, &GNUNET_GETOPT_set_uint, &connection_limit},
-    {'d', "details", "FILENAME",
-     gettext_noop ("name of the file for writing connection information and statistics"),
-     1, &GNUNET_GETOPT_set_string, &data_filename},
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     1, &GNUNET_GETOPT_set_string, &hosts_file},
-    {'o', "output", "FILENAME",
-     gettext_noop ("name of the file for writing the main results"),
-     1, &GNUNET_GETOPT_set_string, &output_filename},
-    {'p', "peers", "NETWORKSIZESPEC",
-     gettext_noop ("Number of peers to run in each round, separated by commas"),
-     1, &GNUNET_GETOPT_set_string, &num_peer_spec},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_increment_value, &verbose},
-    {'w', "wait", "DELAY",
-     gettext_noop ("delay between rounds"),
-     1, &GNUNET_GETOPT_set_relative_time, &wait_time},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_uint ('C',
+                                   "connections",
+                                   "COUNT",
+                                   gettext_noop ("limit to the number of connections to NSE services, 0 for none"),
+                                   &connection_limit),
+    GNUNET_GETOPT_option_string ('d',
+                                 "details",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file for writing connection information and statistics"),
+                                 &data_filename),
+
+    GNUNET_GETOPT_option_string ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
+
+    GNUNET_GETOPT_option_string ('o',
+                                 "output",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file for writing the main results"),
+                                 &output_filename),
+
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "peers",
+                                 "NETWORKSIZESPEC",
+                                 gettext_noop ("Number of peers to run in each round, separated by commas"),
+                                 &num_peer_spec),
+
+    GNUNET_GETOPT_option_increment_uint ('V',
+                                          "verbose",
+                                          gettext_noop ("be verbose (print progress information)"),
+                                          &verbose),
+
+    GNUNET_GETOPT_option_relative_time ('w',
+                                            "wait",
+                                            "DELAY",
+                                            gettext_noop ("delay between rounds"),
+                                            &wait_time),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/peerinfo-tool/Makefile.am b/src/peerinfo-tool/Makefile.am
index c79c3ac68..648957a05 100644
--- a/src/peerinfo-tool/Makefile.am
+++ b/src/peerinfo-tool/Makefile.am
@@ -30,7 +30,7 @@ check_SCRIPTS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_SCRIPTS)
 endif
 
diff --git a/src/peerinfo-tool/gnunet-peerinfo.c b/src/peerinfo-tool/gnunet-peerinfo.c
index a5907c63f..de7dcd6fa 100644
--- a/src/peerinfo-tool/gnunet-peerinfo.c
+++ b/src/peerinfo-tool/gnunet-peerinfo.c
@@ -837,31 +837,48 @@ state_machine (void *cls)
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "numeric", NULL,
-     gettext_noop ("don't resolve host names"),
-     0, &GNUNET_GETOPT_set_one, &no_resolve},
-    {'q', "quiet", NULL,
-     gettext_noop ("output only the identity strings"),
-     0, &GNUNET_GETOPT_set_one, &be_quiet},
-    {'f', "friends", NULL,
-     gettext_noop ("include friend-only information"),
-     0, &GNUNET_GETOPT_set_one, &include_friend_only},
-    {'s', "self", NULL,
-     gettext_noop ("output our own identity only"),
-     0, &GNUNET_GETOPT_set_one, &get_self},
-    {'i', "info", NULL,
-     gettext_noop ("list all known peers"),
-     0, &GNUNET_GETOPT_set_one, &get_info},
-    {'d', "dump-hello", NULL,
-     gettext_noop ("dump hello to file"),
-     1, &GNUNET_GETOPT_set_string, &dump_hello},
-    {'g', "get-hello", NULL,
-     gettext_noop ("also output HELLO uri(s)"),
-     0, &GNUNET_GETOPT_set_one, &get_uri},
-    {'p', "put-hello", "HELLO",
-     gettext_noop ("add given HELLO uri to the database"),
-     1, &GNUNET_GETOPT_set_string, &put_uri},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('n',
+                                  "numeric",
+                                  gettext_noop ("don't resolve host names"),
+                                  &no_resolve),
+
+    GNUNET_GETOPT_option_flag ('q',
+                                  "quiet",
+                                  gettext_noop ("output only the identity strings"),
+                                  &be_quiet),
+    GNUNET_GETOPT_option_flag ('f',
+                                  "friends",
+                                  gettext_noop ("include friend-only information"),
+                                  &include_friend_only),
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "self",
+                                  gettext_noop ("output our own identity only"),
+                                  &get_self),
+    
+    GNUNET_GETOPT_option_flag ('i',
+                                  "info",
+                                  gettext_noop ("list all known peers"),
+                                  &get_info),
+
+    GNUNET_GETOPT_option_string ('d',
+                                 "dump-hello",
+                                 NULL,
+                                 gettext_noop ("dump hello to file"),
+                                 &dump_hello),
+
+    GNUNET_GETOPT_option_flag ('g',
+                                  "get-hello",
+                                  gettext_noop ("also output HELLO uri(s)"),
+                                  &get_uri),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "put-hello",
+                                 "HELLO",
+                                 gettext_noop ("add given HELLO uri to the database"),
+                                 &put_uri),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/peerinfo/.gitignore b/src/peerinfo/.gitignore
index e2a79a211..152ca2916 100644
--- a/src/peerinfo/.gitignore
+++ b/src/peerinfo/.gitignore
@@ -3,3 +3,4 @@ test_peerinfo_api
 test_peerinfo_api_friend_only
 test_peerinfo_api_notify_friend_only
 test_peerinfo_shipped_hellos
+perf_peerinfo_api
diff --git a/src/peerinfo/Makefile.am b/src/peerinfo/Makefile.am
index dd41cf15a..eeb5ee54e 100644
--- a/src/peerinfo/Makefile.am
+++ b/src/peerinfo/Makefile.am
@@ -58,7 +58,7 @@ endif
 
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c
index bc4a2a10d..731c24bf1 100644
--- a/src/peerinfo/gnunet-service-peerinfo.c
+++ b/src/peerinfo/gnunet-service-peerinfo.c
@@ -283,7 +283,7 @@ read_host_file (const char *fn,
                 int unlink_garbage,
                 struct ReadHostFileContext *r)
 {
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
   ssize_t size_total;
   struct GNUNET_TIME_Absolute now;
   unsigned int left;
@@ -919,7 +919,7 @@ add_to_tc (void *cls,
   {
         /* Copy public HELLO */
     hs = GNUNET_HELLO_size (pos->hello);
-    GNUNET_assert (hs < GNUNET_SERVER_MAX_MESSAGE_SIZE -
+    GNUNET_assert (hs < GNUNET_MAX_MESSAGE_SIZE -
                    sizeof (struct InfoMessage));
     env = GNUNET_MQ_msg_extra (im,
                                hs,
@@ -937,7 +937,7 @@ add_to_tc (void *cls,
   {
         /* Copy friend only HELLO */
     hs = GNUNET_HELLO_size (pos->friend_only_hello);
-    GNUNET_assert (hs < GNUNET_SERVER_MAX_MESSAGE_SIZE -
+    GNUNET_assert (hs < GNUNET_MAX_MESSAGE_SIZE -
                    sizeof (struct InfoMessage));
     env = GNUNET_MQ_msg_extra (im,
                                hs,
@@ -977,7 +977,7 @@ discard_hosts_helper (void *cls,
                       const char *fn)
 {
   struct GNUNET_TIME_Absolute *now = cls;
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
   const struct GNUNET_HELLO_Message *hello;
   struct GNUNET_HELLO_Message *new_hello;
   int read_size;
diff --git a/src/peerinfo/perf_peerinfo_api.c b/src/peerinfo/perf_peerinfo_api.c
index d446cbe3f..840c85c1b 100644
--- a/src/peerinfo/perf_peerinfo_api.c
+++ b/src/peerinfo/perf_peerinfo_api.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2004, 2009, 2010 GNUnet e.V.
+     Copyright (C) 2004, 2009, 2010, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,7 +22,7 @@
  * @file peerinfo/perf_peerinfo_api.c
  * @brief testcase for peerinfo_api.c, hopefully hammer the peerinfo service,
  * this performance test adds up to 5000 peers with one address each and checks
- * over how many peers it can iterate before receiving a timeout after 30 seconds
+ * over how many peers it can iterate before receiving a timeout after 5 seconds
  * @author Nathan Evans
  */
 
@@ -34,8 +34,6 @@
 #include "peerinfo.h"
 #include <gauger.h>
 
-#define START_SERVICE 1
-
 #define NUM_REQUESTS 5000
 
 static struct GNUNET_PEERINFO_IteratorContext *ic[NUM_REQUESTS];
@@ -46,9 +44,36 @@ static unsigned int numpeers;
 
 static struct GNUNET_PeerIdentity pid;
 
+static struct GNUNET_SCHEDULER_Task *tt;
+
+
+static void
+do_shutdown (void *cls)
+{
+  if (NULL != tt)
+  {
+    GNUNET_SCHEDULER_cancel (tt);
+    tt = NULL;
+  }
+  for (unsigned int i = 0; i < NUM_REQUESTS; i++)
+    if (NULL != ic[i])
+      GNUNET_PEERINFO_iterate_cancel (ic[i]);
+  GNUNET_PEERINFO_disconnect (h);
+  h = NULL;
+}
+
+
+static void
+do_timeout (void *cls)
+{
+  tt = NULL;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
 
 static int
-check_it (void *cls, const struct GNUNET_HELLO_Address *address,
+check_it (void *cls,
+          const struct GNUNET_HELLO_Address *address,
           struct GNUNET_TIME_Absolute expiration)
 {
   return GNUNET_OK;
@@ -87,23 +112,34 @@ add_peer (size_t i)
   struct GNUNET_HELLO_Message *h2;
 
   memset (&pid, i, sizeof (pid));
-  h2 = GNUNET_HELLO_create (&pid.public_key, &address_generator, &i, GNUNET_NO);
+  h2 = GNUNET_HELLO_create (&pid.public_key,
+                            &address_generator,
+                            &i,
+                            GNUNET_NO);
   GNUNET_PEERINFO_add_peer (h, h2, NULL, NULL);
   GNUNET_free (h2);
 }
 
 
 static void
-process (void *cls, const struct GNUNET_PeerIdentity *peer,
-         const struct GNUNET_HELLO_Message *hello, const char *err_msg)
+process (void *cls,
+         const struct GNUNET_PeerIdentity *peer,
+         const struct GNUNET_HELLO_Message *hello,
+         const char *err_msg)
 {
-  if (NULL != peer)
-  {
-    numpeers++;
-    if (0 && (hello != NULL))
-      GNUNET_HELLO_iterate_addresses (hello, GNUNET_NO, &check_it, NULL);
+  struct GNUNET_PEERINFO_IteratorContext **icp = cls;
 
+  if (NULL == peer)
+  {
+    *icp = NULL;
+    return;
   }
+  numpeers++;
+  if (0 && (NULL != hello) )
+    GNUNET_HELLO_iterate_addresses (hello,
+                                    GNUNET_NO,
+                                    &check_it,
+                                    NULL);
 }
 
 
@@ -112,32 +148,43 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *cfg,
      struct GNUNET_TESTING_Peer *peer)
 {
-  size_t i;
-
   h = GNUNET_PEERINFO_connect (cfg);
   GNUNET_assert (h != NULL);
-  for (i = 0; i < NUM_REQUESTS; i++)
+  for (unsigned int i = 0; i < NUM_REQUESTS; i++)
   {
     add_peer (i);
-    ic[i] =
-        GNUNET_PEERINFO_iterate (h, GNUNET_YES, NULL,
-                                 GNUNET_TIME_relative_multiply
-                                 (GNUNET_TIME_UNIT_SECONDS, 30), &process, cls);
+    ic[i] = GNUNET_PEERINFO_iterate (h,
+                                     GNUNET_YES,
+                                     NULL,
+                                     &process,
+                                     &ic[i]);
   }
+  tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                                                    5),
+                                     &do_timeout,
+                                     NULL);
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
 }
 
 
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   if (0 != GNUNET_TESTING_service_run ("perf-gnunet-peerinfo",
                                        "peerinfo",
                                        "test_peerinfo_api_data.conf",
                                        &run, NULL))
     return 1;
-  FPRINTF (stderr, "Received %u/%u calls before timeout\n", numpeers,
+  FPRINTF (stderr,
+           "Received %u/%u calls before timeout\n",
+           numpeers,
            NUM_REQUESTS * NUM_REQUESTS / 2);
-  GAUGER ("PEERINFO", "Peerinfo lookups", numpeers / 30, "peers/s");
+  GAUGER ("PEERINFO",
+          "Peerinfo lookups",
+          numpeers / 5,
+          "peers/s");
   return 0;
 }
 
diff --git a/src/peerstore/.gitignore b/src/peerstore/.gitignore
index 33304d90b..7fc22bbdb 100644
--- a/src/peerstore/.gitignore
+++ b/src/peerstore/.gitignore
@@ -6,3 +6,4 @@ test_peerstore_api_store
 test_peerstore_api_sync
 test_peerstore_api_watch
 test_plugin_peerstore_sqlite
+test_plugin_peerstore_flat
diff --git a/src/peerstore/Makefile.am b/src/peerstore/Makefile.am
index a12fdde2a..3aef05769 100644
--- a/src/peerstore/Makefile.am
+++ b/src/peerstore/Makefile.am
@@ -73,7 +73,9 @@ libgnunet_plugin_peerstore_sqlite_la_SOURCES = \
   plugin_peerstore_sqlite.c
 libgnunet_plugin_peerstore_sqlite_la_LIBADD = \
   libgnunetpeerstore.la  \
-  $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
+  $(top_builddir)/src/sq/libgnunetsq.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
 libgnunet_plugin_peerstore_sqlite_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
@@ -108,7 +110,7 @@ EXTRA_DIST = \
  test_peerstore_api_data.conf
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/peerstore/gnunet-service-peerstore.c b/src/peerstore/gnunet-service-peerstore.c
index 665e625fd..92d020799 100644
--- a/src/peerstore/gnunet-service-peerstore.c
+++ b/src/peerstore/gnunet-service-peerstore.c
@@ -260,15 +260,23 @@ record_iterator (void *cls,
     GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (cls_record->client),
                     env);
     if (NULL == emsg)
+    {
       GNUNET_SERVICE_client_continue (cls_record->client);
+    }
     else
+    {
+      GNUNET_break (0);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to iterate: %s\n",
+                  emsg);
       GNUNET_SERVICE_client_drop (cls_record->client);
+    }
     PEERSTORE_destroy_record (cls_record);
     return;
   }
 
   env = PEERSTORE_create_record_mq_envelope (record->sub_system,
-                                             record->peer,
+                                             &record->peer,
                                              record->key,
                                              record->value,
                                              record->value_size,
@@ -301,7 +309,7 @@ watch_notifier_it (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Found a watcher to update.\n");
   env = PEERSTORE_create_record_mq_envelope (record->sub_system,
-                                             record->peer,
+                                             &record->peer,
                                              record->key,
                                              record->value,
                                              record->value_size,
@@ -325,7 +333,7 @@ watch_notifier (struct GNUNET_PEERSTORE_Record *record)
   struct GNUNET_HashCode keyhash;
 
   PEERSTORE_hash_key (record->sub_system,
-                      record->peer,
+                      &record->peer,
                       record->key,
                       &keyhash);
   GNUNET_CONTAINER_multihashmap_get_multiple (watchers,
@@ -434,17 +442,18 @@ handle_iterate (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Iterate request: ss `%s', peer `%s', key `%s'\n",
               record->sub_system,
-              (NULL == record->peer) ? "NULL" : GNUNET_i2s (record->peer),
+              GNUNET_i2s (&record->peer),
               (NULL == record->key) ? "NULL" : record->key);
   record->client = client;
   if (GNUNET_OK !=
       db->iterate_records (db->cls,
                            record->sub_system,
-                           record->peer,
+                           (ntohs (srm->peer_set)) ? &record->peer : NULL,
                            record->key,
                            &record_iterator,
                            record))
   {
+    GNUNET_break (0);
     GNUNET_SERVICE_client_drop (client);
     PEERSTORE_destroy_record (record);
   }
@@ -470,6 +479,7 @@ store_record_continuation (void *cls,
   }
   else
   {
+    GNUNET_break (0);
     GNUNET_SERVICE_client_drop (record->client);
   }
   PEERSTORE_destroy_record (record);
@@ -496,7 +506,6 @@ check_store (void *cls,
     return GNUNET_SYSERR;
   }
   if ( (NULL == record->sub_system) ||
-       (NULL == record->peer) ||
        (NULL == record->key) )
   {
     GNUNET_break (0);
@@ -523,24 +532,25 @@ handle_store (void *cls,
 
   record = PEERSTORE_parse_record_message (srm);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Received a store request. Sub system `%s' Peer `%s Key `%s' Options: %d.\n",
+              "Received a store request. Sub system `%s' Peer `%s Key `%s' Options: %u.\n",
               record->sub_system,
-              GNUNET_i2s (record->peer),
+              GNUNET_i2s (&record->peer),
               record->key,
-              ntohl (srm->options));
+              (uint32_t) ntohl (srm->options));
   record->client = client;
   if (GNUNET_OK !=
       db->store_record (db->cls,
                         record->sub_system,
-                        record->peer,
+                        &record->peer,
                         record->key,
                         record->value,
                         record->value_size,
-                        *record->expiry,
+                        record->expiry,
                         ntohl (srm->options),
                         &store_record_continuation,
                         record))
   {
+    GNUNET_break (0);
     PEERSTORE_destroy_record (record);
     GNUNET_SERVICE_client_drop (client);
     return;
diff --git a/src/peerstore/peerstore.h b/src/peerstore/peerstore.h
index 8b3c4dd92..c6b954676 100644
--- a/src/peerstore/peerstore.h
+++ b/src/peerstore/peerstore.h
@@ -60,7 +60,7 @@ struct StoreRecordMessage
   /**
    * Expiry time of entry
    */
-  struct GNUNET_TIME_Absolute expiry GNUNET_PACKED;
+  struct GNUNET_TIME_AbsoluteNBO expiry;
 
   /**
    * Size of the key string
diff --git a/src/peerstore/peerstore_api.c b/src/peerstore/peerstore_api.c
index c9fcd17ab..df182fe10 100644
--- a/src/peerstore/peerstore_api.c
+++ b/src/peerstore/peerstore_api.c
@@ -286,6 +286,10 @@ store_request_sent (void *cls)
 /*******************         CONNECTION FUNCTIONS         *********************/
 /******************************************************************************/
 
+
+/**
+ * Function called when we had trouble talking to the service.
+ */
 static void
 handle_client_error (void *cls,
                      enum GNUNET_MQ_Error error)
@@ -293,7 +297,7 @@ handle_client_error (void *cls,
   struct GNUNET_PEERSTORE_Handle *h = cls;
 
   LOG (GNUNET_ERROR_TYPE_ERROR,
-       _("Received an error notification from MQ of type: %d\n"),
+       "Received an error notification from MQ of type: %d\n",
        error);
   reconnect (h);
 }
@@ -341,7 +345,9 @@ iterate_timeout (void *cls)
   callback_cls = ic->callback_cls;
   GNUNET_PEERSTORE_iterate_cancel (ic);
   if (NULL != callback)
-    callback (callback_cls, NULL, _("timeout"));
+    callback (callback_cls,
+              NULL,
+              _("timeout"));
 }
 
 
@@ -510,7 +516,7 @@ GNUNET_PEERSTORE_store (struct GNUNET_PEERSTORE_Handle *h,
        "Storing value (size: %lu) for subsytem `%s', peer `%s', key `%s'\n",
        size, sub_system, GNUNET_i2s (peer), key);
   ev = PEERSTORE_create_record_mq_envelope (sub_system, peer, key, value, size,
-                                            &expiry, options,
+                                            expiry, options,
                                             GNUNET_MESSAGE_TYPE_PEERSTORE_STORE);
   sc = GNUNET_new (struct GNUNET_PEERSTORE_StoreContext);
 
@@ -684,8 +690,12 @@ GNUNET_PEERSTORE_iterate (struct GNUNET_PEERSTORE_Handle *h,
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_PEERSTORE_IterateContext *ic;
 
-  ev = PEERSTORE_create_record_mq_envelope (sub_system, peer, key, NULL, 0,
+  ev = PEERSTORE_create_record_mq_envelope (sub_system,
+                                            peer,
+                                            key,
                                             NULL, 0,
+                                            GNUNET_TIME_UNIT_FOREVER_ABS,
+                                            0,
                                             GNUNET_MESSAGE_TYPE_PEERSTORE_ITERATE);
   ic = GNUNET_new (struct GNUNET_PEERSTORE_IterateContext);
 
@@ -756,7 +766,7 @@ handle_watch_record (void *cls,
     return;
   }
   PEERSTORE_hash_key (record->sub_system,
-                      record->peer,
+                      &record->peer,
                       record->key,
                       &keyhash);
   // FIXME: what if there are multiple watches for the same key?
@@ -848,9 +858,12 @@ reconnect (struct GNUNET_PEERSTORE_Handle *h)
                                               &ic->peer,
                                               ic->key,
                                               NULL, 0,
-                                              NULL, 0,
+                                              GNUNET_TIME_UNIT_FOREVER_ABS,
+                                              0,
                                               GNUNET_MESSAGE_TYPE_PEERSTORE_ITERATE);
     GNUNET_MQ_send (h->mq, ev);
+    if (NULL != ic->timeout_task)
+      GNUNET_SCHEDULER_cancel (ic->timeout_task);
     ic->timeout_task
       = GNUNET_SCHEDULER_add_delayed (ic->timeout,
                                       &iterate_timeout,
@@ -863,7 +876,7 @@ reconnect (struct GNUNET_PEERSTORE_Handle *h)
                                               sc->key,
                                               sc->value,
                                               sc->size,
-                                              &sc->expiry,
+                                              sc->expiry,
                                               sc->options,
                                               GNUNET_MESSAGE_TYPE_PEERSTORE_STORE);
     GNUNET_MQ_notify_sent (ev,
diff --git a/src/peerstore/peerstore_common.c b/src/peerstore/peerstore_common.c
index d12c4e21e..e0ab778fa 100644
--- a/src/peerstore/peerstore_common.c
+++ b/src/peerstore/peerstore_common.c
@@ -77,7 +77,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
                                      const char *key,
                                      const void *value,
                                      size_t value_size,
-                                     struct GNUNET_TIME_Absolute *expiry,
+                                     struct GNUNET_TIME_Absolute expiry,
                                      enum GNUNET_PEERSTORE_StoreOption options,
                                      uint16_t msg_type)
 {
@@ -97,8 +97,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
   msg_size = ss_size + key_size + value_size;
   ev = GNUNET_MQ_msg_extra (srm, msg_size, msg_type);
   srm->key_size = htons (key_size);
-  if (NULL != expiry)
-    srm->expiry = *expiry;
+  srm->expiry = GNUNET_TIME_absolute_hton (expiry);
   if (NULL == peer)
     srm->peer_set = htons (GNUNET_NO);
   else
@@ -147,12 +146,9 @@ PEERSTORE_parse_record_message (const struct StoreRecordMessage *srm)
   record = GNUNET_new (struct GNUNET_PEERSTORE_Record);
   if (GNUNET_YES == ntohs (srm->peer_set))
   {
-    record->peer = GNUNET_new (struct GNUNET_PeerIdentity);
-    *record->peer = srm->peer;
+    record->peer = srm->peer;
   }
-  record->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-
-  *(record->expiry) = srm->expiry;
+  record->expiry = GNUNET_TIME_absolute_ntoh (srm->expiry);
   dummy = (char *) &srm[1];
   if (ss_size > 0)
   {
@@ -167,7 +163,9 @@ PEERSTORE_parse_record_message (const struct StoreRecordMessage *srm)
   if (value_size > 0)
   {
     record->value = GNUNET_malloc (value_size);
-    GNUNET_memcpy (record->value, dummy, value_size);
+    GNUNET_memcpy (record->value,
+                   dummy,
+                   value_size);
   }
   record->value_size = value_size;
   return record;
@@ -184,8 +182,6 @@ PEERSTORE_destroy_record (struct GNUNET_PEERSTORE_Record *record)
 {
   if (NULL != record->sub_system)
     GNUNET_free (record->sub_system);
-  if (NULL != record->peer)
-    GNUNET_free (record->peer);
   if (NULL != record->key)
     GNUNET_free (record->key);
   if (NULL != record->value)
@@ -193,7 +189,5 @@ PEERSTORE_destroy_record (struct GNUNET_PEERSTORE_Record *record)
     GNUNET_free (record->value);
     record->value = 0;
   }
-  if (NULL != record->expiry)
-    GNUNET_free (record->expiry);
   GNUNET_free (record);
 }
diff --git a/src/peerstore/peerstore_common.h b/src/peerstore/peerstore_common.h
index 3d938b5da..0fc14d9b4 100644
--- a/src/peerstore/peerstore_common.h
+++ b/src/peerstore/peerstore_common.h
@@ -56,7 +56,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
                                      const char *key,
                                      const void *value,
                                      size_t value_size,
-                                     struct GNUNET_TIME_Absolute *expiry,
+                                     struct GNUNET_TIME_Absolute expiry,
                                      enum GNUNET_PEERSTORE_StoreOption options,
                                      uint16_t msg_type);
 
diff --git a/src/peerstore/plugin_peerstore_flat.c b/src/peerstore/plugin_peerstore_flat.c
index 2b90719d9..c75b2b1e4 100644
--- a/src/peerstore/plugin_peerstore_flat.c
+++ b/src/peerstore/plugin_peerstore_flat.c
@@ -107,7 +107,9 @@ delete_entries (void *cls,
   struct GNUNET_PEERSTORE_Record *entry = value;
   if (0 != strcmp (plugin->iter_key, entry->key))
     return GNUNET_YES;
-  if (0 != memcmp (plugin->iter_peer, entry->peer, sizeof (struct GNUNET_PeerIdentity)))
+  if (0 != memcmp (plugin->iter_peer,
+                   &entry->peer,
+                   sizeof (struct GNUNET_PeerIdentity)))
     return GNUNET_YES;
   if (0 != strcmp (plugin->iter_sub_system, entry->sub_system))
     return GNUNET_YES;
@@ -153,7 +155,7 @@ expire_entries (void *cls,
   struct Plugin *plugin = cls;
   struct GNUNET_PEERSTORE_Record *entry = value;
 
-  if (entry->expiry->abs_value_us < plugin->iter_now.abs_value_us)
+  if (entry->expiry.abs_value_us < plugin->iter_now.abs_value_us)
   {
     GNUNET_CONTAINER_multihashmap_remove (plugin->hm, key, value);
     plugin->exp_changes++;
@@ -204,7 +206,7 @@ iterate_entries (void *cls,
 
   if ((NULL != plugin->iter_peer) &&
       (0 != memcmp (plugin->iter_peer,
-                    entry->peer,
+                    &entry->peer,
                     sizeof (struct GNUNET_PeerIdentity))))
   {
     return GNUNET_YES;
@@ -296,10 +298,8 @@ peerstore_flat_store_record (void *cls, const char *sub_system,
   entry->value = GNUNET_malloc (size);
   GNUNET_memcpy (entry->value, value, size);
   entry->value_size = size;
-  entry->peer = GNUNET_new (struct GNUNET_PeerIdentity);
-  GNUNET_memcpy (entry->peer, peer, sizeof (struct GNUNET_PeerIdentity));
-  entry->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-  entry->expiry->abs_value_us = expiry.abs_value_us;
+  entry->peer = *peer;
+  entry->expiry = expiry;
 
   peer_id = GNUNET_i2s (peer);
   GNUNET_CRYPTO_hash (peer_id,
@@ -409,7 +409,7 @@ database_setup (struct Plugin *plugin)
     GNUNET_free (buffer);
     return GNUNET_SYSERR;
   }
-  
+
   buffer[size] = '\0';
   GNUNET_DISK_file_close (fh);
   if (0 < size) {
@@ -433,22 +433,35 @@ database_setup (struct Plugin *plugin)
       entry = GNUNET_new (struct GNUNET_PEERSTORE_Record);
       entry->sub_system = GNUNET_strdup (sub_system);
       entry->key = GNUNET_strdup (key);
-      GNUNET_STRINGS_base64_decode (peer,
-                                    strlen (peer),
-                                    (char**)&entry->peer);
+      {
+        size_t s;
+        char *o;
+
+        o = NULL;
+        s = GNUNET_STRINGS_base64_decode (peer,
+                                          strlen (peer),
+                                          &o);
+        if (sizeof (struct GNUNET_PeerIdentity) == s)
+          GNUNET_memcpy (&entry->peer,
+                         o,
+                         s);
+        else
+          GNUNET_break (0);
+        GNUNET_free_non_null (o);
+      }
       entry->value_size = GNUNET_STRINGS_base64_decode (value,
                                                         strlen (value),
                                                         (char**)&entry->value);
-      if (GNUNET_SYSERR == GNUNET_STRINGS_fancy_time_to_absolute (expiry,
-                                             entry->expiry))
+      if (GNUNET_SYSERR ==
+          GNUNET_STRINGS_fancy_time_to_absolute (expiry,
+                                                 &entry->expiry))
       {
         GNUNET_free (entry->sub_system);
         GNUNET_free (entry->key);
-        GNUNET_free (entry->peer);
         GNUNET_free (entry);
         break;
       }
-      peer_id = GNUNET_i2s (entry->peer);
+      peer_id = GNUNET_i2s (&entry->peer);
       GNUNET_CRYPTO_hash (peer_id,
                           strlen (peer_id),
                           &hkey);
@@ -479,8 +492,8 @@ store_and_free_entries (void *cls,
   GNUNET_STRINGS_base64_encode (entry->value,
                                 entry->value_size,
                                 &val);
-  expiry = GNUNET_STRINGS_absolute_time_to_string (*entry->expiry);
-  GNUNET_STRINGS_base64_encode ((char*)entry->peer,
+  expiry = GNUNET_STRINGS_absolute_time_to_string (entry->expiry);
+  GNUNET_STRINGS_base64_encode ((char*)&entry->peer,
                                 sizeof (struct GNUNET_PeerIdentity),
                                 &peer);
   GNUNET_asprintf (&line,
@@ -496,10 +509,8 @@ store_and_free_entries (void *cls,
                           line,
                           strlen (line));
   GNUNET_free (entry->sub_system);
-  GNUNET_free (entry->peer);
   GNUNET_free (entry->key);
   GNUNET_free (entry->value);
-  GNUNET_free (entry->expiry);
   GNUNET_free (entry);
   GNUNET_free (line);
   return GNUNET_YES;
diff --git a/src/peerstore/plugin_peerstore_sqlite.c b/src/peerstore/plugin_peerstore_sqlite.c
index 2cd7e22cf..440263d44 100644
--- a/src/peerstore/plugin_peerstore_sqlite.c
+++ b/src/peerstore/plugin_peerstore_sqlite.c
@@ -1,6 +1,6 @@
 /*
  * This file is part of GNUnet
- * Copyright (C) 2013 GNUnet e.V.
+ * Copyright (C) 2013, 2017 GNUnet e.V.
  *
  * GNUnet is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published
@@ -22,11 +22,13 @@
  * @file peerstore/plugin_peerstore_sqlite.c
  * @brief sqlite-based peerstore backend
  * @author Omar Tarabai
+ * @author Christian Grothoff
  */
 
 #include "platform.h"
 #include "gnunet_peerstore_plugin.h"
 #include "gnunet_peerstore_service.h"
+#include "gnunet_sq_lib.h"
 #include "peerstore.h"
 #include <sqlite3.h>
 
@@ -111,6 +113,7 @@ struct Plugin
 
 };
 
+
 /**
  * Delete records with the given key
  *
@@ -118,40 +121,50 @@ struct Plugin
  * @param sub_system name of sub system
  * @param peer Peer identity (can be NULL)
  * @param key entry key string (can be NULL)
- * @return number of deleted records
+ * @return number of deleted records, #GNUNE_SYSERR on error
  */
 static int
-peerstore_sqlite_delete_records (void *cls, const char *sub_system,
+peerstore_sqlite_delete_records (void *cls,
+                                 const char *sub_system,
                                  const struct GNUNET_PeerIdentity *peer,
                                  const char *key)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->delete_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_string (sub_system),
+    GNUNET_SQ_query_param_auto_from_type (peer),
+    GNUNET_SQ_query_param_string (key),
+    GNUNET_SQ_query_param_end
+  };
+  int ret;
 
-  if ((SQLITE_OK !=
-       sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_text (stmt, 3, key, strlen (key) + 1, SQLITE_STATIC)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
+    return GNUNET_SYSERR;
   }
-  else if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
+    ret = GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
+  else
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-    return 0;
+    ret = sqlite3_changes (plugin->dbh);
   }
-  return sqlite3_changes (plugin->dbh);
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
+  return ret;
 }
 
 
@@ -172,28 +185,36 @@ peerstore_sqlite_expire_records (void *cls, struct GNUNET_TIME_Absolute now,
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->expire_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
 
-  if (SQLITE_OK !=
-      sqlite3_bind_int64 (stmt, 1, (sqlite3_uint64) now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
+    return GNUNET_SYSERR;
   }
-  else if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE != sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-  }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
   if (NULL != cont)
-  {
-    cont (cont_cls, sqlite3_changes (plugin->dbh));
-  }
+    cont (cont_cls,
+          sqlite3_changes (plugin->dbh));
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   return GNUNET_OK;
 }
 
@@ -213,7 +234,8 @@ peerstore_sqlite_expire_records (void *cls, struct GNUNET_TIME_Absolute now,
  * called
  */
 static int
-peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
+peerstore_sqlite_iterate_records (void *cls,
+                                  const char *sub_system,
                                   const struct GNUNET_PeerIdentity *peer,
                                   const char *key,
                                   GNUNET_PEERSTORE_Processor iter,
@@ -223,94 +245,115 @@ peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
   sqlite3_stmt *stmt;
   int err = 0;
   int sret;
-  struct GNUNET_PEERSTORE_Record *ret;
+  struct GNUNET_PEERSTORE_Record rec;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Executing iterate request on sqlite db.\n");
-  if (NULL == peer && NULL == key)
-  {
-    stmt = plugin->select_peerstoredata;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC));
-  }
-  else if (NULL == key)
-  {
-    stmt = plugin->select_peerstoredata_by_pid;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                            SQLITE_STATIC));
-  }
-  else if (NULL == peer)
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Executing iterate request on sqlite db.\n");
+  if (NULL == peer)
   {
-    stmt = plugin->select_peerstoredata_by_key;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 2, key, strlen (key) + 1, SQLITE_STATIC));
+    if (NULL == key)
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
+    else
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_string (key),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_key;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
   }
   else
   {
-    stmt = plugin->select_peerstoredata_by_all;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 3, key, strlen (key) + 1, SQLITE_STATIC));
+    if (NULL == key)
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_auto_from_type (peer),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_pid;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
+    else
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_auto_from_type (peer),
+        GNUNET_SQ_query_param_string (key),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_all;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
   }
 
-  if (err)
+  if (GNUNET_OK != err)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
+
+  err = 0;
   while (SQLITE_ROW == (sret = sqlite3_step (stmt)))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Returning a matched record.\n");
-    ret = GNUNET_new (struct GNUNET_PEERSTORE_Record);
-
-    ret->sub_system = (char *) sqlite3_column_text (stmt, 0);
-    ret->peer = (struct GNUNET_PeerIdentity *) sqlite3_column_blob (stmt, 1);
-    ret->key = (char *) sqlite3_column_text (stmt, 2);
-    ret->value = (void *) sqlite3_column_blob (stmt, 3);
-    ret->value_size = sqlite3_column_bytes (stmt, 3);
-    ret->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-
-    ret->expiry->abs_value_us = (uint64_t) sqlite3_column_int64 (stmt, 4);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Returning a matched record.\n");
+    struct GNUNET_SQ_ResultSpec rs[] = {
+      GNUNET_SQ_result_spec_string (&rec.sub_system),
+      GNUNET_SQ_result_spec_auto_from_type (&rec.peer),
+      GNUNET_SQ_result_spec_string (&rec.key),
+      GNUNET_SQ_result_spec_variable_size (&rec.value, &rec.value_size),
+      GNUNET_SQ_result_spec_absolute_time (&rec.expiry),
+      GNUNET_SQ_result_spec_end
+    };
+
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (stmt,
+                                  rs))
+    {
+      GNUNET_break (0);
+      break;
+    }
     if (NULL != iter)
-      iter (iter_cls, ret, NULL);
-    GNUNET_free (ret->expiry);
-    GNUNET_free (ret);
+      iter (iter_cls,
+            &rec,
+            NULL);
+    GNUNET_SQ_cleanup_result (rs);
   }
   if (SQLITE_DONE != sret)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
-    err = 1;
-  }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite_step");
     err = 1;
   }
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   if (NULL != iter)
-  {
-    iter (iter_cls, NULL, err ? "sqlite error" : NULL);
-  }
+    iter (iter_cls,
+          NULL,
+          err ? "sqlite error" : NULL);
   return GNUNET_OK;
 }
 
@@ -333,9 +376,12 @@ peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
  * @return #GNUNET_OK on success, else #GNUNET_SYSERR and cont is not called
  */
 static int
-peerstore_sqlite_store_record (void *cls, const char *sub_system,
+peerstore_sqlite_store_record (void *cls,
+                               const char *sub_system,
                                const struct GNUNET_PeerIdentity *peer,
-                               const char *key, const void *value, size_t size,
+                               const char *key,
+                               const void *value,
+                               size_t size,
                                struct GNUNET_TIME_Absolute expiry,
                                enum GNUNET_PEERSTORE_StoreOption options,
                                GNUNET_PEERSTORE_Continuation cont,
@@ -343,39 +389,39 @@ peerstore_sqlite_store_record (void *cls, const char *sub_system,
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->insert_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_string (sub_system),
+    GNUNET_SQ_query_param_auto_from_type (peer),
+    GNUNET_SQ_query_param_string (key),
+    GNUNET_SQ_query_param_fixed_size (value, size),
+    GNUNET_SQ_query_param_absolute_time (&expiry),
+    GNUNET_SQ_query_param_end
+  };
 
   if (GNUNET_PEERSTORE_STOREOPTION_REPLACE == options)
   {
-    peerstore_sqlite_delete_records (cls, sub_system, peer, key);
+    peerstore_sqlite_delete_records (cls,
+                                     sub_system,
+                                     peer,
+                                     key);
   }
-  if (SQLITE_OK !=
-      sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                         SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_blob (stmt, 2, peer,
-                                      sizeof (struct GNUNET_PeerIdentity),
-                                      SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_text (stmt, 3, key, strlen (key) + 1,
-                                      SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_blob (stmt, 4, value, size, SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_int64 (stmt, 5,
-                                       (sqlite3_uint64) expiry.abs_value_us))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
   else if (SQLITE_DONE != sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-    return GNUNET_SYSERR;
-  }
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   if (NULL != cont)
-  {
-    cont (cont_cls, GNUNET_OK);
-  }
+    cont (cont_cls,
+          GNUNET_OK);
   return GNUNET_OK;
 }
 
@@ -388,15 +434,25 @@ peerstore_sqlite_store_record (void *cls, const char *sub_system,
  * @return 0 on success
  */
 static int
-sql_exec (sqlite3 * dbh, const char *sql)
+sql_exec (sqlite3 *dbh,
+          const char *sql)
 {
   int result;
 
-  result = sqlite3_exec (dbh, sql, NULL, NULL, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Executed `%s' / %d\n", sql, result);
-  if (result != SQLITE_OK)
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Error executing SQL query: %s\n  %s\n"),
-         sqlite3_errmsg (dbh), sql);
+  result = sqlite3_exec (dbh,
+                         sql,
+                         NULL,
+                         NULL,
+                         NULL);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Executed `%s' / %d\n",
+       sql,
+       result);
+  if (SQLITE_OK != result)
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Error executing SQL query: %s\n  %s\n"),
+         sqlite3_errmsg (dbh),
+         sql);
   return result;
 }
 
@@ -410,38 +466,33 @@ sql_exec (sqlite3 * dbh, const char *sql)
  * @return 0 on success
  */
 static int
-sql_prepare (sqlite3 * dbh, const char *sql, sqlite3_stmt ** stmt)
+sql_prepare (sqlite3 *dbh,
+             const char *sql,
+             sqlite3_stmt ** stmt)
 {
   char *tail;
   int result;
 
-  result =
-      sqlite3_prepare_v2 (dbh, sql, strlen (sql), stmt, (const char **) &tail);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Prepared `%s' / %p: %d\n", sql, *stmt, result);
-  if (result != SQLITE_OK)
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Error preparing SQL query: %s\n  %s\n"),
-         sqlite3_errmsg (dbh), sql);
+  result = sqlite3_prepare_v2 (dbh,
+                               sql,
+                               strlen (sql),
+                               stmt,
+                               (const char **) &tail);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Prepared `%s' / %p: %d\n",
+       sql,
+       *stmt,
+       result);
+  if (SQLITE_OK != result)
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Error preparing SQL query: %s\n  %s\n"),
+         sqlite3_errmsg (dbh),
+         sql);
   return result;
 }
 
 
 /**
- * sqlite3 custom function for comparison of uint64_t values
- * since it is not supported by default
- */
-void
-sqlite3_lessthan (sqlite3_context * ctx, int dummy, sqlite3_value ** values)
-{
-  uint64_t v1;
-  uint64_t v2;
-
-  v1 = (uint64_t) sqlite3_value_int64 (values[0]);
-  v2 = (uint64_t) sqlite3_value_int64 (values[1]);
-  sqlite3_result_int (ctx, v1 < v2);
-}
-
-
-/**
  * Initialize the database connections and associated
  * data structures (create tables and indices
  * as needed as well).
@@ -455,10 +506,13 @@ database_setup (struct Plugin *plugin)
   char *filename;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (plugin->cfg, "peerstore-sqlite",
-                                               "FILENAME", &filename))
+      GNUNET_CONFIGURATION_get_value_filename (plugin->cfg,
+                                               "peerstore-sqlite",
+                                               "FILENAME",
+                                               &filename))
   {
-    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "peerstore-sqlite",
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "peerstore-sqlite",
                                "FILENAME");
     return GNUNET_SYSERR;
   }
@@ -474,60 +528,81 @@ database_setup (struct Plugin *plugin)
   /* filename should be UTF-8-encoded. If it isn't, it's a bug */
   plugin->fn = filename;
   /* Open database and precompile statements */
-  if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
+  if (SQLITE_OK != sqlite3_open (plugin->fn,
+                                 &plugin->dbh))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Unable to initialize SQLite: %s.\n"),
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Unable to initialize SQLite: %s.\n"),
          sqlite3_errmsg (plugin->dbh));
     return GNUNET_SYSERR;
   }
-  sql_exec (plugin->dbh, "PRAGMA temp_store=MEMORY");
-  sql_exec (plugin->dbh, "PRAGMA synchronous=OFF");
-  sql_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF");
-  sql_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL");
-  sql_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"");
-  sql_exec (plugin->dbh, "PRAGMA page_size=4096");
-  sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS);
+  sql_exec (plugin->dbh,
+            "PRAGMA temp_store=MEMORY");
+  sql_exec (plugin->dbh,
+            "PRAGMA synchronous=OFF");
+  sql_exec (plugin->dbh,
+            "PRAGMA legacy_file_format=OFF");
+  sql_exec (plugin->dbh,
+            "PRAGMA auto_vacuum=INCREMENTAL");
+  sql_exec (plugin->dbh,
+            "PRAGMA encoding=\"UTF-8\"");
+  sql_exec (plugin->dbh,
+            "PRAGMA page_size=4096");
+  sqlite3_busy_timeout (plugin->dbh,
+                        BUSY_TIMEOUT_MS);
   /* Create tables */
   sql_exec (plugin->dbh,
             "CREATE TABLE IF NOT EXISTS peerstoredata (\n"
-            "  sub_system TEXT NOT NULL,\n" "  peer_id BLOB NOT NULL,\n"
-            "  key TEXT NOT NULL,\n" "  value BLOB NULL,\n"
-            "  expiry sqlite3_uint64 NOT NULL" ");");
-  sqlite3_create_function (plugin->dbh, "UINT64_LT", 2, SQLITE_UTF8, NULL,
-                           &sqlite3_lessthan, NULL, NULL);
+            "  sub_system TEXT NOT NULL,\n"
+            "  peer_id BLOB NOT NULL,\n"
+            "  key TEXT NOT NULL,\n"
+            "  value BLOB NULL,\n"
+            "  expiry INT8 NOT NULL" ");");
   /* Create Indices */
   if (SQLITE_OK !=
       sqlite3_exec (plugin->dbh,
                     "CREATE INDEX IF NOT EXISTS peerstoredata_key_index ON peerstoredata (sub_system, peer_id, key)",
-                    NULL, NULL, NULL))
+                    NULL,
+                    NULL,
+                    NULL))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Unable to create indices: %s.\n"),
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Unable to create indices: %s.\n"),
          sqlite3_errmsg (plugin->dbh));
     return GNUNET_SYSERR;
   }
   /* Prepare statements */
 
   sql_prepare (plugin->dbh,
-               "INSERT INTO peerstoredata (sub_system, peer_id, key, value, expiry) VALUES (?,?,?,?,?);",
+               "INSERT INTO peerstoredata (sub_system, peer_id, key, value, expiry)"
+               " VALUES (?,?,?,?,?);",
                &plugin->insert_peerstoredata);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?",
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?",
                &plugin->select_peerstoredata);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
-               " AND peer_id = ?", &plugin->select_peerstoredata_by_pid);
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
+               " AND peer_id = ?",
+               &plugin->select_peerstoredata_by_pid);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
-               " AND key = ?", &plugin->select_peerstoredata_by_key);
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
+               " AND key = ?",
+               &plugin->select_peerstoredata_by_key);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
                " AND peer_id = ?" " AND key = ?",
                &plugin->select_peerstoredata_by_all);
   sql_prepare (plugin->dbh,
-               "DELETE FROM peerstoredata" " WHERE UINT64_LT(expiry, ?)",
+               "DELETE FROM peerstoredata"
+               " WHERE expiry < ?",
                &plugin->expire_peerstoredata);
   sql_prepare (plugin->dbh,
-               "DELETE FROM peerstoredata" " WHERE sub_system = ?"
+               "DELETE FROM peerstoredata"
+               " WHERE sub_system = ?"
                " AND peer_id = ?" " AND key = ?",
                &plugin->delete_peerstoredata);
   return GNUNET_OK;
@@ -545,15 +620,20 @@ database_shutdown (struct Plugin *plugin)
   int result;
   sqlite3_stmt *stmt;
 
-  while (NULL != (stmt = sqlite3_next_stmt (plugin->dbh, NULL)))
+  while (NULL != (stmt = sqlite3_next_stmt (plugin->dbh,
+                                            NULL)))
   {
     result = sqlite3_finalize (stmt);
     if (SQLITE_OK != result)
-      LOG (GNUNET_ERROR_TYPE_WARNING, "Failed to close statement %p: %d\n",
-           stmt, result);
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           "Failed to close statement %p: %d\n",
+           stmt,
+           result);
   }
   if (SQLITE_OK != sqlite3_close (plugin->dbh))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
   GNUNET_free_non_null (plugin->fn);
 }
 
@@ -573,7 +653,9 @@ libgnunet_plugin_peerstore_sqlite_init (void *cls)
 
   if (NULL != plugin.cfg)
     return NULL;                /* can only initialize once! */
-  memset (&plugin, 0, sizeof (struct Plugin));
+  memset (&plugin,
+          0,
+          sizeof (struct Plugin));
   plugin.cfg = cfg;
   if (GNUNET_OK != database_setup (&plugin))
   {
@@ -585,7 +667,8 @@ libgnunet_plugin_peerstore_sqlite_init (void *cls)
   api->store_record = &peerstore_sqlite_store_record;
   api->iterate_records = &peerstore_sqlite_iterate_records;
   api->expire_records = &peerstore_sqlite_expire_records;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sqlite plugin is running\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sqlite plugin is running\n");
   return api;
 }
 
@@ -605,7 +688,8 @@ libgnunet_plugin_peerstore_sqlite_done (void *cls)
   database_shutdown (plugin);
   plugin->cfg = NULL;
   GNUNET_free (api);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sqlite plugin is finished\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sqlite plugin is finished\n");
   return NULL;
 }
 
diff --git a/src/peerstore/test_peerstore_api_iterate.c b/src/peerstore/test_peerstore_api_iterate.c
index 83a6bf7b7..c607d9fb3 100644
--- a/src/peerstore/test_peerstore_api_iterate.c
+++ b/src/peerstore/test_peerstore_api_iterate.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C)
+     Copyright (C) 2013-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -42,7 +42,8 @@ static int count = 0;
 
 
 static void
-iter3_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter3_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -63,7 +64,8 @@ iter3_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
 
 
 static void
-iter2_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter2_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -78,13 +80,19 @@ iter2_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
   }
   GNUNET_assert (count == 2);
   count = 0;
-  ic = GNUNET_PEERSTORE_iterate (h, ss, NULL, NULL, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter3_cb, NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 NULL,
+                                 NULL,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &iter3_cb,
+                                 NULL);
 }
 
 
 static void
-iter1_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter1_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -99,30 +107,61 @@ iter1_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
   }
   GNUNET_assert (count == 1);
   count = 0;
-  ic = GNUNET_PEERSTORE_iterate (h, ss, &p1, NULL, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter2_cb, NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 &p1,
+                                 NULL,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 iter2_cb,
+                                 NULL);
 }
 
 
 static void
-run (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
      struct GNUNET_TESTING_Peer *peer)
 {
   h = GNUNET_PEERSTORE_connect (cfg);
   GNUNET_assert (NULL != h);
   memset (&p1, 1, sizeof (p1));
   memset (&p2, 2, sizeof (p2));
-  GNUNET_PEERSTORE_store (h, ss, &p1, k1, val, strlen (val) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p1,
+                          k1,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  GNUNET_PEERSTORE_store (h, ss, &p1, k2, val, strlen (val) + 1,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p1,
+                          k2,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  GNUNET_PEERSTORE_store (h, ss, &p2, k3, val, strlen (val) + 1,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p2,
+                          k3,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  ic = GNUNET_PEERSTORE_iterate (h, ss, &p1, k1, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter1_cb, NULL);
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 &p1,
+                                 k1,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &iter1_cb, NULL);
 }
 
 
diff --git a/src/peerstore/test_peerstore_api_store.c b/src/peerstore/test_peerstore_api_store.c
index bfe1b5b55..978009dbb 100644
--- a/src/peerstore/test_peerstore_api_store.c
+++ b/src/peerstore/test_peerstore_api_store.c
@@ -39,6 +39,7 @@ static char *val3 = "test_peerstore_api_store_val3--";
 
 static int count = 0;
 
+
 static void
 test3_cont2 (void *cls,
              const struct GNUNET_PEERSTORE_Record *record,
@@ -49,7 +50,8 @@ test3_cont2 (void *cls,
   if (NULL != record)
   {
     GNUNET_assert ((strlen (val3) + 1) == record->value_size);
-    GNUNET_assert (0 == strcmp ((char *) val3, (char *) record->value));
+    GNUNET_assert (0 == strcmp ((char *) val3,
+                                (char *) record->value));
     count++;
     return;
   }
@@ -61,7 +63,8 @@ test3_cont2 (void *cls,
 
 
 static void
-test3_cont (void *cls, int success)
+test3_cont (void *cls,
+            int success)
 {
   if (GNUNET_YES != success)
     return;
@@ -71,7 +74,8 @@ test3_cont (void *cls, int success)
                             &pid,
                             key,
                             GNUNET_TIME_UNIT_SECONDS,
-                            &test3_cont2, NULL);
+                            &test3_cont2,
+                            NULL);
 }
 
 
@@ -81,9 +85,15 @@ test3_cont (void *cls, int success)
 static void
 test3 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val3, strlen (val3) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val3,
+                          strlen (val3) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, &test3_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          &test3_cont,
                           NULL);
 }
 
@@ -130,9 +140,15 @@ test2_cont (void *cls, int success)
 void
 test2 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val2, strlen (val2) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val2,
+                          strlen (val2) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_MULTIPLE, &test2_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_MULTIPLE,
+                          &test2_cont,
                           NULL);
 }
 
@@ -163,8 +179,13 @@ test1_cont (void *cls, int success)
   if (GNUNET_YES != success)
     return;
   count = 0;
-  GNUNET_PEERSTORE_iterate (h, subsystem, &pid, key, GNUNET_TIME_UNIT_SECONDS,
-                            &test1_cont2, NULL);
+  GNUNET_PEERSTORE_iterate (h,
+                            subsystem,
+                            &pid,
+                            key,
+                            GNUNET_TIME_UNIT_SECONDS,
+                            &test1_cont2,
+                            NULL);
 }
 
 
@@ -174,9 +195,15 @@ test1_cont (void *cls, int success)
 static void
 test1 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val1, strlen (val1) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val1,
+                          strlen (val1) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, &test1_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          &test1_cont,
                           NULL);
 }
 
@@ -196,8 +223,10 @@ int
 main (int argc, char *argv[])
 {
   if (0 !=
-      GNUNET_TESTING_service_run ("test-gnunet-peerstore", "peerstore",
-                                  "test_peerstore_api_data.conf", &run, NULL))
+      GNUNET_TESTING_service_run ("test-gnunet-peerstore",
+                                  "peerstore",
+                                  "test_peerstore_api_data.conf",
+                                  &run, NULL))
     return 1;
   return ok;
 }
diff --git a/src/peerstore/test_plugin_peerstore.c b/src/peerstore/test_plugin_peerstore.c
index 179e32b52..62c06be8e 100644
--- a/src/peerstore/test_plugin_peerstore.c
+++ b/src/peerstore/test_plugin_peerstore.c
@@ -36,6 +36,11 @@ static int ok;
 static const char *plugin_name;
 
 
+static struct GNUNET_PEERSTORE_PluginFunctions *psp;
+
+static struct GNUNET_PeerIdentity p1;
+
+
 /**
  * Function called when the service shuts down.  Unloads our namestore
  * plugin.
@@ -47,8 +52,12 @@ unload_plugin (struct GNUNET_PEERSTORE_PluginFunctions *api)
 {
   char *libname;
 
-  GNUNET_asprintf (&libname, "libgnunet_plugin_peer_%s", plugin_name);
-  GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_peer_%s",
+                   plugin_name);
+  GNUNET_break (NULL ==
+                GNUNET_PLUGIN_unload (libname,
+                                      api));
   GNUNET_free (libname);
 }
 
@@ -65,12 +74,18 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
   struct GNUNET_PEERSTORE_PluginFunctions *ret;
   char *libname;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' peer plugin\n"),
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' peer plugin\n"),
               plugin_name);
-  GNUNET_asprintf (&libname, "libgnunet_plugin_peerstore_%s", plugin_name);
-  if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg)))
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_peerstore_%s",
+                   plugin_name);
+  if (NULL == (ret = GNUNET_PLUGIN_load (libname,
+                                         (void*) cfg)))
   {
-    FPRINTF (stderr, "Failed to load plugin `%s'!\n", plugin_name);
+    FPRINTF (stderr,
+             "Failed to load plugin `%s'!\n",
+             plugin_name);
     GNUNET_free (libname);
     return NULL;
   }
@@ -81,19 +96,28 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
 static void
 test_record (void *cls,
-                                                 const struct GNUNET_PEERSTORE_Record *record,
-                                                 const char *error)
+             const struct GNUNET_PEERSTORE_Record *record,
+             const char *error)
 {
-  struct GNUNET_PeerIdentity *id = cls;
-  char* testval = "test_val";
+  const struct GNUNET_PeerIdentity *id = cls;
+  const char* testval = "test_val";
 
   if (NULL == record)
+  {
+    unload_plugin (psp);
     return;
-
-  GNUNET_assert (0 == memcmp (record->peer, id, sizeof (struct GNUNET_PeerIdentity)));
-  GNUNET_assert (0 == strcmp ("subsys", record->sub_system));
-  GNUNET_assert (0 == strcmp ("key", record->key));
-  GNUNET_assert (0 == memcmp (testval, record->value, strlen (testval)));
+  }
+  GNUNET_assert (0 == memcmp (&record->peer,
+                              id,
+                              sizeof (struct GNUNET_PeerIdentity)));
+  GNUNET_assert (0 == strcmp ("subsys",
+                              record->sub_system));
+  GNUNET_assert (0 == strcmp ("key",
+                              record->key));
+  GNUNET_assert (0 == memcmp (testval,
+                              record->value,
+                              strlen (testval)));
+  ok = 0;
 }
 
 
@@ -101,38 +125,52 @@ static void
 get_record (struct GNUNET_PEERSTORE_PluginFunctions *psp,
             const struct GNUNET_PeerIdentity *identity)
 {
-  GNUNET_assert (GNUNET_OK == psp->iterate_records (psp->cls,
-                                            "subsys", identity, "key", &test_record, (void*)identity));
+  GNUNET_assert (GNUNET_OK ==
+                 psp->iterate_records (psp->cls,
+                                       "subsys",
+                                       identity,
+                                       "key",
+                                       &test_record,
+                                       (void*)identity));
 }
 
+
 static void
-store_cont (void *cls, int status)
+store_cont (void *cls,
+            int status)
 {
   GNUNET_assert (GNUNET_OK == status);
+  get_record (psp,
+              &p1);
 }
 
+
 static void
-put_record (struct GNUNET_PEERSTORE_PluginFunctions *psp, struct GNUNET_PeerIdentity *identity)
+put_record (struct GNUNET_PEERSTORE_PluginFunctions *psp,
+            const struct GNUNET_PeerIdentity *identity)
 {
-  GNUNET_assert (GNUNET_OK == psp->store_record (psp->cls,
-            "subsys",
-                                                identity,
-            "key", "test_value", strlen ("test_value"),
-            GNUNET_TIME_absolute_get (),
-            GNUNET_PEERSTORE_STOREOPTION_REPLACE,
-                                                &store_cont,
-            identity));
+  GNUNET_assert (GNUNET_OK ==
+                 psp->store_record (psp->cls,
+                                    "subsys",
+                                    identity,
+                                    "key",
+                                    "test_value",
+                                    strlen ("test_value"),
+                                    GNUNET_TIME_absolute_get (),
+                                    GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                                    &store_cont,
+                                    NULL));
 }
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
-  struct GNUNET_PEERSTORE_PluginFunctions *psp;
-  struct GNUNET_PeerIdentity p1;
 
-  ok = 0;
+  ok = 1;
   psp = load_plugin (cfg);
   if (NULL == psp)
   {
@@ -142,10 +180,8 @@ run (void *cls, char *const *args, const char *cfgfile,
     return;
   }
   memset (&p1, 1, sizeof (p1));
-  put_record (psp, &p1);
-  get_record (psp, &p1);
-
-  unload_plugin (psp);
+  put_record (psp,
+              &p1);
 }
 
 
@@ -163,19 +199,26 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
-  //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
   GNUNET_log_setup ("test-plugin-peerstore",
                     "WARNING",
                     NULL);
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_plugin_peerstore_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_plugin_peerstore_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-plugin-peerstore", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-plugin-peerstore",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if (ok != 0)
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
-  //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
-/* end of test_plugin_namestore.c */
+/* end of test_plugin_peerstore.c */
diff --git a/src/postgres/postgres.c b/src/postgres/postgres.c
index 4798c129d..14095c5a4 100644
--- a/src/postgres/postgres.c
+++ b/src/postgres/postgres.c
@@ -182,22 +182,22 @@ GNUNET_POSTGRES_connect (const struct GNUNET_CONFIGURATION_Handle * cfg,
                                              &conninfo))
     conninfo = NULL;
   dbh = PQconnectdb (conninfo == NULL ? "" : conninfo);
-  GNUNET_free_non_null (conninfo);
-  if (NULL == dbh)
-  {
-    /* FIXME: warn about out-of-memory? */
-    return NULL;
-  }
-  if (PQstatus (dbh) != CONNECTION_OK)
+
+  if (NULL != dbh)
   {
-    GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
-                     "postgres",
-                     _("Unable to connect to Postgres database '%s': %s\n"),
-                     conninfo,
-                     PQerrorMessage (dbh));
-    PQfinish (dbh);
-    return NULL;
+    if (PQstatus (dbh) != CONNECTION_OK)
+    {
+      GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR,
+                       "postgres",
+                       _("Unable to connect to Postgres database '%s': %s\n"),
+                       conninfo,
+                       PQerrorMessage (dbh));
+      PQfinish (dbh);
+      dbh = NULL;
+    }
   }
+  // FIXME: warn about out-of-memory when dbh is NULL?
+  GNUNET_free_non_null (conninfo);
   return dbh;
 }
 
diff --git a/src/pq/pq.c b/src/pq/pq.c
index f4bab4e00..4e24e1ef8 100644
--- a/src/pq/pq.c
+++ b/src/pq/pq.c
@@ -85,9 +85,10 @@ GNUNET_PQ_exec_prepared (PGconn *db_conn,
       off += x->num_params;
     }
     GNUNET_assert (off == len);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Executing prepared SQL statement `%s'\n",
-                name);
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "pq",
+                     "Executing prepared SQL statement `%s'\n",
+                     name);
     res = PQexecPrepared (db_conn,
                           name,
                           len,
diff --git a/src/pq/pq_result_helper.c b/src/pq/pq_result_helper.c
index 180c5fc18..c5c8e56b9 100644
--- a/src/pq/pq_result_helper.c
+++ b/src/pq/pq_result_helper.c
@@ -77,9 +77,7 @@ extract_varsize_blob (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -156,9 +154,7 @@ extract_fixed_blob (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -173,11 +169,7 @@ extract_fixed_blob (void *cls,
                      fnum);
   if (*dst_size != len)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' has wrong size (got %u, expected %u)\n",
-                fname,
-                (unsigned int) len,
-                (unsigned int) *dst_size);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   res = PQgetvalue (result,
@@ -243,9 +235,7 @@ extract_rsa_public_key (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -265,9 +255,7 @@ extract_rsa_public_key (void *cls,
                                              len);
   if (NULL == *pk)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' contains bogus value (fails to decode)\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -346,9 +334,7 @@ extract_rsa_signature (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -368,9 +354,7 @@ extract_rsa_signature (void *cls,
                                              len);
   if (NULL == *sig)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' contains bogus value (fails to decode)\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -449,9 +433,7 @@ extract_string (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -471,9 +453,7 @@ extract_string (void *cls,
                          len);
   if (NULL == *str)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' contains bogus value (fails to decode)\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -583,9 +563,7 @@ extract_uint16 (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -655,9 +633,7 @@ extract_uint32 (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
@@ -727,9 +703,7 @@ extract_uint64 (void *cls,
                     fname);
   if (fnum < 0)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Field `%s' does not exist in result\n",
-                fname);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
   if (PQgetisnull (result,
diff --git a/src/pq/test_pq.c b/src/pq/test_pq.c
index 49705103f..b54318110 100644
--- a/src/pq/test_pq.c
+++ b/src/pq/test_pq.c
@@ -218,8 +218,8 @@ run_queries (PGconn *conn)
 
 
 int
-main(int argc,
-     const char *const argv[])
+main (int argc,
+      const char *const argv[])
 {
   PGconn *conn;
   PGresult *result;
@@ -236,7 +236,7 @@ main(int argc,
              PQerrorMessage (conn));
     GNUNET_break (0);
     PQfinish (conn);
-    return 0; /* We ignore this type of error... */
+    return 77; /* signal test was skipped */
   }
 
   result = PQexec (conn,
diff --git a/src/psyc/.gitignore b/src/psyc/.gitignore
index e12b3210c..14a175367 100644
--- a/src/psyc/.gitignore
+++ b/src/psyc/.gitignore
@@ -1 +1,2 @@
 gnunet-service-psyc
+test_psyc
diff --git a/src/psyc/Makefile.am b/src/psyc/Makefile.am
index 0016b1b3b..a16128624 100644
--- a/src/psyc/Makefile.am
+++ b/src/psyc/Makefile.am
@@ -53,7 +53,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/psyc/gnunet-service-psyc.c b/src/psyc/gnunet-service-psyc.c
index eab74b467..73a3ae4ee 100644
--- a/src/psyc/gnunet-service-psyc.c
+++ b/src/psyc/gnunet-service-psyc.c
@@ -467,6 +467,9 @@ schedule_transmit_message (void *cls)
 static void
 shutdown_task (void *cls)
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "shutting down...\n");
+  GNUNET_PSYCSTORE_disconnect (store);
   if (NULL != stats)
   {
     GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
diff --git a/src/psyc/psyc_api.c b/src/psyc/psyc_api.c
index c6544df3a..c93d8b383 100644
--- a/src/psyc/psyc_api.c
+++ b/src/psyc/psyc_api.c
@@ -1309,7 +1309,7 @@ channel_history_replay (struct GNUNET_PSYC_Channel *chn,
 
   GNUNET_assert (NULL != method_prefix);
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -1454,7 +1454,7 @@ channel_state_get (struct GNUNET_PSYC_Channel *chn,
   sr->op_id = GNUNET_OP_add (chn->op, op_recv_state_result, sr, NULL);
 
   GNUNET_assert (NULL != name);
-  size_t name_size = strnlen (name, GNUNET_SERVER_MAX_MESSAGE_SIZE
+  size_t name_size = strnlen (name, GNUNET_MAX_MESSAGE_SIZE
                               - sizeof (*req)) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (req, name_size, type);
diff --git a/src/psycstore/.gitignore b/src/psycstore/.gitignore
index fc2e4cf8e..5ec783202 100644
--- a/src/psycstore/.gitignore
+++ b/src/psycstore/.gitignore
@@ -1 +1,5 @@
 gnunet-service-psycstore
+test_plugin_psycstore_mysql
+test_plugin_psycstore_sqlite
+test_plugin_psycstore_postgres
+test_psycstore
diff --git a/src/psycstore/Makefile.am b/src/psycstore/Makefile.am
index 30e2a9673..a342c06e6 100644
--- a/src/psycstore/Makefile.am
+++ b/src/psycstore/Makefile.am
@@ -94,7 +94,10 @@ libgnunet_plugin_psycstore_postgres_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lpq \
   $(LTLIBINTL)
 libgnunet_plugin_psycstore_postgres_la_LDFLAGS = \
- $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS)
+  $(GN_PLUGIN_LDFLAGS) $(POSTGRESQL_LDFLAGS)
+libgnunet_plugin_psycstore_postgres_la_CPPFLAGS = \
+  $(POSTGRESQL_CPPFLAGS) $(AM_CPPFLAGS)
+
 
 libgnunet_plugin_psycstore_sqlite_la_SOURCES = \
   plugin_psycstore_sqlite.c
@@ -118,7 +121,7 @@ endif
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/psycstore/gnunet-service-psycstore.c b/src/psycstore/gnunet-service-psycstore.c
index 10c92a878..1bee8da65 100644
--- a/src/psycstore/gnunet-service-psycstore.c
+++ b/src/psycstore/gnunet-service-psycstore.c
@@ -106,7 +106,7 @@ send_result_code (struct GNUNET_SERVICE_Client *client,
 
   if (NULL != err_msg)
     err_size = strnlen (err_msg,
-                        GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*res) - 1) + 1;
+                        GNUNET_MAX_MESSAGE_SIZE - sizeof (*res) - 1) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (res, err_size,
                                GNUNET_MESSAGE_TYPE_PSYCSTORE_RESULT_CODE);
diff --git a/src/psycstore/psycstore_api.c b/src/psycstore/psycstore_api.c
index 40322662e..d79daa357 100644
--- a/src/psycstore/psycstore_api.c
+++ b/src/psycstore/psycstore_api.c
@@ -809,7 +809,7 @@ GNUNET_PSYCSTORE_message_get (struct GNUNET_PSYCSTORE_Handle *h,
   if (NULL == method_prefix)
     method_prefix = "";
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
 
   struct GNUNET_MQ_Envelope *
@@ -875,7 +875,7 @@ GNUNET_PSYCSTORE_message_get_latest (struct GNUNET_PSYCSTORE_Handle *h,
   if (NULL == method_prefix)
     method_prefix = "";
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
diff --git a/src/psycutil/.gitignore b/src/psycutil/.gitignore
new file mode 100644
index 000000000..03d8197fb
--- /dev/null
+++ b/src/psycutil/.gitignore
@@ -0,0 +1 @@
+test_psyc_env
diff --git a/src/psycutil/Makefile.am b/src/psycutil/Makefile.am
index 2a916fe30..2732c3a21 100644
--- a/src/psycutil/Makefile.am
+++ b/src/psycutil/Makefile.am
@@ -33,7 +33,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/pt/.gitignore b/src/pt/.gitignore
index 22f803ed6..ea678ffe5 100644
--- a/src/pt/.gitignore
+++ b/src/pt/.gitignore
@@ -1 +1,6 @@
 gnunet-daemon-pt
+test_gns_vpn
+test_gnunet_vpn-4_over
+test_gnunet_vpn-4_to_6
+test_gnunet_vpn-6_over
+test_gnunet_vpn-6_to_4
diff --git a/src/pt/Makefile.am b/src/pt/Makefile.am
index e56990f08..e36630ae4 100644
--- a/src/pt/Makefile.am
+++ b/src/pt/Makefile.am
@@ -82,7 +82,7 @@ endif
 check_PROGRAMS = $(VPN_TEST)
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/pt/gnunet-daemon-pt.c b/src/pt/gnunet-daemon-pt.c
index 06ef88832..295082c0e 100644
--- a/src/pt/gnunet-daemon-pt.c
+++ b/src/pt/gnunet-daemon-pt.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2010, 2012 Christian Grothoff
+     Copyright (C) 2010, 2012, 2017 Christian Grothoff
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,7 +17,6 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
 /**
  * @file pt/gnunet-daemon-pt.c
  * @brief tool to manipulate DNS and VPN services to perform protocol translation (IPvX over GNUnet)
@@ -161,21 +160,6 @@ struct CadetExit
   struct RequestContext *receive_queue_tail;
 
   /**
-   * Head of DLL of requests to be transmitted to a cadet_channel.
-   */
-  struct RequestContext *transmit_queue_head;
-
-  /**
-   * Tail of DLL of requests to be transmitted to a cadet_channel.
-   */
-  struct RequestContext *transmit_queue_tail;
-
-  /**
-   * Active transmission request for this channel (or NULL).
-   */
-  struct GNUNET_CADET_TransmitHandle *cadet_th;
-
-  /**
    * Identity of the peer that is providing the exit for us.
    */
   struct GNUNET_PeerIdentity peer;
@@ -190,6 +174,11 @@ struct CadetExit
    */
   unsigned int num_answered;
 
+  /**
+   * Size of the window, 0 if we are busy.
+   */
+  /* unsigned */ int idle;
+
 };
 
 
@@ -220,10 +209,9 @@ struct RequestContext
   struct GNUNET_DNS_RequestHandle *rh;
 
   /**
-   * Message we're sending out via CADET, allocated at the
-   * end of this struct.
+   * Envelope with the request we are transmitting.
    */
-  const struct GNUNET_MessageHeader *cadet_message;
+  struct GNUNET_MQ_Envelope *env;
 
   /**
    * Task used to abort this operation with timeout.
@@ -240,12 +228,6 @@ struct RequestContext
    */
   uint16_t dns_id;
 
-  /**
-   * #GNUNET_NO if this request is still in the transmit_queue,
-   * #GNUNET_YES if we are in the receive_queue.
-   */
-  int16_t was_transmitted;
-
 };
 
 
@@ -328,59 +310,7 @@ static unsigned int dns_exit_available;
  * We are short on cadet exits, try to open another one.
  */
 static void
-try_open_exit ()
-{
-  struct CadetExit *pos;
-  uint32_t candidate_count;
-  uint32_t candidate_selected;
-  struct GNUNET_HashCode port;
-
-  GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_INTERNET_RESOLVER,
-                      strlen (GNUNET_APPLICATION_PORT_INTERNET_RESOLVER),
-                      &port);
-  candidate_count = 0;
-  for (pos = exit_head; NULL != pos; pos = pos->next)
-    if (NULL == pos->cadet_channel)
-      candidate_count++;
-  if (0 == candidate_count)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "No DNS exits available yet.\n");
-    return;
-  }
-  candidate_selected = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                                 candidate_count);
-  candidate_count = 0;
-  for (pos = exit_head; NULL != pos; pos = pos->next)
-    if (NULL == pos->cadet_channel)
-    {
-      candidate_count++;
-      if (candidate_selected < candidate_count)
-      {
-        /* move to the head of the DLL */
-        pos->cadet_channel
-          = GNUNET_CADET_channel_create (cadet_handle,
-                                         pos,
-                                         &pos->peer,
-                                         &port,
-                                         GNUNET_CADET_OPTION_DEFAULT);
-        if (NULL == pos->cadet_channel)
-        {
-          GNUNET_break (0);
-          continue;
-        }
-        GNUNET_CONTAINER_DLL_remove (exit_head,
-                                     exit_tail,
-                                     pos);
-        GNUNET_CONTAINER_DLL_insert (exit_head,
-                                     exit_tail,
-                                     pos);
-        dns_exit_available++;
-        return;
-      }
-    }
-  GNUNET_assert (NULL == exit_head);
-}
+try_open_exit (void);
 
 
 /**
@@ -443,7 +373,7 @@ choose_exit ()
     channel_weight = get_channel_weight (pos);
     total_transmitted += channel_weight;
     /* double weight for idle channels */
-    if (NULL == pos->cadet_th)
+    if (0 != pos->idle)
       total_transmitted += channel_weight;
   }
   if (0 == total_transmitted)
@@ -461,7 +391,7 @@ choose_exit ()
     channel_weight = get_channel_weight (pos);
     total_transmitted += channel_weight;
     /* double weight for idle channels */
-    if (NULL == pos->cadet_th)
+    if (0 != pos->idle)
       total_transmitted += channel_weight;
     if (total_transmitted > selected_offset)
       return pos;
@@ -768,62 +698,6 @@ dns_post_request_handler (void *cls,
 
 
 /**
- * Transmit a DNS request via CADET and move the request
- * handle to the receive queue.
- *
- * @param cls the `struct CadetExit`
- * @param size number of bytes available in buf
- * @param buf where to copy the message
- * @return number of bytes written to buf
- */
-static size_t
-transmit_dns_request_to_cadet (void *cls,
-                              size_t size,
-                              void *buf)
-{
-  struct CadetExit *exit = cls;
-  struct RequestContext *rc;
-  size_t mlen;
-
-  exit->cadet_th = NULL;
-  if (NULL == (rc = exit->transmit_queue_head))
-    return 0;
-  mlen = rc->mlen;
-  if (mlen > size)
-  {
-    exit->cadet_th
-      = GNUNET_CADET_notify_transmit_ready (exit->cadet_channel,
-                                            GNUNET_NO,
-                                            TIMEOUT,
-                                            mlen,
-                                            &transmit_dns_request_to_cadet,
-                                            exit);
-    return 0;
-  }
-  GNUNET_assert (GNUNET_NO == rc->was_transmitted);
-  GNUNET_memcpy (buf,
-                 rc->cadet_message,
-                 mlen);
-  GNUNET_CONTAINER_DLL_remove (exit->transmit_queue_head,
-                               exit->transmit_queue_tail,
-                               rc);
-  rc->was_transmitted = GNUNET_YES;
-  GNUNET_CONTAINER_DLL_insert (exit->receive_queue_head,
-                               exit->receive_queue_tail,
-                               rc);
-  rc = exit->transmit_queue_head;
-  if (NULL != rc)
-    exit->cadet_th = GNUNET_CADET_notify_transmit_ready (exit->cadet_channel,
-                                                       GNUNET_NO,
-                                                       TIMEOUT,
-                                                       rc->mlen,
-                                                       &transmit_dns_request_to_cadet,
-                                                       exit);
-  return mlen;
-}
-
-
-/**
  * Task run if the time to answer a DNS request via CADET is over.
  *
  * @param cls the `struct RequestContext` to abort
@@ -834,19 +708,6 @@ timeout_request (void *cls)
   struct RequestContext *rc = cls;
   struct CadetExit *exit = rc->exit;
 
-  if (rc->was_transmitted)
-  {
-    exit->num_transmitted++;
-    GNUNET_CONTAINER_DLL_remove (exit->receive_queue_head,
-                                 exit->receive_queue_tail,
-                                 rc);
-  }
-  else
-  {
-    GNUNET_CONTAINER_DLL_remove (exit->transmit_queue_head,
-                                 exit->transmit_queue_tail,
-                                 rc);
-  }
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# DNS requests dropped (timeout)"),
                             1,
@@ -854,12 +715,10 @@ timeout_request (void *cls)
   GNUNET_DNS_request_drop (rc->rh);
   GNUNET_free (rc);
   if ( (0 == get_channel_weight (exit)) &&
-       (NULL == exit->receive_queue_head) &&
-       (NULL == exit->transmit_queue_head) )
+       (NULL == exit->receive_queue_head) )
   {
     /* this straw broke the camel's back: this channel now has
        such a low score that it will not be used; close it! */
-    GNUNET_assert (NULL == exit->cadet_th);
     GNUNET_CADET_channel_destroy (exit->cadet_channel);
     exit->cadet_channel = NULL;
     GNUNET_CONTAINER_DLL_remove (exit_head,
@@ -870,7 +729,7 @@ timeout_request (void *cls)
                                       exit);
     /* go back to semi-innocent: mark as not great, but
        avoid a prohibitively negative score (see
-       #get_channel_weight, which checks for a certain
+       #get_channel_weight(), which checks for a certain
        minimum number of transmissions before making
        up an opinion) */
     exit->num_transmitted = 5;
@@ -900,8 +759,8 @@ dns_pre_request_handler (void *cls,
                          const char *request)
 {
   struct RequestContext *rc;
-  size_t mlen;
-  struct GNUNET_MessageHeader hdr;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *hdr;
   struct GNUNET_TUN_DnsHeader dns;
   struct CadetExit *exit;
 
@@ -924,93 +783,115 @@ dns_pre_request_handler (void *cls,
     GNUNET_DNS_request_drop (rh);
     return;
   }
-  GNUNET_memcpy (&dns, request, sizeof (dns));
-  mlen = sizeof (struct GNUNET_MessageHeader) + request_length;
   exit = choose_exit ();
   GNUNET_assert (NULL != exit);
   GNUNET_assert (NULL != exit->cadet_channel);
-  rc = GNUNET_malloc (sizeof (struct RequestContext) + mlen);
+
+  env = GNUNET_MQ_msg_extra (hdr,
+                             request_length,
+                             GNUNET_MESSAGE_TYPE_VPN_DNS_TO_INTERNET);
+  GNUNET_memcpy (&hdr[1],
+                 request,
+                 request_length);
+  rc = GNUNET_new (struct RequestContext);
   rc->exit = exit;
   rc->rh = rh;
-  rc->cadet_message = (const struct GNUNET_MessageHeader*) &rc[1];
   rc->timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
                                                    &timeout_request,
                                                    rc);
+  GNUNET_memcpy (&dns,
+                 request,
+                 sizeof (dns));
   rc->dns_id = dns.id;
-  rc->mlen = mlen;
-  hdr.type = htons (GNUNET_MESSAGE_TYPE_VPN_DNS_TO_INTERNET);
-  hdr.size = htons (mlen);
-  GNUNET_memcpy (&rc[1], &hdr, sizeof (struct GNUNET_MessageHeader));
-  GNUNET_memcpy (&(((char*)&rc[1])[sizeof (struct GNUNET_MessageHeader)]),
-          request,
-          request_length);
-  GNUNET_CONTAINER_DLL_insert_tail (exit->transmit_queue_head,
-                                    exit->transmit_queue_tail,
-                                    rc);
-  if (NULL == exit->cadet_th)
-    exit->cadet_th = GNUNET_CADET_notify_transmit_ready (exit->cadet_channel,
-                                                       GNUNET_NO,
-                                                       TIMEOUT,
-                                                       mlen,
-                                                       &transmit_dns_request_to_cadet,
-                                                       exit);
+  rc->env = env;
+  GNUNET_CONTAINER_DLL_remove (exit->receive_queue_head,
+                               exit->receive_queue_tail,
+                               rc);
+  if (0 < exit->idle)
+    exit->idle--;
+  exit->num_transmitted++;
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (exit->cadet_channel),
+                  GNUNET_MQ_env_copy (env));
 }
 
 
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
+ * Message with a DNS response.
+ */
+struct DnsResponseMessage
+{
+  /**
+   * GNUnet header, of type #GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * DNS header.
+   */
+  struct GNUNET_TUN_DnsHeader dns;
+
+  /* Followed by more DNS payload */
+};
+
+GNUNET_NETWORK_STRUCT_END
+
 /**
  * Process a request via cadet to perform a DNS query.
  *
- * @param cls NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct CadetExit`
- * @param message the actual message
+ * @param cls the `struct CadetExit` which got the message
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-receive_dns_response (void *cls,
-                      struct GNUNET_CADET_Channel *channel,
-                      void **channel_ctx,
-                      const struct GNUNET_MessageHeader *message)
+check_dns_response (void *cls,
+                     const struct DnsResponseMessage *msg)
 {
-  struct CadetExit *exit = *channel_ctx;
-  struct GNUNET_TUN_DnsHeader dns;
+  return GNUNET_OK; /* all OK */
+}
+
+
+/**
+ * Process a request via cadet to perform a DNS query.
+ *
+ * @param cls the `struct CadetExit` which got the message
+ * @param msg the actual message
+ */
+static void
+handle_dns_response (void *cls,
+                     const struct DnsResponseMessage *msg)
+{
+  struct CadetExit *exit = cls;
   size_t mlen;
   struct RequestContext *rc;
 
-  mlen = ntohs (message->size);
-  mlen -= sizeof (struct GNUNET_MessageHeader);
-  if (mlen < sizeof (struct GNUNET_TUN_DnsHeader))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  GNUNET_memcpy (&dns, &message[1], sizeof (dns));
+  mlen = ntohs (msg->header.size) - sizeof (*msg);
   for (rc = exit->receive_queue_head; NULL != rc; rc = rc->next)
   {
-    GNUNET_assert (GNUNET_YES == rc->was_transmitted);
-    if (dns.id == rc->dns_id)
+    if (msg->dns.id == rc->dns_id)
     {
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# DNS replies received"),
-                                1, GNUNET_NO);
+                                1,
+                                GNUNET_NO);
       GNUNET_DNS_request_answer (rc->rh,
-                                 mlen,
-                                 (const void*) &message[1]);
+                                 mlen + sizeof (struct GNUNET_TUN_DnsHeader),
+                                 (const void*) &msg->dns);
       GNUNET_CONTAINER_DLL_remove (exit->receive_queue_head,
                                    exit->receive_queue_tail,
                                    rc);
       GNUNET_SCHEDULER_cancel (rc->timeout_task);
+      GNUNET_MQ_discard (rc->env);
       GNUNET_free (rc);
       exit->num_answered++;
-      exit->num_transmitted++;
-      return GNUNET_OK;
+      return;
     }
   }
   GNUNET_STATISTICS_update (stats,
                             gettext_noop ("# DNS replies dropped (too late?)"),
                             1, GNUNET_NO);
-  return GNUNET_OK;
 }
 
 
@@ -1031,15 +912,7 @@ abort_all_requests (struct CadetExit *exit)
                                  rc);
     GNUNET_DNS_request_drop (rc->rh);
     GNUNET_SCHEDULER_cancel (rc->timeout_task);
-    GNUNET_free (rc);
-  }
-  while (NULL != (rc = exit->transmit_queue_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (exit->transmit_queue_head,
-                                 exit->transmit_queue_tail,
-                                 rc);
-    GNUNET_DNS_request_drop (rc->rh);
-    GNUNET_SCHEDULER_cancel (rc->timeout_task);
+    GNUNET_MQ_discard (rc->env);
     GNUNET_free (rc);
   }
 }
@@ -1067,11 +940,6 @@ cleanup (void *cls)
     GNUNET_CONTAINER_DLL_remove (exit_head,
                                  exit_tail,
                                  exit);
-    if (NULL != exit->cadet_th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (exit->cadet_th);
-      exit->cadet_th = NULL;
-    }
     if (NULL != exit->cadet_channel)
     {
       GNUNET_CADET_channel_destroy (exit->cadet_channel);
@@ -1126,63 +994,120 @@ cleanup (void *cls)
  */
 static void
 cadet_channel_end_cb (void *cls,
-                    const struct GNUNET_CADET_Channel *channel,
-                    void *channel_ctx)
+                      const struct GNUNET_CADET_Channel *channel)
 {
-  struct CadetExit *exit = channel_ctx;
+  struct CadetExit *exit = cls;
   struct CadetExit *alt;
   struct RequestContext *rc;
 
-  if (NULL != exit->cadet_th)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (exit->cadet_th);
-    exit->cadet_th = NULL;
-  }
   exit->cadet_channel = NULL;
   dns_exit_available--;
   /* open alternative channels */
-  try_open_exit ();
-  if (NULL == exit->cadet_channel)
+  /* our channel is now closed, move our requests to an alternative
+     channel */
+  alt = choose_exit ();
+  while (NULL != (rc = exit->receive_queue_head))
   {
-    /* our channel is now closed, move our requests to an alternative
-       channel */
-    alt = choose_exit ();
-    while (NULL != (rc = exit->transmit_queue_head))
-    {
-      GNUNET_CONTAINER_DLL_remove (exit->transmit_queue_head,
-                                   exit->transmit_queue_tail,
-                                   rc);
-      rc->exit = alt;
-      GNUNET_CONTAINER_DLL_insert (alt->transmit_queue_head,
-                                   alt->transmit_queue_tail,
-                                   rc);
-    }
-    while (NULL != (rc = exit->receive_queue_head))
-    {
-      GNUNET_CONTAINER_DLL_remove (exit->receive_queue_head,
-                                   exit->receive_queue_tail,
-                                   rc);
-      rc->was_transmitted = GNUNET_NO;
-      rc->exit = alt;
-      GNUNET_CONTAINER_DLL_insert (alt->transmit_queue_head,
-                                   alt->transmit_queue_tail,
-                                   rc);
-    }
+    GNUNET_CONTAINER_DLL_remove (exit->receive_queue_head,
+                                 exit->receive_queue_tail,
+                                 rc);
+    rc->exit = alt;
+    GNUNET_CONTAINER_DLL_insert (alt->receive_queue_head,
+                                 alt->receive_queue_tail,
+                                 rc);
+    GNUNET_MQ_send (GNUNET_CADET_get_mq (alt->cadet_channel),
+                    GNUNET_MQ_env_copy (rc->env));
   }
-  else
+  try_open_exit ();
+}
+
+
+/**
+ * Function called whenever a channel has excess capacity.
+ *
+ * @param cls the `struct CadetExit`
+ * @param channel connection to the other end
+ * @param window_size how much capacity do we have
+ */
+static void
+channel_idle_notify_cb (void *cls,
+                        const struct GNUNET_CADET_Channel *channel,
+                        int window_size)
+{
+  struct CadetExit *pos = cls;
+
+  pos->idle = window_size;
+}
+
+
+/**
+ * We are short on cadet exits, try to open another one.
+ */
+static void
+try_open_exit ()
+{
+  struct CadetExit *pos;
+  uint32_t candidate_count;
+  uint32_t candidate_selected;
+  struct GNUNET_HashCode port;
+
+  GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_INTERNET_RESOLVER,
+                      strlen (GNUNET_APPLICATION_PORT_INTERNET_RESOLVER),
+                      &port);
+  candidate_count = 0;
+  for (pos = exit_head; NULL != pos; pos = pos->next)
+    if (NULL == pos->cadet_channel)
+      candidate_count++;
+  if (0 == candidate_count)
   {
-    /* the same peer was chosen, just make sure the queue processing is restarted */
-    alt = exit;
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "No DNS exits available yet.\n");
+    return;
   }
-  if ( (NULL == alt->cadet_th) &&
-       (NULL != (rc = alt->transmit_queue_head)) )
-    alt->cadet_th
-      = GNUNET_CADET_notify_transmit_ready (alt->cadet_channel,
-                                            GNUNET_NO,
-                                            TIMEOUT,
-                                            rc->mlen,
-                                            &transmit_dns_request_to_cadet,
-                                            alt);
+  candidate_selected = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                                 candidate_count);
+  candidate_count = 0;
+  for (pos = exit_head; NULL != pos; pos = pos->next)
+    if (NULL == pos->cadet_channel)
+    {
+      candidate_count++;
+      if (candidate_selected < candidate_count)
+      {
+        struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+          GNUNET_MQ_hd_var_size (dns_response,
+                                 GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET,
+                                 struct DnsResponseMessage,
+                                 pos),
+          GNUNET_MQ_handler_end ()
+        };
+
+
+        /* move to the head of the DLL */
+        pos->cadet_channel
+          = GNUNET_CADET_channel_create (cadet_handle,
+                                         pos,
+                                         &pos->peer,
+                                         &port,
+                                         GNUNET_CADET_OPTION_DEFAULT,
+                                         &channel_idle_notify_cb,
+                                         &cadet_channel_end_cb,
+                                         cadet_handlers);
+        if (NULL == pos->cadet_channel)
+        {
+          GNUNET_break (0);
+          continue;
+        }
+        GNUNET_CONTAINER_DLL_remove (exit_head,
+                                     exit_tail,
+                                     pos);
+        GNUNET_CONTAINER_DLL_insert (exit_head,
+                                     exit_tail,
+                                     pos);
+        dns_exit_available++;
+        return;
+      }
+    }
+  GNUNET_assert (NULL == exit_head);
 }
 
 
@@ -1308,11 +1233,6 @@ run (void *cls, char *const *args GNUNET_UNUSED,
   }
   if (dns_channel)
   {
-    static struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-      {&receive_dns_response, GNUNET_MESSAGE_TYPE_VPN_DNS_FROM_INTERNET, 0},
-      {NULL, 0, 0}
-    };
-
     dns_pre_handle
       = GNUNET_DNS_connect (cfg,
                             GNUNET_DNS_FLAG_PRE_RESOLUTION,
@@ -1326,10 +1246,7 @@ run (void *cls, char *const *args GNUNET_UNUSED,
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
-    cadet_handle = GNUNET_CADET_connect (cfg,
-                                         NULL,
-                                         &cadet_channel_end_cb,
-                                         cadet_handlers);
+    cadet_handle = GNUNET_CADET_connect (cfg);
     if (NULL == cadet_handle)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
diff --git a/src/pt/test_gns_vpn.c b/src/pt/test_gns_vpn.c
index 4b7e817e8..53f27610b 100644
--- a/src/pt/test_gns_vpn.c
+++ b/src/pt/test_gns_vpn.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2007, 2009, 2011, 2012, 2015 Christian Grothoff
+     Copyright (C) 2007, 2009, 2011, 2012, 2015, 2017 Christian Grothoff
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,6 +22,19 @@
  * @file test_gns_vpn.c
  * @brief testcase for accessing VPN services via GNS
  * @author Martin Schanzenbach
+ * @author Christian Grothoff
+ *
+ * This test requires libcurl/libgnurl *with* support for C-ARES.
+ * This is NOT the default on most platforms, which means the test
+ * will be skipped in many cases.   Compile libcurl/libgnurl with
+ * "--enable-ares" to get this test to pass.
+ *
+ * Furthermore, the test relies on gnunet-dns2gns being able to bind
+ * to port 53.  This means that 'setcap' has to have worked during
+ * 'make install'.  If this failed, but everything else is OK, the
+ * test may FAIL hard even though it is just an installation issue (we
+ * cannot conveniently test for the setcap to have worked).  However,
+ * you should get a warning that gnunet-dns2gns failed to 'bind'.
  */
 #include "platform.h"
 #if HAVE_CURL_CURL_H
@@ -39,7 +52,7 @@
 #define PORT 8080
 #define TEST_DOMAIN "www.gnu"
 
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
 
 /**
  * Return value for #main().
@@ -96,7 +109,10 @@ static struct CBC cbc;
 
 
 static size_t
-copy_buffer (void *ptr, size_t size, size_t nmemb, void *ctx)
+copy_buffer (void *ptr,
+             size_t size,
+             size_t nmemb,
+             void *ctx)
 {
   struct CBC *cbc = ctx;
 
@@ -174,6 +190,11 @@ do_shutdown (void *cls)
     GNUNET_NAMESTORE_cancel (qe);
     qe = NULL;
   }
+  if (NULL != namestore)
+  {
+    GNUNET_NAMESTORE_disconnect (namestore);
+    namestore = NULL;
+  }
   GNUNET_free_non_null (url);
   url = NULL;
 }
@@ -280,6 +301,9 @@ curl_main ()
 static void
 start_curl (void *cls)
 {
+  CURLcode ec;
+
+  curl_task_id = NULL;
   GNUNET_asprintf (&url,
                    "http://%s/hello_world",
                    TEST_DOMAIN);
@@ -291,7 +315,18 @@ start_curl (void *cls)
   curl_easy_setopt (curl, CURLOPT_TIMEOUT, 150L);
   curl_easy_setopt (curl, CURLOPT_CONNECTTIMEOUT, 150L);
   curl_easy_setopt (curl, CURLOPT_NOSIGNAL, 1);
-
+  if (CURLE_OK !=
+      (ec = curl_easy_setopt (curl,
+                              CURLOPT_DNS_SERVERS,
+                              "127.0.0.1:53")))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "curl build without support for CURLOPT_DNS_SERVERS (%s), cannot run test\n",
+                curl_easy_strerror (ec));
+    global_ret = 77;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
   multi = curl_multi_init ();
   GNUNET_assert (multi != NULL);
   GNUNET_assert (CURLM_OK == curl_multi_add_handle (multi, curl));
@@ -302,14 +337,6 @@ start_curl (void *cls)
 }
 
 
-static void
-disco_ns (void* cls)
-{
-  GNUNET_NAMESTORE_disconnect (namestore);
-  namestore = NULL;
-}
-
-
 /**
  * Callback invoked from the namestore service once record is
  * created.
@@ -328,9 +355,8 @@ commence_testing (void *cls,
                   const char *emsg)
 {
   qe = NULL;
-  GNUNET_SCHEDULER_add_now (&disco_ns, NULL);
-
-  if ((emsg != NULL) && (GNUNET_YES != success))
+  if ( (NULL != emsg) &&
+       (GNUNET_YES != success) )
   {
     fprintf (stderr,
              "NS failed to create record %s\n",
@@ -338,11 +364,14 @@ commence_testing (void *cls,
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
+
   /* wait a little bit before downloading, as we just created the record */
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                (GNUNET_TIME_UNIT_SECONDS, 1),
-                                &start_curl,
-                                NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching cURL request\n");
+  curl_task_id
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
+                                    &start_curl,
+                                    NULL);
 }
 
 
@@ -402,7 +431,6 @@ mhd_main ()
 
 
 
-
 /**
  * Open '/dev/null' and make the result the given
  * file descriptor.
@@ -448,9 +476,8 @@ fork_and_exec (const char *file,
   pid = fork ();
   if (-1 == pid)
   {
-    fprintf (stderr,
-             "fork failed: %s\n",
-             strerror (errno));
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                         "fork");
     return 1;
   }
   if (0 == pid)
@@ -464,10 +491,9 @@ fork_and_exec (const char *file,
     open_dev_null (1, O_WRONLY);
     (void) execv (file, cmd);
     /* can only get here on error */
-    fprintf (stderr,
-             "exec `%s' failed: %s\n",
-             file,
-             strerror (errno));
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                              "exec",
+                              file);
     _exit (1);
   }
   /* keep running waitpid as long as the only error we get is 'EINTR' */
@@ -475,13 +501,20 @@ fork_and_exec (const char *file,
           (errno == EINTR) );
   if (-1 == ret)
   {
-    fprintf (stderr,
-             "waitpid failed: %s\n",
-             strerror (errno));
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                         "waitpid");
     return 1;
   }
-  if (! (WIFEXITED (status) && (0 == WEXITSTATUS (status))))
+  if (! (WIFEXITED (status) &&
+         (0 == WEXITSTATUS (status))) )
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Process `%s` returned status code %d/%d.\n",
+                file,
+                WIFEXITED (status),
+                WEXITSTATUS (status));
     return 1;
+  }
   /* child process completed and returned success, we're happy */
   return 0;
 }
@@ -572,6 +605,8 @@ identity_cb (void *cls,
                                                    &rd.data_size));
   rd.record_type = GNUNET_GNSRECORD_TYPE_VPN;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating `www` record\n");
   qe = GNUNET_NAMESTORE_records_store (namestore,
                                        zone_key,
                                        "www",
@@ -593,15 +628,18 @@ run (void *cls,
   char *bin;
   char *bin_identity;
   char *bin_gns;
+  char *bin_arm;
   char *config;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Test logic starting...\n");
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
                                              "arm",
                                              "CONFIG",
                                              &config))
   {
-    fprintf (stderr,
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
              "Failed to locate configuration file. Skipping test.\n");
     GNUNET_SCHEDULER_shutdown ();
     return;
@@ -626,18 +664,27 @@ run (void *cls,
   {
     "gnunet-identity",
     "-e", "master-zone",
-    "-s", "gns-intercept",
+    "-s", "dns2gns",
+    "-c", config,
+    NULL
+  };
+  char *const arm_args[] =
+  {
+    "gnunet-arm",
+    "-i", "dns2gns",
     "-c", config,
     NULL
   };
   char *const gns_args[] =
   {
     "gnunet-gns",
-    "-u", "www.gns",
+    "-u", "www.gnu",
     "-c", config,
     NULL
   };
-  GNUNET_TESTING_peer_get_identity (peer, &id);
+
+  GNUNET_TESTING_peer_get_identity (peer,
+                                    &id);
   GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                  NULL);
   timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
@@ -648,20 +695,24 @@ run (void *cls,
                    "%s/%s",
                    bin,
                    "gnunet-identity");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating `master-zone` ego\n");
   if (0 != fork_and_exec (bin_identity, identity_args))
   {
-    fprintf (stderr,
-             "Failed to run `gnunet-identity -C. Skipping test.\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to run `gnunet-identity -C`. Skipping test.\n");
     GNUNET_SCHEDULER_shutdown ();
     GNUNET_free (bin_identity);
     GNUNET_free (config);
     GNUNET_free (bin);
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Setting `master-zone` ego as default for `gns-master` and `dns2gns`\n");
   if (0 != fork_and_exec (bin_identity, identity2_args))
   {
-    fprintf (stderr,
-             "Failed to run `gnunet-identity -e. Skipping test.\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to run `gnunet-identity -e`. Skipping test.\n");
     GNUNET_SCHEDULER_shutdown ();
     GNUNET_free (bin_identity);
     GNUNET_free (config);
@@ -670,8 +721,8 @@ run (void *cls,
   }
   if (0 != fork_and_exec (bin_identity, identity3_args))
   {
-    fprintf (stderr,
-             "Failed to run `gnunet-identity -e. Skipping test.\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to run `gnunet-identity -e`. Skipping test.\n");
     GNUNET_SCHEDULER_shutdown ();
     GNUNET_free (bin_identity);
     GNUNET_free (config);
@@ -681,14 +732,17 @@ run (void *cls,
   GNUNET_free (bin_identity);
 
   /* do lookup just to launch GNS service */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Resolving `www.gnu` zone entry to launch GNS (will yield no answer yet)\n");
   GNUNET_asprintf (&bin_gns,
                    "%s/%s",
                    bin,
                    "gnunet-gns");
-  if (0 != fork_and_exec (bin_gns, gns_args))
+  if (0 != fork_and_exec (bin_gns,
+                          gns_args))
   {
-    fprintf (stderr,
-             "Failed to run `gnunet-gns -u. Skipping test.\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to run `gnunet-gns -u. Skipping test.\n");
     GNUNET_SCHEDULER_shutdown ();
     GNUNET_free (bin_gns);
     GNUNET_free (config);
@@ -696,9 +750,27 @@ run (void *cls,
     return;
   }
   GNUNET_free (bin_gns);
+
+  GNUNET_asprintf (&bin_arm,
+                   "%s/%s",
+                   bin,
+                   "gnunet-arm");
+  if (0 != fork_and_exec (bin_arm,
+                          arm_args))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to run `gnunet-arm -i dns2gns. Skipping test.\n");
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_free (bin_arm);
+    GNUNET_free (config);
+    GNUNET_free (bin);
+    return;
+  }
+  GNUNET_free (bin_arm);
+
   GNUNET_free (config);
   GNUNET_free (bin);
-
+  sleep (1); /* give dns2gns chance to really run */
 
   namestore = GNUNET_NAMESTORE_connect (cfg);
   GNUNET_assert (NULL != namestore);
@@ -720,39 +792,15 @@ run (void *cls,
 
 
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
-  char *sbin_iptables;
   char *bin_vpn;
   char *bin_exit;
-  char *bin_dns;
-  char *srv_dns;
-  struct stat s;
-  gid_t my_gid;
-  char *const iptables_args[] =
-  {
-    "iptables", "-t", "mangle", "-L", "-v", NULL
-  };
-
-  if (0 == access ("/sbin/iptables", X_OK))
-    sbin_iptables = "/sbin/iptables";
-  else if (0 == access ("/usr/sbin/iptables", X_OK))
-    sbin_iptables = "/usr/sbin/iptables";
-  else
-  {
-    fprintf (stderr,
-             "Executable iptables not found in approved directories: %s, skipping\n",
-             strerror (errno));
-    return 77;
-  }
-
-  if (0 != fork_and_exec (sbin_iptables, iptables_args))
-  {
-    fprintf (stderr,
-             "Failed to run `iptables -t mangle -L -v'. Skipping test.\n");
-    return 77;
-  }
 
+  GNUNET_log_setup ("test-gns-vpn",
+                    "WARNING",
+                    NULL);
   if (0 != ACCESS ("/dev/net/tun", R_OK))
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
@@ -765,59 +813,26 @@ main (int argc, char *const *argv)
 
   bin_vpn = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-vpn");
   bin_exit = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-exit");
-  bin_dns = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-dns");
-  srv_dns = GNUNET_OS_get_libexec_binary_path ("gnunet-service-dns");
   if ( (0 != geteuid ()) &&
        ( (GNUNET_YES !=
-          GNUNET_OS_check_helper_binary (bin_vpn, GNUNET_YES, "-d gnunet-vpn - - 169.1.3.3.7 255.255.255.0")) || //ipv4 only please!
+          GNUNET_OS_check_helper_binary (bin_vpn,
+                                         GNUNET_YES,
+                                         "-d gnunet-vpn - - 169.1.3.3.7 255.255.255.0")) || //ipv4 only please!
          (GNUNET_YES !=
-          GNUNET_OS_check_helper_binary (bin_exit, GNUNET_YES, "-d gnunet-vpn - - - 169.1.3.3.7 255.255.255.0")) || //no nat, ipv4 only
-         (GNUNET_YES !=
-          GNUNET_OS_check_helper_binary (bin_dns, GNUNET_YES, NULL))) ) // TODO: once we have a windows-testcase, add test parameters here
+          GNUNET_OS_check_helper_binary (bin_exit,
+                                         GNUNET_YES,
+                                         "-d gnunet-vpn - - - 169.1.3.3.7 255.255.255.0")) ) ) //no nat, ipv4 only
   {
     fprintf (stderr,
-             "WARNING: gnunet-helper-{exit,vpn,dns} binaries in $PATH are not SUID, refusing to run test (as it would have to fail).\n");
+             "WARNING: gnunet-helper-{exit,vpn} binaries in $PATH are not SUID, refusing to run test (as it would have to fail).\n");
     fprintf (stderr,
              "Change $PATH ('.' in $PATH before $GNUNET_PREFIX/bin is problematic) or permissions (run 'make install' as root) to fix this!\n");
     GNUNET_free (bin_vpn);
     GNUNET_free (bin_exit);
-    GNUNET_free (bin_dns);
-    GNUNET_free (srv_dns);
     return 77;
   }
   GNUNET_free (bin_vpn);
   GNUNET_free (bin_exit);
-  my_gid = getgid ();
-  if ( (0 != stat (bin_dns, &s)) ||
-       (my_gid == s.st_gid) ||
-       ( (0 == (S_ISUID & s.st_mode)) && (0 != getuid()) ) )
-  {
-    fprintf (stderr,
-             "WARNING: %s has wrong permissions (%d, %d, %d), refusing to run test (as it would have to fail).\n",
-             bin_dns,
-             (0 != stat (bin_dns, &s)),
-             (my_gid == s.st_gid),
-             (0 == (S_ISUID & s.st_mode)) || (0 != getuid()) );
-    GNUNET_free (bin_dns);
-    GNUNET_free (srv_dns);
-    return 77;
-  }
-  if ( (0 != stat (srv_dns, &s)) ||
-       (my_gid == s.st_gid) ||
-       (0 == (S_ISGID & s.st_mode)) )
-  {
-    fprintf (stderr,
-             "WARNING: %s has wrong permissions (%d, %d, %d), refusing to run test (as it would have to fail).\n",
-             srv_dns,
-             (0 != stat (bin_dns, &s)),
-             (my_gid == s.st_gid),
-             (0 == (S_ISGID & s.st_mode)) );
-    GNUNET_free (bin_dns);
-    GNUNET_free (srv_dns);
-    return 77;
-  }
-  GNUNET_free (bin_dns);
-  GNUNET_free (srv_dns);
 
   dest_ip = "169.254.86.1";
   dest_af = AF_INET;
@@ -842,9 +857,11 @@ main (int argc, char *const *argv)
   }
 
 
-  if (0 != GNUNET_TESTING_peer_run ("test-gnunet-vpn",
-                                    "test_gns_vpn.conf",
-                                    &run, NULL))
+  if (0 !=
+      GNUNET_TESTING_peer_run ("test_gns_vpn",
+                               "test_gns_vpn.conf",
+                               &run,
+                               NULL))
     return 1;
   GNUNET_DISK_directory_remove ("/tmp/gnunet-test-vpn");
   return global_ret;
diff --git a/src/pt/test_gns_vpn.conf b/src/pt/test_gns_vpn.conf
index ac9724c04..86642465f 100644
--- a/src/pt/test_gns_vpn.conf
+++ b/src/pt/test_gns_vpn.conf
@@ -35,6 +35,10 @@ FORCESTART = YES
 AUTOSTART = NO
 FORCESTART = NO
 
+[zonemaster]
+AUTOSTART = YES
+FORCESTART = YES
+
 #[vpn]
 #PREFIX = valgrind
 
diff --git a/src/regex/Makefile.am b/src/regex/Makefile.am
index 7f0fb707f..80997db40 100644
--- a/src/regex/Makefile.am
+++ b/src/regex/Makefile.am
@@ -80,6 +80,7 @@ libgnunet_plugin_block_regex_la_SOURCES = \
 libgnunet_plugin_block_regex_la_LIBADD = \
  libgnunetregexblock.la \
  $(top_builddir)/src/block/libgnunetblock.la \
+ $(top_builddir)/src/block/libgnunetblockgroup.la \
  $(top_builddir)/src/util/libgnunetutil.la
 libgnunet_plugin_block_regex_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
@@ -152,7 +153,7 @@ check_PROGRAMS = \
   test_regex_api
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/regex/gnunet-regex-profiler.c b/src/regex/gnunet-regex-profiler.c
index dfbcd388a..7d907fec6 100644
--- a/src/regex/gnunet-regex-profiler.c
+++ b/src/regex/gnunet-regex-profiler.c
@@ -1477,22 +1477,39 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'o', "output-file", "FILENAME",
-     gettext_noop ("name of the file for writing statistics"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &data_filename},
-    {'t', "matching-timeout", "TIMEOUT",
-      gettext_noop ("wait TIMEOUT before ending the experiment"),
-      GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &search_timeout_time},
-    {'p', "policy-dir", "DIRECTORY",
-      gettext_noop ("directory with policy files"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &policy_dir},
-    {'s', "strings-file", "FILENAME",
-      gettext_noop ("name of file with input strings"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &strings_file},
-    {'H', "hosts-file", "FILENAME",
-      gettext_noop ("name of file with hosts' names"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &hosts_file},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_filename ('o',
+                                   "output-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of the file for writing statistics"),
+                                   &data_filename),
+
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "matching-timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("wait TIMEOUT before ending the experiment"),
+                                            &search_timeout_time), 
+
+    GNUNET_GETOPT_option_filename ('p',
+                                   "policy-dir",
+                                   "DIRECTORY",
+                                   gettext_noop ("directory with policy files"),
+                                   &policy_dir),
+
+
+    GNUNET_GETOPT_option_filename ('s',
+                                   "strings-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of file with input strings"),
+                                   &strings_file),
+
+    GNUNET_GETOPT_option_filename ('H',
+                                   "hosts-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of file with hosts' names"),
+                                   &hosts_file),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/regex/gnunet-regex-simulation-profiler.c b/src/regex/gnunet-regex-simulation-profiler.c
index 025a70316..67c2ecfff 100644
--- a/src/regex/gnunet-regex-simulation-profiler.c
+++ b/src/regex/gnunet-regex-simulation-profiler.c
@@ -691,13 +691,20 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'t', "table", "TABLENAME",
-     gettext_noop ("name of the table to write DFAs"),
-     1, &GNUNET_GETOPT_set_string, &table_name},
-    {'p', "max-path-compression", "MAX_PATH_COMPRESSION",
-     gettext_noop ("maximum path compression length"),
-     1, &GNUNET_GETOPT_set_uint, &max_path_compression},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "table",
+                                 "TABLENAME",
+                                 gettext_noop ("name of the table to write DFAs"),
+                                 &table_name),
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "max-path-compression",
+                                   "MAX_PATH_COMPRESSION",
+                                   gettext_noop ("maximum path compression length"),
+                                   &max_path_compression),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/regex/gnunet-service-regex.c b/src/regex/gnunet-service-regex.c
index 294670be6..9a40a5264 100644
--- a/src/regex/gnunet-service-regex.c
+++ b/src/regex/gnunet-service-regex.c
@@ -37,19 +37,14 @@ struct ClientEntry
 {
 
   /**
-   * Kept in DLL.
+   * Queue for transmissions to @e client.
    */
-  struct ClientEntry *next;
-
-  /**
-   * Kept in DLL.
-   */
-  struct ClientEntry *prev;
+  struct GNUNET_MQ_Handle *mq;
 
   /**
    * Handle identifying the client.
    */
-  struct GNUNET_SERVER_Client *client;
+  struct GNUNET_SERVICE_Client *client;
 
   /**
    * Search handle (if this client is searching).
@@ -69,7 +64,7 @@ struct ClientEntry
   /**
    * Task for re-announcing.
    */
-  struct GNUNET_SCHEDULER_Task * refresh_task;
+  struct GNUNET_SCHEDULER_Task *refresh_task;
 
 };
 
@@ -85,70 +80,12 @@ static struct GNUNET_DHT_Handle *dht;
 static struct GNUNET_STATISTICS_Handle *stats;
 
 /**
- * Head of list of clients.
- */
-static struct ClientEntry *client_head;
-
-/**
- * End of list of clients.
- */
-static struct ClientEntry *client_tail;
-
-/**
- * Our notification context, used to send back results to the client.
- */
-static struct GNUNET_SERVER_NotificationContext *nc;
-
-/**
  * Private key for this peer.
  */
 static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 
 
 /**
- * A client disconnected.  Remove all of its data structure entries.
- *
- * @param cls closure, NULL
- * @param client identification of the client
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct ClientEntry *ce;
-  struct ClientEntry *nx;
-
-  nx = client_head;
-  for (ce = nx; NULL != ce; ce = nx)
-  {
-    nx = ce->next;
-    if (ce->client == client)
-    {
-      if (NULL != ce->refresh_task)
-      {
-        GNUNET_SCHEDULER_cancel (ce->refresh_task);
-        ce->refresh_task = NULL;
-      }
-      if (NULL != ce->ah)
-      {
-        REGEX_INTERNAL_announce_cancel (ce->ah);
-        ce->ah = NULL;
-      }
-      if (NULL != ce->sh)
-      {
-        REGEX_INTERNAL_search_cancel (ce->sh);
-        ce->sh = NULL;
-      }
-      GNUNET_CONTAINER_DLL_remove (client_head,
-                                   client_tail,
-                                   ce);
-      GNUNET_free (ce);
-    }
-  }
-}
-
-
-/**
  * Task run during shutdown.
  *
  * @param cls unused
@@ -156,17 +93,11 @@ handle_client_disconnect (void *cls,
 static void
 cleanup_task (void *cls)
 {
-  struct ClientEntry *ce;
-
-  while (NULL != (ce = client_head))
-    handle_client_disconnect (NULL,
-                              ce->client);
   GNUNET_DHT_disconnect (dht);
   dht = NULL;
-  GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
+  GNUNET_STATISTICS_destroy (stats,
+                             GNUNET_NO);
   stats = NULL;
-  GNUNET_SERVER_notification_context_destroy (nc);
-  nc = NULL;
   GNUNET_free (my_private_key);
   my_private_key = NULL;
 }
@@ -191,35 +122,51 @@ reannounce (void *cls)
 
 
 /**
- * Handle ANNOUNCE message.
+ * Check ANNOUNCE message.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param am the actual message
+ * @return #GNUNET_OK if @am is well-formed
  */
-static void
-handle_announce (void *cls,
-                 struct GNUNET_SERVER_Client *client,
-                 const struct GNUNET_MessageHeader *message)
+static int
+check_announce (void *cls,
+                const struct AnnounceMessage *am)
 {
-  const struct AnnounceMessage *am;
+  struct ClientEntry *ce = cls;
   const char *regex;
-  struct ClientEntry *ce;
   uint16_t size;
 
-  size = ntohs (message->size);
-  am = (const struct AnnounceMessage *) message;
+  size = ntohs (am->header.size) - sizeof (*am);
   regex = (const char *) &am[1];
-  if ( (size <= sizeof (struct AnnounceMessage)) ||
-       ('\0' != regex[size - sizeof (struct AnnounceMessage) - 1]) )
+  if ('\0' != regex[size - 1])
   {
     GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    return GNUNET_SYSERR;
+  }
+  if (NULL != ce->ah)
+  {
+    /* only one announcement per client allowed */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  ce = GNUNET_new (struct ClientEntry);
-  ce->client = client;
+
+/**
+ * Handle ANNOUNCE message.
+ *
+ * @param cls identification of the client
+ * @param am the actual message
+ */
+static void
+handle_announce (void *cls,
+                 const struct AnnounceMessage *am)
+{
+  struct ClientEntry *ce = cls;
+  const char *regex;
+
+  regex = (const char *) &am[1];
   ce->frequency = GNUNET_TIME_relative_ntoh (am->refresh_delay);
   ce->refresh_task = GNUNET_SCHEDULER_add_delayed (ce->frequency,
                                                    &reannounce,
@@ -238,14 +185,11 @@ handle_announce (void *cls,
   {
     GNUNET_break (0);
     GNUNET_SCHEDULER_cancel (ce->refresh_task);
-    GNUNET_free (ce);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    ce->refresh_task = NULL;
+    GNUNET_SERVICE_client_drop (ce->client);
     return;
   }
-  GNUNET_CONTAINER_DLL_insert (client_head,
-                               client_tail,
-                               ce);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (ce->client);
 }
 
 
@@ -268,6 +212,7 @@ handle_search_result (void *cls,
                       unsigned int put_path_length)
 {
   struct ClientEntry *ce = cls;
+  struct GNUNET_MQ_Envelope *env;
   struct ResultMessage *result;
   struct GNUNET_PeerIdentity *gp;
   uint16_t size;
@@ -275,64 +220,78 @@ handle_search_result (void *cls,
   if ( (get_path_length >= 65536) ||
        (put_path_length >= 65536) ||
        ( (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity))
-       + sizeof (struct ResultMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+       + sizeof (struct ResultMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
   }
-  size = (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity) + sizeof (struct ResultMessage);
-  result = GNUNET_malloc (size);
-  result->header.size = htons (size);
-  result->header.type = htons (GNUNET_MESSAGE_TYPE_REGEX_RESULT);
+  size = (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity);
+  env = GNUNET_MQ_msg_extra (result,
+                             size,
+                             GNUNET_MESSAGE_TYPE_REGEX_RESULT);
   result->get_path_length = htons ((uint16_t) get_path_length);
   result->put_path_length = htons ((uint16_t) put_path_length);
   result->id = *id;
   gp = &result->id;
   GNUNET_memcpy (&gp[1],
-          get_path,
-          get_path_length * sizeof (struct GNUNET_PeerIdentity));
+                 get_path,
+                 get_path_length * sizeof (struct GNUNET_PeerIdentity));
   GNUNET_memcpy (&gp[1 + get_path_length],
-          put_path,
-          put_path_length * sizeof (struct GNUNET_PeerIdentity));
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              ce->client,
-                                              &result->header, GNUNET_NO);
-  GNUNET_free (result);
+                 put_path,
+                 put_path_length * sizeof (struct GNUNET_PeerIdentity));
+  GNUNET_MQ_send (ce->mq,
+                  env);
 }
 
 
 /**
- * Handle SEARCH message.
+ * Check SEARCH message.
  *
- * @param cls closure
- * @param client identification of the client
+ * @param cls identification of the client
  * @param message the actual message
  */
-static void
-handle_search (void *cls,
-               struct GNUNET_SERVER_Client *client,
-               const struct GNUNET_MessageHeader *message)
+static int
+check_search (void *cls,
+              const struct RegexSearchMessage *sm)
 {
-  const struct RegexSearchMessage *sm;
+  struct ClientEntry *ce = cls;
   const char *string;
-  struct ClientEntry *ce;
   uint16_t size;
 
-  size = ntohs (message->size);
-  sm = (const struct RegexSearchMessage *) message;
+  size = ntohs (sm->header.size) - sizeof (*sm);
   string = (const char *) &sm[1];
-  if ( (size <= sizeof (struct RegexSearchMessage)) ||
-       ('\0' != string[size - sizeof (struct RegexSearchMessage) - 1]) )
+  if ('\0' != string[size - 1])
   {
     GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    return GNUNET_SYSERR;
   }
+  if (NULL != ce->sh)
+  {
+    /* only one search allowed per client */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle SEARCH message.
+ *
+ * @param cls identification of the client
+ * @param message the actual message
+ */
+static void
+handle_search (void *cls,
+               const struct RegexSearchMessage *sm)
+{
+  struct ClientEntry *ce = cls;
+  const char *string;
+
+  string = (const char *) &sm[1];
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Starting to search for `%s'\n",
               string);
-  ce = GNUNET_new (struct ClientEntry);
-  ce->client = client;
   ce->sh = REGEX_INTERNAL_search (dht,
                                   string,
                                   &handle_search_result,
@@ -341,15 +300,10 @@ handle_search (void *cls,
   if (NULL == ce->sh)
   {
     GNUNET_break (0);
-    GNUNET_free (ce);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    GNUNET_SERVICE_client_drop (ce->client);
     return;
   }
-  GNUNET_CONTAINER_DLL_insert (client_head,
-                               client_tail,
-                               ce);
-  GNUNET_SERVER_notification_context_add (nc, client);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (ce->client);
 }
 
 
@@ -357,19 +311,14 @@ handle_search (void *cls,
  * Process regex requests.
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *cfg)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {&handle_announce, NULL, GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE, 0},
-    {&handle_search, NULL, GNUNET_MESSAGE_TYPE_REGEX_SEARCH, 0},
-    {NULL, NULL, 0, 0}
-  };
-
   my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (cfg);
   if (NULL == my_private_key)
   {
@@ -386,28 +335,84 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   }
   GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
                                  NULL);
-  nc = GNUNET_SERVER_notification_context_create (server, 1);
   stats = GNUNET_STATISTICS_create ("regex", cfg);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect,
-                                   NULL);
 }
 
 
 /**
- * The main function for the regex service.
+ * Callback called when a client connects to the service.
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
  */
-int
-main (int argc, char *const *argv)
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "regex",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+  struct ClientEntry *ce;
+
+  ce = GNUNET_new (struct ClientEntry);
+  ce->client = c;
+  ce->mq = mq;
+  return ce;
 }
 
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
+{
+  struct ClientEntry *ce = internal_cls;
+
+  if (NULL != ce->refresh_task)
+  {
+    GNUNET_SCHEDULER_cancel (ce->refresh_task);
+    ce->refresh_task = NULL;
+  }
+  if (NULL != ce->ah)
+  {
+    REGEX_INTERNAL_announce_cancel (ce->ah);
+    ce->ah = NULL;
+  }
+  if (NULL != ce->sh)
+  {
+    REGEX_INTERNAL_search_cancel (ce->sh);
+    ce->sh = NULL;
+  }
+  GNUNET_free (ce);
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("regex",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (announce,
+                        GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE,
+                        struct AnnounceMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (search,
+                        GNUNET_MESSAGE_TYPE_REGEX_SEARCH,
+                        struct RegexSearchMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
+
 /* end of gnunet-service-regex.c */
diff --git a/src/regex/plugin_block_regex.c b/src/regex/plugin_block_regex.c
index 36926c25d..287ecf905 100644
--- a/src/regex/plugin_block_regex.c
+++ b/src/regex/plugin_block_regex.c
@@ -23,16 +23,75 @@
  * @brief blocks used for regex storage and search
  * @author Bartlomiej Polot
  */
-
 #include "platform.h"
 #include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
 #include "block_regex.h"
 #include "regex_block_lib.h"
-#include "gnunet_constants.h"
 #include "gnunet_signatures.h"
 
 
 /**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * How big is the BF we use for REGEX blocks?
+ */
+#define REGEX_BF_SIZE 8
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_regex_create_group (void *cls,
+                                 enum GNUNET_BLOCK_Type type,
+                                 uint32_t nonce,
+                                 const void *raw_data,
+                                 size_t raw_data_size,
+                                 va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = REGEX_BF_SIZE;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
+
+/**
  * Function called to validate a reply or a request of type
  * #GNUNET_BLOCK_TYPE_REGEX.
  * For request evaluation, pass "NULL" for the reply_block.
@@ -42,10 +101,9 @@
  *
  * @param cls closure
  * @param type block type
+ * @param bg block group to evaluate against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -55,15 +113,16 @@
 static enum GNUNET_BLOCK_EvaluationResult
 evaluate_block_regex (void *cls,
                       enum GNUNET_BLOCK_Type type,
+                      struct GNUNET_BLOCK_Group *bg,
                       enum GNUNET_BLOCK_EvaluationOptions eo,
                       const struct GNUNET_HashCode *query,
-                      struct GNUNET_CONTAINER_BloomFilter **bf,
-                      int32_t bf_mutator,
                       const void *xquery,
                       size_t xquery_size,
                       const void *reply_block,
                       size_t reply_block_size)
 {
+  struct GNUNET_HashCode chash;
+
   if (NULL == reply_block)
   {
     if (0 != xquery_size)
@@ -112,24 +171,13 @@ evaluate_block_regex (void *cls,
     default:
       break;
   }
-  if (NULL != bf)
-  {
-    struct GNUNET_HashCode chash;
-    struct GNUNET_HashCode mhash;
-
-    GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash);
-    GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash);
-    if (NULL != *bf)
-    {
-      if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-        return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-    }
-    else
-    {
-      *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, GNUNET_CONSTANTS_BLOOMFILTER_K);
-    }
-    GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-  }
+  GNUNET_CRYPTO_hash (reply_block,
+                      reply_block_size,
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_OK_MORE;
 }
 
@@ -144,10 +192,9 @@ evaluate_block_regex (void *cls,
  *
  * @param cls closure
  * @param type block type
+ * @param bg block group to evaluate against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in @a xquery
  * @param reply_block response to validate
@@ -157,14 +204,15 @@ evaluate_block_regex (void *cls,
 static enum GNUNET_BLOCK_EvaluationResult
 evaluate_block_regex_accept (void *cls,
                              enum GNUNET_BLOCK_Type type,
+                             struct GNUNET_BLOCK_Group *bg,
                              enum GNUNET_BLOCK_EvaluationOptions eo,
-                             const struct GNUNET_HashCode * query,
-                             struct GNUNET_CONTAINER_BloomFilter **bf,
-                             int32_t bf_mutator, const void *xquery,
+                             const struct GNUNET_HashCode *query,
+                             const void *xquery,
                              size_t xquery_size, const void *reply_block,
                              size_t reply_block_size)
 {
   const struct RegexAcceptBlock *rba;
+  struct GNUNET_HashCode chash;
 
   if (0 != xquery_size)
   {
@@ -202,24 +250,13 @@ evaluate_block_regex_accept (void *cls,
     GNUNET_break_op(0);
     return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
   }
-  if (NULL != bf)
-  {
-    struct GNUNET_HashCode chash;
-    struct GNUNET_HashCode mhash;
-
-    GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash);
-    GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash);
-    if (NULL != *bf)
-    {
-      if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
-        return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
-    }
-    else
-    {
-      *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, GNUNET_CONSTANTS_BLOOMFILTER_K);
-    }
-    GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
-  }
+  GNUNET_CRYPTO_hash (reply_block,
+                      reply_block_size,
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
   return GNUNET_BLOCK_EVALUATION_OK_MORE;
 }
 
@@ -232,11 +269,11 @@ evaluate_block_regex_accept (void *cls,
  * be done with the #GNUNET_BLOCK_get_key() function.
  *
  * @param cls closure
+ * @param ctx block context
  * @param type block type
+ * @param bg group to evaluate against
  * @param eo control flags
  * @param query original query (hash)
- * @param bf pointer to bloom filter associated with query; possibly updated (!)
- * @param bf_mutator mutation value for bf
  * @param xquery extrended query data (can be NULL, depending on type)
  * @param xquery_size number of bytes in xquery
  * @param reply_block response to validate
@@ -245,11 +282,11 @@ evaluate_block_regex_accept (void *cls,
  */
 static enum GNUNET_BLOCK_EvaluationResult
 block_plugin_regex_evaluate (void *cls,
+                             struct GNUNET_BLOCK_Context *ctx,
                              enum GNUNET_BLOCK_Type type,
+                             struct GNUNET_BLOCK_Group *bg,
                              enum GNUNET_BLOCK_EvaluationOptions eo,
                              const struct GNUNET_HashCode *query,
-                             struct GNUNET_CONTAINER_BloomFilter **bf,
-                             int32_t bf_mutator,
                              const void *xquery,
                              size_t xquery_size,
                              const void *reply_block,
@@ -262,18 +299,18 @@ block_plugin_regex_evaluate (void *cls,
     case GNUNET_BLOCK_TYPE_REGEX:
       result = evaluate_block_regex (cls,
                                      type,
+                                     bg,
                                      eo,
                                      query,
-                                     bf, bf_mutator,
                                      xquery, xquery_size,
                                      reply_block, reply_block_size);
       break;
     case GNUNET_BLOCK_TYPE_REGEX_ACCEPT:
       result = evaluate_block_regex_accept (cls,
                                             type,
+                                            bg,
                                             eo,
                                             query,
-                                            bf, bf_mutator,
                                             xquery, xquery_size,
                                             reply_block, reply_block_size);
       break;
@@ -346,6 +383,7 @@ libgnunet_plugin_block_regex_init (void *cls)
   api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
   api->evaluate = &block_plugin_regex_evaluate;
   api->get_key = &block_plugin_regex_get_key;
+  api->create_group = &block_plugin_regex_create_group;
   api->types = types;
   return api;
 }
@@ -357,7 +395,7 @@ libgnunet_plugin_block_regex_init (void *cls)
 void *
 libgnunet_plugin_block_regex_done (void *cls)
 {
-  struct GNUNET_TRANSPORT_PluginFunctions *api = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
 
   GNUNET_free (api);
   return NULL;
diff --git a/src/regex/regex_api_announce.c b/src/regex/regex_api_announce.c
index 70bf34bc8..8e018f26c 100644
--- a/src/regex/regex_api_announce.c
+++ b/src/regex/regex_api_announce.c
@@ -146,7 +146,7 @@ GNUNET_REGEX_announce (const struct GNUNET_CONFIGURATION_Handle *cfg,
   size_t slen;
 
   slen = strlen (regex) + 1;
-  if (slen + sizeof (struct AnnounceMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (slen + sizeof (struct AnnounceMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Regex `%s' is too long!\n"),
diff --git a/src/regex/regex_api_search.c b/src/regex/regex_api_search.c
index b7a015f87..a5480ac7a 100644
--- a/src/regex/regex_api_search.c
+++ b/src/regex/regex_api_search.c
@@ -206,7 +206,7 @@ GNUNET_REGEX_search (const struct GNUNET_CONFIGURATION_Handle *cfg,
   struct GNUNET_REGEX_Search *s;
   size_t slen = strlen (string) + 1;
 
-  if (slen + sizeof (struct RegexSearchMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (slen + sizeof (struct RegexSearchMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Search string `%s' is too long!\n"),
diff --git a/src/rest/gnunet-rest-server.c b/src/rest/gnunet-rest-server.c
index b115deb20..5b92c6c43 100644
--- a/src/rest/gnunet-rest-server.c
+++ b/src/rest/gnunet-rest-server.c
@@ -66,7 +66,7 @@ static struct GNUNET_SCHEDULER_Task *httpd_task;
 /**
  * The port the service is running on (default 7776)
  */
-static unsigned long port = GNUNET_REST_SERVICE_PORT;
+static unsigned long long port = GNUNET_REST_SERVICE_PORT;
 
 /**
  * The listen socket of the service for IPv4
@@ -748,7 +748,7 @@ run (void *cls,
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Service listens on port %lu\n",
+              "Service listens on port %llu\n",
               port);
   httpd = MHD_start_daemon (MHD_USE_DEBUG | MHD_USE_NO_LISTEN_SOCKET,
                             0,
@@ -783,10 +783,12 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", NULL,
-      gettext_noop ("listen on specified port (default: 7776)"), 1,
-      &GNUNET_GETOPT_set_ulong, &port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_ulong ('p',
+                                    "port",
+                                    "PORT",
+                                    gettext_noop ("listen on specified port (default: 7776)"),
+                                    &port),
     GNUNET_GETOPT_OPTION_END
   };
   static const char* err_page =
diff --git a/src/revocation/.gitignore b/src/revocation/.gitignore
index dee463129..9acd3ac33 100644
--- a/src/revocation/.gitignore
+++ b/src/revocation/.gitignore
@@ -1,2 +1,4 @@
 gnunet-service-revocation
 gnunet-revocation
+test_revocation
+test_local_revocation.py
diff --git a/src/revocation/Makefile.am b/src/revocation/Makefile.am
index a681b2f33..82755b485 100644
--- a/src/revocation/Makefile.am
+++ b/src/revocation/Makefile.am
@@ -1,6 +1,8 @@
 # This Makefile.am is in the public domain
 AM_CPPFLAGS = -I$(top_srcdir)/src/include
 
+plugindir = $(libdir)/gnunet
+
 if MINGW
  WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols
 endif
@@ -21,6 +23,20 @@ bin_PROGRAMS = \
  gnunet-revocation
 
 
+plugin_LTLIBRARIES = \
+  libgnunet_plugin_block_revocation.la
+
+libgnunet_plugin_block_revocation_la_SOURCES = \
+  plugin_block_revocation.c
+libgnunet_plugin_block_revocation_la_LIBADD = \
+  libgnunetrevocation.la \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
+  $(top_builddir)/src/util/libgnunetutil.la  \
+  $(LTLIBINTL)
+libgnunet_plugin_block_revocation_la_LDFLAGS = \
+ $(GN_PLUGIN_LDFLAGS)
+
+
 gnunet_revocation_SOURCES = \
  gnunet-revocation.c
 gnunet_revocation_LDADD = \
@@ -72,7 +88,7 @@ check_SCRIPTS = \
   test_local_revocation.py
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = \
  $(check_SCRIPTS) \
  $(check_PROGRAMS)
@@ -86,4 +102,3 @@ test_local_revocation.py: test_local_revocation.py.in Makefile
 
 EXTRA_DIST = test_revocation.conf \
         test_local_revocation.py.in
-
diff --git a/src/revocation/gnunet-revocation.c b/src/revocation/gnunet-revocation.c
index 133468789..13e6b23a5 100644
--- a/src/revocation/gnunet-revocation.c
+++ b/src/revocation/gnunet-revocation.c
@@ -527,19 +527,31 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'f', "filename", "NAME",
-     gettext_noop ("use NAME for the name of the revocation file"),
-     1, &GNUNET_GETOPT_set_string, &filename},
-    {'R', "revoke", "NAME",
-     gettext_noop ("revoke the private key associated for the the private key associated with the ego NAME "),
-     1, &GNUNET_GETOPT_set_string, &revoke_ego},
-    {'p', "perform", NULL,
-     gettext_noop ("actually perform revocation, otherwise we just do the precomputation"),
-     0, &GNUNET_GETOPT_set_one, &perform},
-    {'t', "test", "KEY",
-     gettext_noop ("test if the public key KEY has been revoked"),
-     1, &GNUNET_GETOPT_set_string, &test_ego},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('f',
+                                 "filename",
+                                 "NAME",
+                                 gettext_noop ("use NAME for the name of the revocation file"),
+                                 &filename),
+
+    GNUNET_GETOPT_option_string ('R',
+                                 "revoke",
+                                 "NAME",
+                                 gettext_noop ("revoke the private key associated for the the private key associated with the ego NAME "),
+                                 &revoke_ego), 
+
+    GNUNET_GETOPT_option_flag ('p',
+                                  "perform",
+                                  gettext_noop ("actually perform revocation, otherwise we just do the precomputation"),
+                                  &perform),
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "test",
+                                 "KEY",
+                                 gettext_noop ("test if the public key KEY has been revoked"),
+                                 &test_ego), 
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/revocation/gnunet-service-revocation.c b/src/revocation/gnunet-service-revocation.c
index 2965808fa..9d077f874 100644
--- a/src/revocation/gnunet-service-revocation.c
+++ b/src/revocation/gnunet-service-revocation.c
@@ -38,6 +38,7 @@
 #include "platform.h"
 #include <math.h>
 #include "gnunet_util_lib.h"
+#include "gnunet_block_lib.h"
 #include "gnunet_constants.h"
 #include "gnunet_protocols.h"
 #include "gnunet_signatures.h"
@@ -215,7 +216,7 @@ client_connect_cb (void *cls,
  * @param client the new client
  * @param app_cls must alias @a client
  */
-static void 
+static void
 client_disconnect_cb (void *cls,
                       struct GNUNET_SERVICE_Client *client,
                       void *app_cls)
@@ -352,7 +353,7 @@ publicize_rm (const struct RevokeMessage *rm)
                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   /* add to set for future connections */
   e.size = htons (rm->header.size);
-  e.element_type = 0;
+  e.element_type = GNUNET_BLOCK_TYPE_REVOCATION;
   e.data = rm;
   if (GNUNET_OK !=
       GNUNET_SET_add_element (revocation_set,
@@ -432,11 +433,13 @@ handle_p2p_revoke (void *cls,
  *
  * @param cls closure
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status see `enum GNUNET_SET_Status`
  */
 static void
 add_revocation (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t current_size,
                 enum GNUNET_SET_Status status)
 {
   struct PeerEntry *peer_entry = cls;
@@ -450,11 +453,12 @@ add_revocation (void *cls,
       GNUNET_break_op (0);
       return;
     }
-    if (0 != element->element_type)
+    if (GNUNET_BLOCK_TYPE_REVOCATION != element->element_type)
     {
       GNUNET_STATISTICS_update (stats,
                                 gettext_noop ("# unsupported revocations received via set union"),
-                                1, GNUNET_NO);
+                                1,
+                                GNUNET_NO);
       return;
     }
     rm = element->data;
@@ -509,6 +513,7 @@ transmit_task_cb (void *cls)
                                        &revocation_set_union_app_id,
                                        NULL,
                                        GNUNET_SET_RESULT_ADDED,
+                                       (struct GNUNET_SET_Option[]) {{ 0 }},
                                        &add_revocation,
                                        peer_entry);
   if (GNUNET_OK !=
@@ -601,12 +606,12 @@ handle_core_disconnect (void *cls,
                         void *internal_cls)
 {
   struct PeerEntry *peer_entry = internal_cls;
-  
+
   if (0 == memcmp (peer,
                    &my_identity,
                    sizeof (my_identity)))
     return;
-  GNUNET_assert (NULL != peer_entry);  
+  GNUNET_assert (NULL != peer_entry);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer `%s' disconnected from us\n",
               GNUNET_i2s (peer));
@@ -755,6 +760,7 @@ handle_revocation_union_request (void *cls,
   }
   peer_entry->so = GNUNET_SET_accept (request,
                                       GNUNET_SET_RESULT_ADDED,
+                                      (struct GNUNET_SET_Option[]) {{ 0 }},
                                       &add_revocation,
                                       peer_entry);
   if (GNUNET_OK !=
@@ -779,7 +785,7 @@ handle_revocation_union_request (void *cls,
 static void
 run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
-     struct GNUNET_SERVICE_Handle *service)     
+     struct GNUNET_SERVICE_Handle *service)
 {
   struct GNUNET_MQ_MessageHandler core_handlers[] = {
     GNUNET_MQ_hd_fixed_size (p2p_revoke,
diff --git a/src/revocation/plugin_block_revocation.c b/src/revocation/plugin_block_revocation.c
new file mode 100644
index 000000000..eb0766b81
--- /dev/null
+++ b/src/revocation/plugin_block_revocation.c
@@ -0,0 +1,257 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file block/plugin_block_revocation.c
+ * @brief revocation for a block plugin
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_signatures.h"
+#include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
+#include "revocation.h"
+#include "gnunet_revocation_service.h"
+
+#define DEBUG_REVOCATION GNUNET_EXTRA_LOGGING
+
+/**
+ * Number of bits we set per entry in the bloomfilter.
+ * Do not change!
+ */
+#define BLOOMFILTER_K 16
+
+
+/**
+ * How big is the BF we use for DHT blocks?
+ */
+#define REVOCATION_BF_SIZE 8
+
+
+/**
+ * Context used inside the plugin.
+ */
+struct InternalContext
+{
+
+  unsigned int matching_bits;
+
+};
+
+
+/**
+ * Create a new block group.
+ *
+ * @param ctx block context in which the block group is created
+ * @param type type of the block for which we are creating the group
+ * @param nonce random value used to seed the group creation
+ * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
+ * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
+ * @param va variable arguments specific to @a type
+ * @return block group handle, NULL if block groups are not supported
+ *         by this @a type of block (this is not an error)
+ */
+static struct GNUNET_BLOCK_Group *
+block_plugin_revocation_create_group (void *cls,
+                                      enum GNUNET_BLOCK_Type type,
+                                      uint32_t nonce,
+                                      const void *raw_data,
+                                      size_t raw_data_size,
+                                      va_list va)
+{
+  unsigned int bf_size;
+  const char *guard;
+
+  guard = va_arg (va, const char *);
+  if (0 == strcmp (guard,
+                   "seen-set-size"))
+    bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int),
+                                                           BLOOMFILTER_K);
+  else if (0 == strcmp (guard,
+                        "filter-size"))
+    bf_size = va_arg (va, unsigned int);
+  else
+  {
+    GNUNET_break (0);
+    bf_size = REVOCATION_BF_SIZE;
+  }
+  GNUNET_break (NULL == va_arg (va, const char *));
+  return GNUNET_BLOCK_GROUP_bf_create (cls,
+                                       bf_size,
+                                       BLOOMFILTER_K,
+                                       type,
+                                       nonce,
+                                       raw_data,
+                                       raw_data_size);
+}
+
+
+/**
+ * Function called to validate a reply or a request.  For
+ * request evaluation, simply pass "NULL" for the reply_block.
+ *
+ * @param cls our `struct InternalContext`
+ * @param ctx context
+ * @param type block type
+ * @param group block group to use
+ * @param eo control flags
+ * @param query original query (hash)
+ * @param xquery extrended query data (can be NULL, depending on type)
+ * @param xquery_size number of bytes in xquery
+ * @param reply_block response to validate
+ * @param reply_block_size number of bytes in reply block
+ * @return characterization of result
+ */
+static enum GNUNET_BLOCK_EvaluationResult
+block_plugin_revocation_evaluate (void *cls,
+                                  struct GNUNET_BLOCK_Context *ctx,
+                                  enum GNUNET_BLOCK_Type type,
+                                  struct GNUNET_BLOCK_Group *group,
+                                  enum GNUNET_BLOCK_EvaluationOptions eo,
+                                  const struct GNUNET_HashCode *query,
+                                  const void *xquery,
+                                  size_t xquery_size,
+                                  const void *reply_block,
+                                  size_t reply_block_size)
+{
+  struct InternalContext *ic = cls;
+  struct GNUNET_HashCode chash;
+  const struct RevokeMessage *rm = reply_block;
+
+  if (NULL == reply_block)
+    return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
+  if (reply_block_size != sizeof (*rm))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
+  }
+  if (GNUNET_YES !=
+      GNUNET_REVOCATION_check_pow (&rm->public_key,
+                                   rm->proof_of_work,
+                                   ic->matching_bits))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_ecdsa_verify (GNUNET_SIGNATURE_PURPOSE_REVOCATION,
+                                  &rm->purpose,
+                                  &rm->signature,
+                                  &rm->public_key))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
+  }
+  GNUNET_CRYPTO_hash (&rm->public_key,
+                      sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
+                      &chash);
+  if (GNUNET_YES ==
+      GNUNET_BLOCK_GROUP_bf_test_and_set (group,
+                                          &chash))
+    return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
+  return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
+}
+
+
+/**
+ * Function called to obtain the key for a block.
+ *
+ * @param cls closure
+ * @param type block type
+ * @param block block to get the key for
+ * @param block_size number of bytes in block
+ * @param key set to the key (query) for the given block
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
+ *         (or if extracting a key from a block of this type does not work)
+ */
+static int
+block_plugin_revocation_get_key (void *cls,
+                                 enum GNUNET_BLOCK_Type type,
+                                 const void *block,
+                                 size_t block_size,
+                                 struct GNUNET_HashCode *key)
+{
+  const struct RevokeMessage *rm = block;
+
+  if (block_size != sizeof (*rm))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_CRYPTO_hash (&rm->public_key,
+                      sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
+                      key);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Entry point for the plugin.
+ *
+ * @param cls the configuration to use
+ */
+void *
+libgnunet_plugin_block_revocation_init (void *cls)
+{
+  static enum GNUNET_BLOCK_Type types[] =
+  {
+    GNUNET_BLOCK_TYPE_REVOCATION,
+    GNUNET_BLOCK_TYPE_ANY       /* end of list */
+  };
+  const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
+  struct GNUNET_BLOCK_PluginFunctions *api;
+  struct InternalContext *ic;
+  unsigned long long matching_bits;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             "REVOCATION",
+                                             "WORKBITS",
+                                             &matching_bits))
+    return NULL;
+
+  api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
+  api->evaluate = &block_plugin_revocation_evaluate;
+  api->get_key = &block_plugin_revocation_get_key;
+  api->create_group = &block_plugin_revocation_create_group;
+  api->types = types;
+  ic = GNUNET_new (struct InternalContext);
+  ic->matching_bits = (unsigned int) matching_bits;
+  api->cls = ic;
+  return api;
+}
+
+
+/**
+ * Exit point from the plugin.
+ */
+void *
+libgnunet_plugin_block_revocation_done (void *cls)
+{
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
+  struct InternalContext *ic = api->cls;
+
+  GNUNET_free (ic);
+  GNUNET_free (api);
+  return NULL;
+}
+
+/* end of plugin_block_revocation.c */
diff --git a/src/revocation/revocation_api.c b/src/revocation/revocation_api.c
index fde0296a4..ef659baa0 100644
--- a/src/revocation/revocation_api.c
+++ b/src/revocation/revocation_api.c
@@ -91,7 +91,7 @@ handle_revocation_query_response (void *cls,
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Revocation query result: %d\n",
-              ntohl (qrm->is_valid));
+              (uint32_t) ntohl (qrm->is_valid));
   q->func (q->func_cls,
            ntohl (qrm->is_valid));
   GNUNET_REVOCATION_query_cancel (q);
@@ -225,7 +225,7 @@ handle_revocation_response (void *cls,
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Revocation transmission result: %d\n",
-              ntohl (rrm->is_valid));
+              (uint32_t) ntohl (rrm->is_valid));
   h->func (h->func_cls,
            ntohl (rrm->is_valid));
   GNUNET_REVOCATION_revoke_cancel (h);
diff --git a/src/revocation/test_local_revocation.py.in b/src/revocation/test_local_revocation.py.in
index 69e68a197..28257715f 100644
--- a/src/revocation/test_local_revocation.py.in
+++ b/src/revocation/test_local_revocation.py.in
@@ -42,6 +42,7 @@ TEST_REVOCATION_EGO = "revoc_test"
 
 get_clean = subprocess.Popen ([config, '-c', TEST_CONFIGURATION, '-s', 'PATHS', '-o', 'GNUNET_HOME', '-f'], stdout=subprocess.PIPE)
 cleandir, x = get_clean.communicate ()
+cleandir = cleandir.decode("utf-8")
 cleandir = cleandir.rstrip ('\n').rstrip ('\r')
 
 if os.path.isdir (cleandir):
@@ -64,6 +65,7 @@ try:
   sys.stderr.flush ()
   idd = subprocess.Popen ([ident, '-d'], stdout=subprocess.PIPE)
   rev_key, x = idd.communicate ()
+  rev_key = rev_key.decode("utf-8")
   if len (rev_key.split ()) < 3:
     raise Exception ("can't get revocation key out of `" + rev_key + "'")
   rev_key = rev_key.split ()[2]
@@ -73,6 +75,7 @@ try:
   sys.stderr.flush ()
   tst = subprocess.Popen ([revoc, '-t', rev_key, '-c', TEST_CONFIGURATION], stdout=subprocess.PIPE)
   output_not_revoked, x = tst.communicate ()
+  output_not_revoked = output_not_revoked.decode("utf-8")
   if tst.returncode != 0:
     raise Exception ("gnunet-revocation failed to test a key - " + str (tst.returncode) + ": " + output_not_revoked)
   if 'valid' not in output_not_revoked:
@@ -94,6 +97,7 @@ try:
   sys.stderr.flush ()
   tst = subprocess.Popen ([revoc, '-t', rev_key, '-c', TEST_CONFIGURATION], stdout=subprocess.PIPE)
   output_revoked, x = tst.communicate ()
+  output_revoked = output_revoked.decode("utf-8")
   if tst.returncode != 0:
     raise Exception ("gnunet-revocation failed to test a revoked key")
   if 'revoked' not in output_revoked:
diff --git a/src/revocation/test_revocation.c b/src/revocation/test_revocation.c
index d3bbb879a..8d5593694 100644
--- a/src/revocation/test_revocation.c
+++ b/src/revocation/test_revocation.c
@@ -104,8 +104,8 @@ revocation_remote_cb (void *cls,
 
   if (GNUNET_NO == is_valid)
   {
-    fprintf (stderr,
-             "Local revocation successful\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Local revocation successful\n");
     ok = 0;
     GNUNET_SCHEDULER_shutdown ();
     return;
@@ -118,8 +118,8 @@ revocation_remote_cb (void *cls,
                                   NULL);
     return;
   }
-  fprintf (stderr,
-           "Flooding of revocation failed\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Flooding of revocation failed\n");
   ok = 2;
   GNUNET_SCHEDULER_shutdown ();
 }
@@ -141,8 +141,8 @@ revocation_cb (void *cls,
   testpeers[1].revok_handle = NULL;
   if (GNUNET_NO == is_valid)
   {
-    fprintf (stderr,
-             "Revocation successful\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Revocation successful\n");
     check_revocation (NULL);
   }
 }
@@ -386,8 +386,8 @@ test_connection (void *cls,
    /* We are generating a CLIQUE */
   if (NUM_TEST_PEERS * (NUM_TEST_PEERS -1) == links_succeeded)
   {
-    fprintf (stderr,
-             "Testbed connected peers, initializing test\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Testbed connected peers, initializing test\n");
     for (c = 0; c < num_peers; c++)
     {
       testpeers[c].p = peers[c];
@@ -403,8 +403,8 @@ test_connection (void *cls,
   }
   else
   {
-    fprintf (stderr,
-             "Testbed failed to connect peers\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Testbed failed to connect peers\n");
     ok = 5;
     GNUNET_SCHEDULER_shutdown ();
     return;
diff --git a/src/rps/Makefile.am b/src/rps/Makefile.am
index eee1a91b9..e6c8cd929 100644
--- a/src/rps/Makefile.am
+++ b/src/rps/Makefile.am
@@ -96,26 +96,32 @@ rps_test_src = \
  gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
-test_service_rps_view_SOURCES = gnunet-service-rps_view.h gnunet-service-rps_view.c \
-                                                                                                                                test_service_rps_view.c
+test_service_rps_view_SOURCES = \
+  gnunet-service-rps_view.h gnunet-service-rps_view.c \
+  test_service_rps_view.c
 test_service_rps_view_LDADD = $(top_builddir)/src/util/libgnunetutil.la
 
-test_service_rps_peers_SOURCES = gnunet-service-rps_peers.h gnunet-service-rps_peers.c \
-                                                                                                                                 test_service_rps_peers.c
-test_service_rps_peers_LDADD = $(top_builddir)/src/util/libgnunetutil.la \
-                                                                                                                         $(top_builddir)/src/cadet/libgnunetcadet.la
-
-test_service_rps_custommap_SOURCES = gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \
-                                                                                                                                 test_service_rps_custommap.c
-test_service_rps_custommap_LDADD = $(top_builddir)/src/util/libgnunetutil.la
-
-test_service_rps_sampler_elem_SOURCES = gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \
-                                                                                                                                                                rps-test_util.h rps-test_util.c \
-                                                                                                                                        test_service_rps_sampler_elem.c
+test_service_rps_peers_SOURCES = \
+  gnunet-service-rps_peers.h gnunet-service-rps_peers.c \
+  test_service_rps_peers.c
+test_service_rps_peers_LDADD = \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la
+
+test_service_rps_custommap_SOURCES = \
+  gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \
+  test_service_rps_custommap.c
+test_service_rps_custommap_LDADD = \
+  $(top_builddir)/src/util/libgnunetutil.la
+
+test_service_rps_sampler_elem_SOURCES = \
+  gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \
+  rps-test_util.h rps-test_util.c \
+  test_service_rps_sampler_elem.c
 test_service_rps_sampler_elem_LDADD = $(top_builddir)/src/util/libgnunetutil.la
 
 test_rps_malicious_1_SOURCES = $(rps_test_src)
diff --git a/src/rps/gnunet-rps.c b/src/rps/gnunet-rps.c
index 3dbb8053e..bbac0d634 100644
--- a/src/rps/gnunet-rps.c
+++ b/src/rps/gnunet-rps.c
@@ -43,43 +43,7 @@ static struct GNUNET_RPS_Request_Handle *req_handle;
 /**
  * PeerID (Option --seed)
  */
-static struct GNUNET_PeerIdentity *peer_id;
-
-
-/**
- * Set an option of type 'struct GNUNET_PeerIdentity *' from the command line.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'struct GNUNET_PeerIdentity *', which will be allocated with the requested string.
- *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'char *',
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a PeerID)
- * @return #GNUNET_OK
- */
-static int
-GNUNET_GETOPT_set_peerid (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value)
-{
-  struct GNUNET_PeerIdentity **val = (struct GNUNET_PeerIdentity **) scls;
-
-  GNUNET_assert (NULL != value);
-  GNUNET_free_non_null (*val);
-  /* Not quite sure whether that is a sane way */
-  *val = GNUNET_new (struct GNUNET_PeerIdentity);
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_eddsa_public_key_from_string (value,
-                                                  strlen (value),
-                                                  &((*val)->public_key)))
-  {
-    FPRINTF (stderr, "Invalid peer ID %s\n", value);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
+static struct GNUNET_PeerIdentity peer_id;
 
 
 /**
@@ -139,10 +103,13 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   static uint64_t num_peers;
+  static struct GNUNET_PeerIdentity zero_pid;
 
   rps_handle = GNUNET_RPS_connect (cfg);
 
-  if (NULL == peer_id)
+  if (0 == memcmp (&zero_pid,
+                   &peer_id,
+                   sizeof (peer_id)))
   { /* Request n PeerIDs */
     /* If number was specified use it, else request single peer. */
     num_peers = (NULL == args[0]) ? 1 : atoi (args[0]);
@@ -153,8 +120,8 @@ run (void *cls,
   }
   else
   { /* Seed PeerID */
-    GNUNET_RPS_seed_ids (rps_handle, 1, peer_id);
-    FPRINTF (stdout, "Seeded PeerID %s\n", GNUNET_i2s_full (peer_id));
+    GNUNET_RPS_seed_ids (rps_handle, 1, &peer_id);
+    FPRINTF (stdout, "Seeded PeerID %s\n", GNUNET_i2s_full (&peer_id));
     ret = 0;
     GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
   }
@@ -172,10 +139,12 @@ main (int argc, char *const *argv)
 {
   const char helpstr[] =
     "Get random GNUnet peers. If none is specified a single is requested.";
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'s', "seed", "PEER_ID",
-      gettext_noop ("Seed a PeerID"),
-      GNUNET_YES, &GNUNET_GETOPT_set_peerid, &peer_id},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_base32_auto ('s',
+                                          "seed",
+                                          "PEER_ID",
+                                          gettext_noop ("Seed a PeerID"),
+                                          &peer_id),
     GNUNET_GETOPT_OPTION_END
   };
   return (GNUNET_OK ==
diff --git a/src/rps/gnunet-service-rps.c b/src/rps/gnunet-service-rps.c
index 9de1f8d3a..0a4543b30 100644
--- a/src/rps/gnunet-service-rps.c
+++ b/src/rps/gnunet-service-rps.c
@@ -217,6 +217,11 @@ static struct GNUNET_NSE_Handle *nse;
 static struct GNUNET_CADET_Handle *cadet_handle;
 
 /**
+ * @brief Port to communicate to other peers.
+ */
+static struct GNUNET_CADET_Port *cadet_port;
+
+/**
  * Handler to PEERINFO.
  */
 static struct GNUNET_PEERINFO_Handle *peerinfo_handle;
@@ -838,14 +843,10 @@ clean_peer (const struct GNUNET_PeerIdentity *peer)
  */
 static void
 cleanup_destroyed_channel (void *cls,
-                           const struct GNUNET_CADET_Channel *channel,
-                           void *channel_ctx)
+                           const struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_PeerIdentity *peer;
-
-  peer = (struct GNUNET_PeerIdentity *) GNUNET_CADET_channel_get_info (
-      (struct GNUNET_CADET_Channel *) channel, GNUNET_CADET_OPTION_PEER);
-       // FIXME wait for cadet to change this function
+  struct GNUNET_PeerIdentity *peer = cls;
+  uint32_t *channel_flag;
 
   if (GNUNET_NO == Peers_check_peer_known (peer))
   { /* We don't know a context to that peer */
@@ -858,7 +859,7 @@ cleanup_destroyed_channel (void *cls,
   if (GNUNET_YES == Peers_check_peer_flag (peer, Peers_TO_DESTROY))
   { /* We are in the middle of removing that peer from our knowledge. In this
        case simply make sure that the channels are cleaned. */
-    Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+    Peers_cleanup_destroyed_channel (cls, channel);
     to_file (file_name_view_log,
              "-%s\t(cleanup channel, ourself)",
              GNUNET_i2s_full (peer));
@@ -872,16 +873,17 @@ cleanup_destroyed_channel (void *cls,
      *  - ourselves  -> cleaning send channel -> clean context
      *  - other peer -> peer probably went down -> remove
      */
-    if (GNUNET_YES == Peers_check_channel_flag (channel_ctx, Peers_CHANNEL_CLEAN))
+    channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_SENDING);
+    if (GNUNET_YES == Peers_check_channel_flag (channel_flag, Peers_CHANNEL_CLEAN))
     { /* We are about to clean the sending channel. Clean the respective
        * context */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       return;
     }
     else
     { /* Other peer destroyed our sending channel that he is supposed to keep
        * open. It probably went down. Remove it from our knowledge. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       remove_peer (peer);
       return;
     }
@@ -893,17 +895,18 @@ cleanup_destroyed_channel (void *cls,
      *  - ourselves  -> peer tried to establish channel twice -> clean context
      *  - other peer -> peer doesn't want to send us data -> clean
      */
+    channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_RECEIVING);
     if (GNUNET_YES ==
-        Peers_check_channel_flag (channel_ctx, Peers_CHANNEL_ESTABLISHED_TWICE))
+        Peers_check_channel_flag (channel_flag, Peers_CHANNEL_ESTABLISHED_TWICE))
     { /* Other peer tried to establish a channel to us twice. We do not accept
        * that. Clean the context. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       return;
     }
     else
     { /* Other peer doesn't want to send us data anymore. We are free to clean
        * it. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       clean_peer (peer);
       return;
     }
@@ -1023,7 +1026,7 @@ client_respond (void *cls,
   size_needed = sizeof (struct GNUNET_RPS_CS_ReplyMessage) +
                 num_peers * sizeof (struct GNUNET_PeerIdentity);
 
-  GNUNET_assert (GNUNET_SERVER_MAX_MESSAGE_SIZE >= size_needed);
+  GNUNET_assert (GNUNET_MAX_MESSAGE_SIZE >= size_needed);
 
   ev = GNUNET_MQ_msg_extra (out_msg,
                             num_peers * sizeof (struct GNUNET_PeerIdentity),
@@ -1048,7 +1051,6 @@ client_respond (void *cls,
  * Handle RPS request from the client.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message
  */
 static void
@@ -1065,7 +1067,7 @@ handle_client_request (void *cls,
   size_needed = sizeof (struct GNUNET_RPS_CS_RequestMessage) +
                 num_peers * sizeof (struct GNUNET_PeerIdentity);
 
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  if (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Message received from client has size larger than expected\n");
@@ -1100,12 +1102,11 @@ handle_client_request (void *cls,
  * @brief Handle a message that requests the cancellation of a request
  *
  * @param cls unused
- * @param client the client that requests the cancellation
  * @param message the message containing the id of the request
  */
 static void
 handle_client_request_cancel (void *cls,
-                          const struct GNUNET_RPS_CS_RequestCancelMessage *msg)
+                              const struct GNUNET_RPS_CS_RequestCancelMessage *msg)
 {
   struct ClientContext *cli_ctx = cls;
   struct ReplyCls *rep_cls;
@@ -1157,7 +1158,6 @@ check_client_seed (void *cls, const struct GNUNET_RPS_CS_SeedMessage *msg)
  * Handle seed from the client.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message
  */
 static void
@@ -1172,7 +1172,7 @@ handle_client_seed (void *cls,
   num_peers = ntohl (msg->num_peers);
   peers = (struct GNUNET_PeerIdentity *) &msg[1];
   //peers = GNUNET_new_array (num_peers, struct GNUNET_PeerIdentity);
-  //GNUNET_memcpy (peers, &in_msg[1], num_peers * sizeof (struct GNUNET_PeerIdentity));
+  //GNUNET_memcpy (peers, &msg[1], num_peers * sizeof (struct GNUNET_PeerIdentity));
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Client seeded peers:\n");
@@ -1200,18 +1200,15 @@ handle_client_seed (void *cls,
  * the channel is blocked for all other communication.
  *
  * @param cls Closure
- * @param channel The channel the CHECK was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_check (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *msg)
+                   const struct GNUNET_MessageHeader *msg)
 {
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  const struct GNUNET_PeerIdentity *peer = cls;
+
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 /**
@@ -1221,24 +1218,16 @@ handle_peer_check (void *cls,
  * in the temporary list for pushed PeerIDs.
  *
  * @param cls Closure
- * @param channel The channel the PUSH was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_push (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *msg)
+                  const struct GNUNET_MessageHeader *msg)
 {
-  const struct GNUNET_PeerIdentity *peer;
+  const struct GNUNET_PeerIdentity *peer = cls;
 
   // (check the proof of work (?))
 
-  peer = (const struct GNUNET_PeerIdentity *)
-    GNUNET_CADET_channel_get_info (channel, GNUNET_CADET_OPTION_PEER);
-  // FIXME wait for cadet to change this function
-
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Received PUSH (%s)\n",
        GNUNET_i2s (peer));
@@ -1261,23 +1250,20 @@ handle_peer_push (void *cls,
                                    tmp_att_peer);
       add_peer_array_to_set (peer, 1, att_peer_set);
     }
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
 
 
   else if (2 == mal_type)
   { /* We attack one single well-known peer - simply ignore */
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
   #endif /* ENABLE_MALICIOUS */
 
   /* Add the sending peer to the push_map */
   CustomPeerMap_put (push_map, peer);
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 
@@ -1287,24 +1273,15 @@ handle_peer_push (void *cls,
  * Reply with the view of PeerIDs.
  *
  * @param cls Closure
- * @param channel The channel the PULL REQUEST was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_pull_request (void *cls,
-                          struct GNUNET_CADET_Channel *channel,
-                          void **channel_ctx,
-                          const struct GNUNET_MessageHeader *msg)
+                          const struct GNUNET_MessageHeader *msg)
 {
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity *peer = cls;
   const struct GNUNET_PeerIdentity *view_array;
 
-  peer = (struct GNUNET_PeerIdentity *)
-    GNUNET_CADET_channel_get_info (channel,
-                                   GNUNET_CADET_OPTION_PEER);
-  // FIXME wait for cadet to change this function
-
   LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REQUEST (%s)\n", GNUNET_i2s (peer));
 
   #ifdef ENABLE_MALICIOUS
@@ -1312,8 +1289,7 @@ handle_peer_pull_request (void *cls,
       || 3 == mal_type)
   { /* Try to maximise representation */
     send_pull_reply (peer, mal_peers, num_mal_peers);
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
 
   else if (2 == mal_type)
@@ -1322,101 +1298,93 @@ handle_peer_pull_request (void *cls,
     {
       send_pull_reply (peer, mal_peers, num_mal_peers);
     }
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
   #endif /* ENABLE_MALICIOUS */
 
   view_array = View_get_as_array ();
-
   send_pull_reply (peer, view_array, View_size ());
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 
 /**
- * Handle PULL REPLY message from another peer.
- *
  * Check whether we sent a corresponding request and
  * whether this reply is the first one.
  *
  * @param cls Closure
- * @param channel The channel the PUSH was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
 static int
-handle_peer_pull_reply (void *cls,
-                        struct GNUNET_CADET_Channel *channel,
-                        void **channel_ctx,
-                        const struct GNUNET_MessageHeader *msg)
+check_peer_pull_reply (void *cls,
+                       const struct GNUNET_RPS_P2P_PullReplyMessage *msg)
 {
-  struct GNUNET_RPS_P2P_PullReplyMessage *in_msg;
-  struct GNUNET_PeerIdentity *peers;
-  struct GNUNET_PeerIdentity *sender;
-  uint32_t i;
-#ifdef ENABLE_MALICIOUS
-  struct AttackedPeer *tmp_att_peer;
-#endif /* ENABLE_MALICIOUS */
+  struct GNUNET_PeerIdentity *sender = cls;
 
-  /* Check for protocol violation */
-  if (sizeof (struct GNUNET_RPS_P2P_PullReplyMessage) > ntohs (msg->size))
+  if (sizeof (struct GNUNET_RPS_P2P_PullReplyMessage) > ntohs (msg->header.size))
   {
     GNUNET_break_op (0);
-    GNUNET_CADET_receive_done (channel);
     return GNUNET_SYSERR;
   }
 
-  in_msg = (struct GNUNET_RPS_P2P_PullReplyMessage *) msg;
-  if ((ntohs (msg->size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
-      sizeof (struct GNUNET_PeerIdentity) != ntohl (in_msg->num_peers))
+  if ((ntohs (msg->header.size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
+      sizeof (struct GNUNET_PeerIdentity) != ntohl (msg->num_peers))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
         "message says it sends %" PRIu32 " peers, have space for %lu peers\n",
-        ntohl (in_msg->num_peers),
-        (ntohs (msg->size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
+        ntohl (msg->num_peers),
+        (ntohs (msg->header.size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
             sizeof (struct GNUNET_PeerIdentity));
     GNUNET_break_op (0);
-    GNUNET_CADET_receive_done (channel);
     return GNUNET_SYSERR;
   }
 
-  // Guess simply casting isn't the nicest way...
-  // FIXME wait for cadet to change this function
-  sender = (struct GNUNET_PeerIdentity *)
-      GNUNET_CADET_channel_get_info (channel, GNUNET_CADET_OPTION_PEER);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REPLY (%s)\n", GNUNET_i2s (sender));
-
   if (GNUNET_YES != Peers_check_peer_flag (sender, Peers_PULL_REPLY_PENDING))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
         "Received a pull reply from a peer we didn't request one from!\n");
-    GNUNET_CADET_receive_done (channel);
     GNUNET_break_op (0);
-    return GNUNET_OK;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+/**
+ * Handle PULL REPLY message from another peer.
+ *
+ * @param cls Closure
+ * @param msg The message header
+ */
+static void
+handle_peer_pull_reply (void *cls,
+                        const struct GNUNET_RPS_P2P_PullReplyMessage *msg)
+{
+  struct GNUNET_PeerIdentity *peers;
+  struct GNUNET_PeerIdentity *sender = cls;
+  uint32_t i;
+#ifdef ENABLE_MALICIOUS
+  struct AttackedPeer *tmp_att_peer;
+#endif /* ENABLE_MALICIOUS */
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REPLY (%s)\n", GNUNET_i2s (sender));
 
   #ifdef ENABLE_MALICIOUS
   // We shouldn't even receive pull replies as we're not sending
   if (2 == mal_type)
   {
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (sender));
   }
   #endif /* ENABLE_MALICIOUS */
 
   /* Do actual logic */
-  peers = (struct GNUNET_PeerIdentity *) &in_msg[1];
+  peers = (struct GNUNET_PeerIdentity *) &msg[1];
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "PULL REPLY received, got following %u peers:\n",
-       ntohl (in_msg->num_peers));
+       ntohl (msg->num_peers));
 
-  for (i = 0 ; i < ntohl (in_msg->num_peers) ; i++)
+  for (i = 0; i < ntohl (msg->num_peers); i++)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "%u. %s\n",
@@ -1466,8 +1434,7 @@ handle_peer_pull_reply (void *cls,
   Peers_unset_peer_flag (sender, Peers_PULL_REPLY_PENDING);
   clean_peer (sender);
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (sender));
 }
 
 
@@ -2241,8 +2208,8 @@ client_connect_cb (void *cls,
  */
 static void
 client_disconnect_cb (void *cls,
-                                  struct GNUNET_SERVICE_Client *client,
-                                  void *internal_cls)
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
 {
   struct ClientContext *cli_ctx = internal_cls;
 
@@ -2273,15 +2240,24 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    {&handle_peer_check       , GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_push        , GNUNET_MESSAGE_TYPE_RPS_PP_PUSH,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_pull_request, GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_pull_reply  , GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY, 0},
-    {NULL, 0, 0}
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (peer_check,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (peer_push,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_PUSH,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (peer_pull_request,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_var_size (peer_pull_reply,
+                           GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY,
+                           struct GNUNET_RPS_P2P_PullReplyMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
   };
 
   int size;
@@ -2373,21 +2349,23 @@ run (void *cls,
 
 
   /* Initialise cadet */
-  cadet_handle = GNUNET_CADET_connect (cfg,
-                                       cls,
-                                       &cleanup_destroyed_channel,
-                                       cadet_handlers);
+  cadet_handle = GNUNET_CADET_connect (cfg);
   GNUNET_assert (NULL != cadet_handle);
   GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS,
                       strlen (GNUNET_APPLICATION_PORT_RPS),
                       &port);
-  GNUNET_CADET_open_port (cadet_handle,
-                          &port,
-                          &Peers_handle_inbound_channel, cls);
+  cadet_port = GNUNET_CADET_open_port (cadet_handle,
+                                       &port,
+                                       &Peers_handle_inbound_channel, /* Connect handler */
+                                       NULL, /* cls */
+                                       NULL, /* WindowSize handler */
+                                       cleanup_destroyed_channel, /* Disconnect handler */
+                                       cadet_handlers);
 
 
   peerinfo_handle = GNUNET_PEERINFO_connect (cfg);
-  Peers_initialise (fn_valid_peers, cadet_handle, &own_identity);
+  Peers_initialise (fn_valid_peers, cadet_handle, cleanup_destroyed_channel,
+                    cadet_handlers, &own_identity);
   GNUNET_free (fn_valid_peers);
 
   /* Initialise sampler */
diff --git a/src/rps/gnunet-service-rps_peers.c b/src/rps/gnunet-service-rps_peers.c
index e0b278bd0..58aa84ccf 100644
--- a/src/rps/gnunet-service-rps_peers.c
+++ b/src/rps/gnunet-service-rps_peers.c
@@ -251,6 +251,17 @@ static const struct GNUNET_PeerIdentity *own_identity;
  */
 static struct GNUNET_CADET_Handle *cadet_handle;
 
+/**
+ * @brief Disconnect handler
+ */
+static GNUNET_CADET_DisconnectEventHandler cleanup_destroyed_channel;
+
+/**
+ * @brief cadet handlers
+ */
+static const struct GNUNET_MQ_MessageHandler *cadet_handlers;
+
+
 
 /**
  * @brief Get the #PeerContext associated with a peer
@@ -523,10 +534,13 @@ get_channel (const struct GNUNET_PeerIdentity *peer)
                         &port);
     peer_ctx->send_channel =
       GNUNET_CADET_channel_create (cadet_handle,
-                                   peer_ctx->send_channel_flags, /* context */
+                                   (struct GNUNET_PeerIdentity *) peer, /* context */
                                    peer,
                                    &port,
-                                   GNUNET_CADET_OPTION_RELIABLE);
+                                   GNUNET_CADET_OPTION_RELIABLE,
+                                   NULL, /* WindowSize handler */
+                                   cleanup_destroyed_channel, /* Disconnect handler */
+                                   cadet_handlers);
   }
   GNUNET_assert (NULL != peer_ctx->send_channel);
   return peer_ctx->send_channel;
@@ -552,7 +566,7 @@ get_mq (const struct GNUNET_PeerIdentity *peer)
   if (NULL == peer_ctx->mq)
   {
     (void) get_channel (peer);
-    peer_ctx->mq = GNUNET_CADET_mq_create (peer_ctx->send_channel);
+    peer_ctx->mq = GNUNET_CADET_get_mq (peer_ctx->send_channel);
   }
   return peer_ctx->mq;
 }
@@ -649,9 +663,7 @@ remove_pending_message (struct PendingMessage *pending_msg)
   GNUNET_CONTAINER_DLL_remove (peer_ctx->pending_messages_head,
                                peer_ctx->pending_messages_tail,
                                pending_msg);
-  /* FIXME We are not able to cancel messages as #GNUNET_CADET_mq_create () does
-   * not set a #GNUNET_MQ_CancelImpl */
-  /* GNUNET_MQ_send_cancel (peer_ctx->pending_messages_head->ev); */
+  GNUNET_MQ_send_cancel (peer_ctx->pending_messages_head->ev);
   GNUNET_free (pending_msg);
 }
 
@@ -932,15 +944,21 @@ restore_valid_peers ()
  *
  * @param fn_valid_peers filename of the file used to store valid peer ids
  * @param cadet_h cadet handle
+ * @param disconnect_handler Disconnect handler
+ * @param c_handlers cadet handlers
  * @param own_id own peer identity
  */
 void
 Peers_initialise (char* fn_valid_peers,
                   struct GNUNET_CADET_Handle *cadet_h,
+                  GNUNET_CADET_DisconnectEventHandler disconnect_handler,
+                  const struct GNUNET_MQ_MessageHandler *c_handlers,
                   const struct GNUNET_PeerIdentity *own_id)
 {
   filename_valid_peers = GNUNET_strdup (fn_valid_peers);
   cadet_handle = cadet_h;
+  cleanup_destroyed_channel = disconnect_handler;
+  cadet_handlers = c_handlers;
   own_identity = own_id;
   peer_map = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO);
   valid_peers = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO);
@@ -1279,6 +1297,34 @@ Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags
   return check_channel_flag_set (channel_flags, flags);
 }
 
+/**
+ * @brief Get the flags for the channel in @a role for @a peer.
+ *
+ * @param peer Peer to get the channel flags for.
+ * @param role Role of channel to get flags for
+ *
+ * @return The flags.
+ */
+uint32_t *
+Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer,
+                        enum Peers_ChannelRole role)
+{
+  const struct PeerContext *peer_ctx;
+
+  peer_ctx = get_peer_ctx (peer);
+  if (Peers_CHANNEL_ROLE_SENDING == role)
+  {
+    return peer_ctx->send_channel_flags;
+  }
+  else if (Peers_CHANNEL_ROLE_RECEIVING == role)
+  {
+    return peer_ctx->recv_channel_flags;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
+}
 
 /**
  * @brief Check whether we have information about the given peer.
@@ -1358,8 +1404,6 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer)
  * @param cls The closure
  * @param channel The channel the peer wants to establish
  * @param initiator The peer's peer ID
- * @param port The port the channel is being established over
- * @param options Further options
  *
  * @return initial channel context for the channel
  *         (can be NULL -- that's not an error)
@@ -1367,9 +1411,7 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer)
 void *
 Peers_handle_inbound_channel (void *cls,
                               struct GNUNET_CADET_Channel *channel,
-                              const struct GNUNET_PeerIdentity *initiator,
-                              const struct GNUNET_HashCode *port,
-                              enum GNUNET_CADET_ChannelOption options)
+                              const struct GNUNET_PeerIdentity *initiator)
 {
   struct PeerContext *peer_ctx;
 
@@ -1387,10 +1429,10 @@ Peers_handle_inbound_channel (void *cls,
                       Peers_CHANNEL_ESTABLISHED_TWICE);
     GNUNET_CADET_channel_destroy (channel);
     /* return the channel context */
-    return peer_ctx->recv_channel_flags;
+    return &peer_ctx->peer_id;
   }
   peer_ctx->recv_channel = channel;
-  return peer_ctx->recv_channel_flags;
+  return &peer_ctx->peer_id;
 }
 
 
@@ -1500,16 +1542,11 @@ Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer)
  */
 void
 Peers_cleanup_destroyed_channel (void *cls,
-                                 const struct GNUNET_CADET_Channel *channel,
-                                 void *channel_ctx)
+                                 const struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity *peer = cls;
   struct PeerContext *peer_ctx;
 
-  peer = (struct GNUNET_PeerIdentity *) GNUNET_CADET_channel_get_info (
-      (struct GNUNET_CADET_Channel *) channel, GNUNET_CADET_OPTION_PEER);
-       // FIXME wait for cadet to change this function
-
   if (GNUNET_NO == Peers_check_peer_known (peer))
   {/* We don't want to implicitly create a context that we're about to kill */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -1635,4 +1672,23 @@ Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer,
   return GNUNET_NO;
 }
 
+/**
+ * @brief Get the recv_channel of @a peer.
+ * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming
+ * messages.
+ *
+ * @param peer The peer to get the recv_channel from.
+ *
+ * @return The recv_channel.
+ */
+struct GNUNET_CADET_Channel *
+Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer)
+{
+  struct PeerContext *peer_ctx;
+
+  GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer));
+  peer_ctx = get_peer_ctx (peer);
+  return peer_ctx->recv_channel;
+}
+
 /* end of gnunet-service-rps_peers.c */
diff --git a/src/rps/gnunet-service-rps_peers.h b/src/rps/gnunet-service-rps_peers.h
index bbac86003..15970a7ce 100644
--- a/src/rps/gnunet-service-rps_peers.h
+++ b/src/rps/gnunet-service-rps_peers.h
@@ -117,11 +117,15 @@ typedef int
  *
  * @param fn_valid_peers filename of the file used to store valid peer ids
  * @param cadet_h cadet handle
+ * @param disconnect_handler Disconnect handler
+ * @param c_handlers cadet handlers
  * @param own_id own peer identity
  */
 void
 Peers_initialise (char* fn_valid_peers,
                   struct GNUNET_CADET_Handle *cadet_h,
+                  GNUNET_CADET_DisconnectEventHandler disconnect_handler,
+                  const struct GNUNET_MQ_MessageHandler *c_handlers,
                   const struct GNUNET_PeerIdentity *own_id);
 
 /**
@@ -259,6 +263,18 @@ int
 Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags);
 
 /**
+ * @brief Get the flags for the channel in @a role for @a peer.
+ *
+ * @param peer Peer to get the channel flags for.
+ * @param role Role of channel to get flags for
+ *
+ * @return The flags.
+ */
+uint32_t *
+Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer,
+                        enum Peers_ChannelRole role);
+
+/**
  * @brief Check whether we have information about the given peer.
  *
  * FIXME probably deprecated. Make this the new _online.
@@ -312,8 +328,6 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer);
  * @param cls The closure
  * @param channel The channel the peer wants to establish
  * @param initiator The peer's peer ID
- * @param port The port the channel is being established over
- * @param options Further options
  *
  * @return initial channel context for the channel
  *         (can be NULL -- that's not an error)
@@ -321,9 +335,7 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer);
 void *
 Peers_handle_inbound_channel (void *cls,
                               struct GNUNET_CADET_Channel *channel,
-                              const struct GNUNET_PeerIdentity *initiator,
-                              const struct GNUNET_HashCode *port,
-                              enum GNUNET_CADET_ChannelOption options);
+                              const struct GNUNET_PeerIdentity *initiator);
 
 /**
  * @brief Check whether a sending channel towards the given peer exists
@@ -379,8 +391,7 @@ Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer);
  */
 void
 Peers_cleanup_destroyed_channel (void *cls,
-                                 const struct GNUNET_CADET_Channel *channel,
-                                 void *channel_ctx);
+                                 const struct GNUNET_CADET_Channel *channel);
 
 /**
  * @brief Send a message to another peer.
@@ -411,4 +422,16 @@ int
 Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer,
                           const PeerOp peer_op);
 
+/**
+ * @brief Get the recv_channel of @a peer.
+ * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming
+ * messages.
+ *
+ * @param peer The peer to get the recv_channel from.
+ *
+ * @return The recv_channel.
+ */
+struct GNUNET_CADET_Channel *
+Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer);
+
 /* end of gnunet-service-rps_peers.h */
diff --git a/src/rps/rps_api.c b/src/rps/rps_api.c
index 504f28b92..ccd480086 100644
--- a/src/rps/rps_api.c
+++ b/src/rps/rps_api.c
@@ -389,12 +389,12 @@ GNUNET_RPS_seed_ids (struct GNUNET_RPS_Handle *h,
     n * sizeof (struct GNUNET_PeerIdentity);
   /* The number of peers that fits in one message together with
    * the respective header */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE -
       sizeof (struct GNUNET_RPS_CS_SeedMessage)) /
     sizeof (struct GNUNET_PeerIdentity);
   tmp_peer_pointer = ids;
 
-  while (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  while (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     ev = GNUNET_MQ_msg_extra (msg, num_peers_max * sizeof (struct GNUNET_PeerIdentity),
         GNUNET_MESSAGE_TYPE_RPS_CS_SEED);
@@ -463,12 +463,12 @@ GNUNET_RPS_act_malicious (struct GNUNET_RPS_Handle *h,
     num_peers * sizeof (struct GNUNET_PeerIdentity);
   /* The number of peers that fit in one message together with
    * the respective header */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE -
       sizeof (struct GNUNET_RPS_CS_SeedMessage)) /
     sizeof (struct GNUNET_PeerIdentity);
   tmp_peer_pointer = peer_ids;
 
-  while (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  while (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Too many peers to send at once, sending %" PRIu32 " (all we can so far)\n",
diff --git a/src/rps/test_rps.c b/src/rps/test_rps.c
index 1ce174454..acd3a165d 100644
--- a/src/rps/test_rps.c
+++ b/src/rps/test_rps.c
@@ -546,7 +546,7 @@ seed_peers_big (void *cls)
   unsigned int i;
 
   seed_msg_size = 8; /* sizeof (struct GNUNET_RPS_CS_SeedMessage) */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE - seed_msg_size) /
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE - seed_msg_size) /
     sizeof (struct GNUNET_PeerIdentity);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
       "Peers that fit in one seed msg; %u\n",
@@ -980,7 +980,7 @@ seed_cb (struct RPSPeer *rps_peer)
 static void
 seed_big_cb (struct RPSPeer *rps_peer)
 {
-  // TODO test seeding > GNUNET_SERVER_MAX_MESSAGE_SIZE peers
+  // TODO test seeding > GNUNET_MAX_MESSAGE_SIZE peers
   GNUNET_SCHEDULER_add_delayed (
       GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2),
       seed_peers_big, rps_peer);
@@ -1457,7 +1457,7 @@ main (int argc, char *argv[])
 
   else if (strstr (argv[0], "_seed_big") != NULL)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding (num_peers > GNUNET_SERVER_MAX_MESSAGE_SIZE)\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding (num_peers > GNUNET_MAX_MESSAGE_SIZE)\n");
     num_peers = 1;
     cur_test_run.name = "test-rps-seed-big";
     cur_test_run.main_test = seed_big_cb;
diff --git a/src/rps/test_service_rps_peers.c b/src/rps/test_service_rps_peers.c
index 37ed1974c..9cd677fef 100644
--- a/src/rps/test_service_rps_peers.c
+++ b/src/rps/test_service_rps_peers.c
@@ -59,25 +59,25 @@ check ()
   memset (&own_id, 1, sizeof (own_id));
 
   /* Do nothing */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   Peers_terminate ();
 
 
   /* Create peer */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
   Peers_terminate ();
 
 
   /* Create peer */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
   CHECK (GNUNET_YES == Peers_remove_peer (&k1));
   Peers_terminate ();
 
 
   /* Insertion and Removal */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_NO  == Peers_check_peer_known (&k1));
 
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
diff --git a/src/scalarproduct/.gitignore b/src/scalarproduct/.gitignore
index e244f1108..19909a3f9 100644
--- a/src/scalarproduct/.gitignore
+++ b/src/scalarproduct/.gitignore
@@ -3,3 +3,4 @@ gnunet-scalarproduct
 gnunet-service-scalarproduct-alice
 gnunet-service-scalarproduct-bob
 gnunet-service-scalarproduct-ecc-alice
+test_ecc_scalarproduct
diff --git a/src/scalarproduct/Makefile.am b/src/scalarproduct/Makefile.am
index e2487d9f3..10e04284f 100644
--- a/src/scalarproduct/Makefile.am
+++ b/src/scalarproduct/Makefile.am
@@ -106,7 +106,7 @@ check_PROGRAMS = \
   test_ecc_scalarproduct
 
 if ENABLE_TEST_RUN
-  AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+  AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
   TESTS = $(check_SCRIPTS) $(check_PROGRAMS)
 endif
 
diff --git a/src/scalarproduct/gnunet-scalarproduct.c b/src/scalarproduct/gnunet-scalarproduct.c
index aa894b61d..4a152e09e 100644
--- a/src/scalarproduct/gnunet-scalarproduct.c
+++ b/src/scalarproduct/gnunet-scalarproduct.c
@@ -343,16 +343,32 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "elements", "\"key1,val1;key2,val2;...,keyn,valn;\"",
-      gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
-      1, &GNUNET_GETOPT_set_string, &input_elements},
-    {'p', "peer", "PEERID",
-      gettext_noop ("[Optional] peer to calculate our scalarproduct with. If this parameter is not given, the service will wait for a remote peer to compute the request."),
-      1, &GNUNET_GETOPT_set_string, &input_peer_id},
-    {'k', "key", "TRANSACTION_ID",
-      gettext_noop ("Transaction ID shared with peer."),
-      1, &GNUNET_GETOPT_set_string, &input_session_key},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "elements",
+                                 "\"key1,val1;key2,val2;...,keyn,valn;\"",
+                                 gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
+                                 &input_elements),
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "elements",
+                                 "\"key1,val1;key2,val2;...,keyn,valn;\"",
+                                 gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
+                                 &input_elements),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "peer",
+                                 "PEERID",
+                                 gettext_noop ("[Optional] peer to calculate our scalarproduct with. If this parameter is not given, the service will wait for a remote peer to compute the request."),
+                                 &input_peer_id),
+
+    GNUNET_GETOPT_option_string ('k',
+                                 "key",
+                                 "TRANSACTION_ID",
+                                 gettext_noop ("Transaction ID shared with peer."),
+                                 &input_session_key),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
index 0b7f24e7e..c0b33f8ef 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013-2016 GNUnet e.V.
+     Copyright (C) 2013-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -238,15 +238,10 @@ destroy_service_session (struct AliceServiceSession *s)
   if (GNUNET_YES == s->in_destroy)
     return;
   s->in_destroy = GNUNET_YES;
-  if (NULL != s->cadet_mq)
-  {
-    GNUNET_MQ_destroy (s->cadet_mq);
-    s->cadet_mq = NULL;
-  }
   if (NULL != s->client)
   {
     struct GNUNET_SERVICE_Client *c = s->client;
-    
+
     s->client = NULL;
     GNUNET_SERVICE_client_drop (c);
   }
@@ -265,16 +260,22 @@ destroy_service_session (struct AliceServiceSession *s)
   }
   if (NULL != s->intersection_listen)
   {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Set intersection, listen still up!\n");
     GNUNET_SET_listen_cancel (s->intersection_listen);
     s->intersection_listen = NULL;
   }
   if (NULL != s->intersection_op)
   {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Set intersection, op still ongoing!\n");
     GNUNET_SET_operation_cancel (s->intersection_op);
     s->intersection_op = NULL;
   }
   if (NULL != s->intersection_set)
   {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Set intersection, set still there!\n");
     GNUNET_SET_destroy (s->intersection_set);
     s->intersection_set = NULL;
   }
@@ -401,34 +402,25 @@ transmit_client_response (struct AliceServiceSession *s)
 }
 
 
-
 /**
  * Function called whenever a channel is destroyed.  Should clean up
  * any associated state.
  *
  * It must NOT call #GNUNET_CADET_channel_destroy() on the channel.
  *
- * @param cls closure (set from #GNUNET_CADET_connect())
+ * @param cls the `struct AliceServiceSession`
  * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
  */
 static void
 cb_channel_destruction (void *cls,
-                        const struct GNUNET_CADET_Channel *channel,
-                        void *channel_ctx)
+                        const struct GNUNET_CADET_Channel *channel)
 {
-  struct AliceServiceSession *s = channel_ctx;
+  struct AliceServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer disconnected, terminating session %s with peer %s\n",
               GNUNET_h2s (&s->session_id),
               GNUNET_i2s (&s->peer));
-  if (NULL != s->cadet_mq)
-  {
-    GNUNET_MQ_destroy (s->cadet_mq);
-    s->cadet_mq = NULL;
-  }
   s->channel = NULL;
   if (GNUNET_SCALARPRODUCT_STATUS_ACTIVE == s->status)
   {
@@ -492,56 +484,40 @@ compute_scalar_product (struct AliceServiceSession *session,
  * Handle a response we got from another service we wanted to
  * calculate a scalarproduct with.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (we are done)
+ * @param cls the `struct AliceServiceSession *`
+ * @param msg the actual message
  */
-static int
+static void
 handle_bobs_cryptodata_message (void *cls,
-                                struct GNUNET_CADET_Channel *channel,
-                                void **channel_ctx,
-                                const struct GNUNET_MessageHeader *message)
+                                const struct EccBobCryptodataMessage *msg)
 {
-  struct AliceServiceSession *s = *channel_ctx;
-  const struct EccBobCryptodataMessage *msg;
-  uint32_t contained;
-  uint16_t msg_size;
+  struct AliceServiceSession *s = cls;
   gcry_mpi_point_t prod_g_i_b_i;
   gcry_mpi_point_t prod_h_i_b_i;
+  uint32_t contained;
 
-  if (NULL == s)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg_size = ntohs (message->size);
-  if (sizeof (struct EccBobCryptodataMessage) > msg_size)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct EccBobCryptodataMessage *) message;
   contained = ntohl (msg->contained_element_count);
   if (2 != contained)
   {
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    destroy_service_session (s);
+    return;
   }
   if (NULL == s->sorted_elements)
   {
     /* we're not ready yet, how can Bob be? */
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    destroy_service_session (s);
+    return;
   }
   if (s->total != s->client_received_element_count)
   {
     /* we're not ready yet, how can Bob be? */
     GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    destroy_service_session (s);
+    return;
   }
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received %u crypto values from Bob\n",
               (unsigned int) contained);
@@ -556,7 +532,6 @@ handle_bobs_cryptodata_message (void *cls,
   gcry_mpi_point_release (prod_g_i_b_i);
   gcry_mpi_point_release (prod_h_i_b_i);
   transmit_client_response (s);
-  return GNUNET_OK;
 }
 
 
@@ -636,8 +611,8 @@ send_alices_cryptodata_message (struct AliceServiceSession *s)
   unsigned int todo_count;
 
   s->sorted_elements
-    = GNUNET_malloc (GNUNET_CONTAINER_multihashmap_size (s->intersected_elements) *
-                     sizeof (struct MpiElement));
+    = GNUNET_new_array (GNUNET_CONTAINER_multihashmap_size (s->intersected_elements),
+                        struct MpiElement);
   s->used_element_count = 0;
   GNUNET_CONTAINER_multihashmap_iterate (s->intersected_elements,
                                          &copy_element_cb,
@@ -712,11 +687,13 @@ send_alices_cryptodata_message (struct AliceServiceSession *s)
  *
  * @param cls closure with the `struct AliceServiceSession`
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status what has happened with the set intersection?
  */
 static void
 cb_intersection_element_removed (void *cls,
                                  const struct GNUNET_SET_Element *element,
+                                 uint64_t current_size,
                                  enum GNUNET_SET_Status status)
 {
   struct AliceServiceSession *s = cls;
@@ -800,6 +777,9 @@ cb_intersection_request_alice (void *cls,
 {
   struct AliceServiceSession *s = cls;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received intersection request from %s!\n",
+              GNUNET_i2s (other_peer));
   if (0 != memcmp (other_peer,
                    &s->peer,
                    sizeof (struct GNUNET_PeerIdentity)))
@@ -810,6 +790,7 @@ cb_intersection_request_alice (void *cls,
   s->intersection_op
     = GNUNET_SET_accept (request,
                          GNUNET_SET_RESULT_REMOVED,
+                         (struct GNUNET_SET_Option[]) {{ 0 }},
                          &cb_intersection_element_removed,
                          s);
   if (NULL == s->intersection_op)
@@ -843,6 +824,13 @@ cb_intersection_request_alice (void *cls,
 static void
 client_request_complete_alice (struct AliceServiceSession *s)
 {
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (bobs_cryptodata_message,
+                             GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_BOB_CRYPTODATA,
+                             struct EccBobCryptodataMessage,
+                             s),
+    GNUNET_MQ_handler_end ()
+  };
   struct EccServiceRequestMessage *msg;
   struct GNUNET_MQ_Envelope *e;
   struct GNUNET_HashCode set_sid;
@@ -858,14 +846,17 @@ client_request_complete_alice (struct AliceServiceSession *s)
                                    s,
                                    &s->peer,
                                    &s->session_id,
-                                   GNUNET_CADET_OPTION_RELIABLE);
+                                   GNUNET_CADET_OPTION_RELIABLE,
+                                   NULL,
+                                   &cb_channel_destruction,
+                                   cadet_handlers);
   if (NULL == s->channel)
   {
     s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
     prepare_client_end_notification (s);
     return;
   }
-  s->cadet_mq = GNUNET_CADET_mq_create (s->channel);
+  s->cadet_mq = GNUNET_CADET_get_mq (s->channel);
   s->intersection_listen
     = GNUNET_SET_listen (cfg,
                          GNUNET_SET_OPERATION_INTERSECTION,
@@ -890,7 +881,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
 
 
 /**
- * We're receiving additional set data. Check if 
+ * We're receiving additional set data. Check if
  * @a msg is well-formed.
  *
  * @param cls client identification of the client
@@ -898,7 +889,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
  * @return #GNUNET_OK if @a msg is well-formed
  */
 static int
-check_alice_client_message_multipart (void *cls,                                       
+check_alice_client_message_multipart (void *cls,
                                       const struct ComputationBobCryptodataMultipartMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -928,7 +919,7 @@ check_alice_client_message_multipart (void *cls,
  * @param msg the actual message
  */
 static void
-handle_alice_client_message_multipart (void *cls,                                      
+handle_alice_client_message_multipart (void *cls,
                                        const struct ComputationBobCryptodataMultipartMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -968,8 +959,12 @@ handle_alice_client_message_multipart (void *cls,
   if (s->total != s->client_received_element_count)
   {
     /* more to come */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received client multipart data, waiting for more!\n");
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching computation\n");
   client_request_complete_alice (s);
 }
 
@@ -983,7 +978,7 @@ handle_alice_client_message_multipart (void *cls,
  * @return #GNUNET_OK if @a msg is well-formed
  */
 static int
-check_alice_client_message (void *cls,                       
+check_alice_client_message (void *cls,
                             const struct AliceComputationMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1012,7 +1007,7 @@ check_alice_client_message (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handler for Alice's client request message.
  * We are doing request-initiation to compute a scalar product with a peer.
@@ -1021,7 +1016,7 @@ check_alice_client_message (void *cls,
  * @param msg the actual message
  */
 static void
-handle_alice_client_message (void *cls,                      
+handle_alice_client_message (void *cls,
                              const struct AliceComputationMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1074,8 +1069,12 @@ handle_alice_client_message (void *cls,
   if (s->total != s->client_received_element_count)
   {
     /* wait for multipart msg */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received partial client request, waiting for more!\n");
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching computation\n");
   client_request_complete_alice (s);
 }
 
@@ -1167,13 +1166,6 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &handle_bobs_cryptodata_message,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_BOB_CRYPTODATA,
-      0},
-    { NULL, 0, 0}
-  };
-
   cfg = c;
   edc = GNUNET_CRYPTO_ecc_dlog_prepare (MAX_RESULT,
                                         MAX_RAM);
@@ -1181,10 +1173,7 @@ run (void *cls,
   GNUNET_CRYPTO_ecc_rnd_mpi (edc,
                              &my_privkey,
                              &my_privkey_inv);
-  my_cadet = GNUNET_CADET_connect (cfg,
-                                   NULL,
-                                   &cb_channel_destruction,
-                                   cadet_handlers);
+  my_cadet = GNUNET_CADET_connect (cfg);
   if (NULL == my_cadet)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -1216,7 +1205,7 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_SCALARPRODUCT_CLIENT_MULTIPART_ALICE,
                         struct ComputationBobCryptodataMultipartMessage,
                         NULL),
-  GNUNET_MQ_handler_end ());
+ GNUNET_MQ_handler_end ());
 
 
 /* end of gnunet-service-scalarproduct-ecc_alice.c */
diff --git a/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c b/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
index 06e30706e..0b0333332 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013-2015 GNUnet e.V.
+     Copyright (C) 2013-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -59,12 +59,6 @@ struct MpiElement
 
 
 /**
- * An incoming session from CADET.
- */
-struct CadetIncomingSession;
-
-
-/**
  * A scalarproduct session which tracks an offer for a
  * multiplication service by a local client.
  */
@@ -72,11 +66,6 @@ struct BobServiceSession
 {
 
   /**
-   * (hopefully) unique transaction ID
-   */
-  struct GNUNET_HashCode session_id;
-
-  /**
    * The client this request is related to.
    */
   struct GNUNET_SERVICE_Client *client;
@@ -124,12 +113,6 @@ struct BobServiceSession
   gcry_mpi_point_t prod_h_i_b_i;
 
   /**
-   * Handle for our associated incoming CADET session, or NULL
-   * if we have not gotten one yet.
-   */
-  struct CadetIncomingSession *cadet;
-
-  /**
    * How many elements will be supplied in total from the client.
    */
   uint32_t total;
@@ -166,20 +149,6 @@ struct BobServiceSession
    */
   int in_destroy;
 
-};
-
-
-/**
- * An incoming session from CADET.
- */
-struct CadetIncomingSession
-{
-
-  /**
-   * Associated client session, or NULL.
-   */
-  struct BobServiceSession *s;
-
   /**
    * The CADET channel.
    */
@@ -200,18 +169,6 @@ struct CadetIncomingSession
    */
   struct GNUNET_MQ_Handle *cadet_mq;
 
-  /**
-   * Has this CADET session been added to the map yet?
-   * #GNUNET_YES if so, in which case @e session_id is
-   * the key.
-   */
-  int in_map;
-
-  /**
-   * Are we already in #destroy_cadet_session()?
-   */
-  int in_destroy;
-
 };
 
 
@@ -221,16 +178,6 @@ struct CadetIncomingSession
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
 /**
- * Map of `struct BobServiceSession`, by session keys.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *client_sessions;
-
-/**
- * Map of `struct CadetIncomingSession`, by session keys.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *cadet_sessions;
-
-/**
  * Handle to the CADET service.
  */
 static struct GNUNET_CADET_Handle *my_cadet;
@@ -242,35 +189,6 @@ static struct GNUNET_CRYPTO_EccDlogContext *edc;
 
 
 /**
- * Finds a not terminated client session in the respective map based on
- * session key.
- *
- * @param key the session key we want to search for
- * @return the matching session, or NULL for none
- */
-static struct BobServiceSession *
-find_matching_client_session (const struct GNUNET_HashCode *key)
-{
-  return GNUNET_CONTAINER_multihashmap_get (client_sessions,
-                                            key);
-}
-
-
-/**
- * Finds a CADET session in the respective map based on session key.
- *
- * @param key the session key we want to search for
- * @return the matching session, or NULL for none
- */
-static struct CadetIncomingSession *
-find_matching_cadet_session (const struct GNUNET_HashCode *key)
-{
-  return GNUNET_CONTAINER_multihashmap_get (cadet_sessions,
-                                            key);
-}
-
-
-/**
  * Callback used to free the elements in the map.
  *
  * @param cls NULL
@@ -295,39 +213,20 @@ free_element_cb (void *cls,
  * @param session the session to free elements from
  */
 static void
-destroy_cadet_session (struct CadetIncomingSession *s);
-
-
-/**
- * Destroy session state, we are done with it.
- *
- * @param session the session to free elements from
- */
-static void
 destroy_service_session (struct BobServiceSession *s)
 {
-  struct CadetIncomingSession *in;
   unsigned int i;
 
   if (GNUNET_YES == s->in_destroy)
     return;
   s->in_destroy = GNUNET_YES;
-  if (NULL != (in = s->cadet))
-  {
-    s->cadet = NULL;
-    destroy_cadet_session (in);
-  }
   if (NULL != s->client)
   {
     struct GNUNET_SERVICE_Client *c = s->client;
-    
+
     s->client = NULL;
     GNUNET_SERVICE_client_drop (c);
   }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (client_sessions,
-                                                       &s->session_id,
-                                                       s));
   if (NULL != s->intersected_elements)
   {
     GNUNET_CONTAINER_multihashmap_iterate (s->intersected_elements,
@@ -363,48 +262,17 @@ destroy_service_session (struct BobServiceSession *s)
     gcry_mpi_point_release (s->prod_h_i_b_i);
     s->prod_h_i_b_i = NULL;
   }
-  GNUNET_CADET_close_port (s->port);
-  GNUNET_free (s);
-}
-
-
-/**
- * Destroy incoming CADET session state, we are done with it.
- *
- * @param in the session to free elements from
- */
-static void
-destroy_cadet_session (struct CadetIncomingSession *in)
-{
-  struct BobServiceSession *s;
-
-  if (GNUNET_YES == in->in_destroy)
-    return;
-  in->in_destroy = GNUNET_YES;
-  if (NULL != (s = in->s))
+  if (NULL != s->port)
   {
-    in->s = NULL;
-    destroy_service_session (s);
+    GNUNET_CADET_close_port (s->port);
+    s->port = NULL;
   }
-  if (GNUNET_YES == in->in_map)
+  if (NULL != s->channel)
   {
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (cadet_sessions,
-                                                         &in->session_id,
-                                                         in));
-    in->in_map = GNUNET_NO;
+    GNUNET_CADET_channel_destroy (s->channel);
+    s->channel = NULL;
   }
-  if (NULL != in->cadet_mq)
-  {
-    GNUNET_MQ_destroy (in->cadet_mq);
-    in->cadet_mq = NULL;
-  }
-  if (NULL != in->channel)
-  {
-    GNUNET_CADET_channel_destroy (in->channel);
-    in->channel = NULL;
-  }
-  GNUNET_free (in);
+  GNUNET_free (s);
 }
 
 
@@ -443,38 +311,28 @@ prepare_client_end_notification (struct BobServiceSession *session)
  *
  * It must NOT call #GNUNET_CADET_channel_destroy() on the channel.
  *
- * @param cls closure (set from #GNUNET_CADET_connect())
+ * @param cls the `struct BobServiceSession`
  * @param channel connection to the other end (henceforth invalid)
  * @param channel_ctx place where local state associated
  *                   with the channel is stored
  */
 static void
 cb_channel_destruction (void *cls,
-                        const struct GNUNET_CADET_Channel *channel,
-                        void *channel_ctx)
+                        const struct GNUNET_CADET_Channel *channel)
 {
-  struct CadetIncomingSession *in = channel_ctx;
-  struct BobServiceSession *s;
+  struct BobServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer disconnected, terminating session %s with peer %s\n",
-              GNUNET_h2s (&in->session_id),
-              GNUNET_i2s (&in->peer));
-  if (NULL != in->cadet_mq)
-  {
-    GNUNET_MQ_destroy (in->cadet_mq);
-    in->cadet_mq = NULL;
-  }
-  in->channel = NULL;
-  if (NULL != (s = in->s))
+              GNUNET_h2s (&s->session_id),
+              GNUNET_i2s (&s->peer));
+  s->channel = NULL;
+  if (GNUNET_SCALARPRODUCT_STATUS_ACTIVE == s->status)
   {
-    if (GNUNET_SCALARPRODUCT_STATUS_ACTIVE == s->status)
-    {
-      s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
-      prepare_client_end_notification (s);
-    }
+    s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
+    prepare_client_end_notification (s);
   }
-  destroy_cadet_session (in);
+  destroy_service_session (s);
 }
 
 
@@ -519,7 +377,7 @@ transmit_bobs_cryptodata_message (struct BobServiceSession *s)
   GNUNET_MQ_notify_sent (e,
                          &bob_cadet_done_cb,
                          s);
-  GNUNET_MQ_send (s->cadet->cadet_mq,
+  GNUNET_MQ_send (s->cadet_mq,
                   e);
 }
 
@@ -577,56 +435,79 @@ element_cmp (const void *a,
 
 
 /**
- * Handle a multipart-chunk of a request from another service to
+ * Check a multipart-chunk of a request from another service to
  * calculate a scalarproduct with us.
  *
  * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the @a channel
- * @param message the actual message
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_alices_cryptodata_message (void *cls,
-                                  struct GNUNET_CADET_Channel *channel,
-                                  void **channel_ctx,
-                                  const struct GNUNET_MessageHeader *message)
+check_alices_cryptodata_message (void *cls,
+                                 const struct EccAliceCryptodataMessage *msg)
 {
-  struct CadetIncomingSession *in = *channel_ctx;
-  struct BobServiceSession *s;
-  const struct EccAliceCryptodataMessage *msg;
-  const struct GNUNET_CRYPTO_EccPoint *payload;
+  struct BobServiceSession *s = cls;
   uint32_t contained_elements;
   size_t msg_length;
   uint16_t msize;
   unsigned int max;
-  unsigned int i;
-  const struct MpiElement *b_i;
-  gcry_mpi_point_t tmp;
-  gcry_mpi_point_t g_i;
-  gcry_mpi_point_t h_i;
-  gcry_mpi_point_t g_i_b_i;
-  gcry_mpi_point_t h_i_b_i;
 
-  /* sanity checks */
-  if (NULL == in)
+  msize = ntohs (msg->header.size);
+  if (msize <= sizeof (struct EccAliceCryptodataMessage))
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
-  s = in->s;
-  if (NULL == s)
+  contained_elements = ntohl (msg->contained_element_count);
+  /* Our intersection may still be ongoing, but this is nevertheless
+     an upper bound on the required array size */
+  max = GNUNET_CONTAINER_multihashmap_size (s->intersected_elements);
+  msg_length = sizeof (struct EccAliceCryptodataMessage)
+    + contained_elements * sizeof (struct GNUNET_CRYPTO_EccPoint) * 2;
+  if ( (msize != msg_length) ||
+       (0 == contained_elements) ||
+       (contained_elements > UINT16_MAX) ||
+       (max < contained_elements + s->cadet_received_element_count) )
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a multipart-chunk of a request from another service to
+ * calculate a scalarproduct with us.
+ *
+ * @param cls closure (set from #GNUNET_CADET_connect)
+ * @param msg the actual message
+ */
+static void
+handle_alices_cryptodata_message (void *cls,
+                                  const struct EccAliceCryptodataMessage *msg)
+{
+  struct BobServiceSession *s = cls;
+  const struct GNUNET_CRYPTO_EccPoint *payload;
+  uint32_t contained_elements;
+  unsigned int max;
+  unsigned int i;
+  const struct MpiElement *b_i;
+  gcry_mpi_point_t tmp;
+  gcry_mpi_point_t g_i;
+  gcry_mpi_point_t h_i;
+  gcry_mpi_point_t g_i_b_i;
+  gcry_mpi_point_t h_i_b_i;
+
+  contained_elements = ntohl (msg->contained_element_count);
+  max = GNUNET_CONTAINER_multihashmap_size (s->intersected_elements);
   /* sort our vector for the computation */
   if (NULL == s->sorted_elements)
   {
     s->sorted_elements
-      = GNUNET_malloc (GNUNET_CONTAINER_multihashmap_size (s->intersected_elements) *
-                       sizeof (struct MpiElement));
+      = GNUNET_new_array (GNUNET_CONTAINER_multihashmap_size (s->intersected_elements),
+                          struct MpiElement);
     s->used_element_count = 0;
     GNUNET_CONTAINER_multihashmap_iterate (s->intersected_elements,
                                            &copy_element_cb,
@@ -637,28 +518,6 @@ handle_alices_cryptodata_message (void *cls,
            &element_cmp);
   }
 
-  /* parse message */
-  msize = ntohs (message->size);
-  if (msize <= sizeof (struct EccAliceCryptodataMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct EccAliceCryptodataMessage *) message;
-  contained_elements = ntohl (msg->contained_element_count);
-  /* Our intersection may still be ongoing, but this is nevertheless
-     an upper bound on the required array size */
-  max = GNUNET_CONTAINER_multihashmap_size (s->intersected_elements);
-  msg_length = sizeof (struct EccAliceCryptodataMessage)
-    + contained_elements * sizeof (struct GNUNET_CRYPTO_EccPoint) * 2;
-  if ( (msize != msg_length) ||
-       (0 == contained_elements) ||
-       (contained_elements > UINT16_MAX) ||
-       (max < contained_elements + s->cadet_received_element_count) )
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received %u crypto values from Alice\n",
               (unsigned int) contained_elements);
@@ -711,8 +570,7 @@ handle_alices_cryptodata_message (void *cls,
        CADET response(s) */
     transmit_bobs_cryptodata_message (s);
   }
-  GNUNET_CADET_receive_done (s->cadet->channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (s->channel);
 }
 
 
@@ -722,11 +580,13 @@ handle_alices_cryptodata_message (void *cls,
  *
  * @param cls closure with the `struct BobServiceSession`
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status what has happened with the set intersection?
  */
 static void
 cb_intersection_element_removed (void *cls,
                                  const struct GNUNET_SET_Element *element,
+                                 uint64_t current_size,
                                  enum GNUNET_SET_Status status)
 {
   struct BobServiceSession *s = cls;
@@ -752,7 +612,7 @@ cb_intersection_element_removed (void *cls,
   case GNUNET_SET_STATUS_DONE:
     s->intersection_op = NULL;
     GNUNET_break (NULL == s->intersection_set);
-    GNUNET_CADET_receive_done (s->cadet->channel);
+    GNUNET_CADET_receive_done (s->channel);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Finished intersection, %d items remain\n",
          GNUNET_CONTAINER_multihashmap_size (s->intersected_elements));
@@ -808,10 +668,11 @@ start_intersection (struct BobServiceSession *s)
               (unsigned int) s->total);
 
   s->intersection_op
-    = GNUNET_SET_prepare (&s->cadet->peer,
+    = GNUNET_SET_prepare (&s->peer,
                           &set_sid,
                           NULL,
                           GNUNET_SET_RESULT_REMOVED,
+                          (struct GNUNET_SET_Option[]) {{ 0 }},
                           &cb_intersection_element_removed,
                           s);
   if (GNUNET_OK !=
@@ -832,53 +693,24 @@ start_intersection (struct BobServiceSession *s)
  * Handle a request from Alice to calculate a scalarproduct with us (Bob).
  *
  * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store the `struct CadetIncomingSession *`
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
+ * @param msg the actual message
  */
-static int
+static void
 handle_alices_computation_request (void *cls,
-                                   struct GNUNET_CADET_Channel *channel,
-                                   void **channel_ctx,
-                                   const struct GNUNET_MessageHeader *message)
+                                   const struct EccServiceRequestMessage *msg)
 {
-  struct CadetIncomingSession *in = *channel_ctx;
-  struct BobServiceSession *s;
-  const struct EccServiceRequestMessage *msg;
+  struct BobServiceSession *s = cls;
 
-  msg = (const struct EccServiceRequestMessage *) message;
-  if (GNUNET_YES == in->in_map)
+  s->session_id = msg->session_id; // ??
+  if (s->client_received_element_count < s->total)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL != find_matching_cadet_session (&msg->session_id))
-  {
-    /* not unique, got one like this already */
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  in->session_id = msg->session_id;
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_put (cadet_sessions,
-                                                    &in->session_id,
-                                                    in,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  s = find_matching_client_session (&in->session_id);
-  if (NULL == s)
-  {
-    /* no client waiting for this request, wait for client */
-    return GNUNET_OK;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Alice ready, still waiting for Bob client data!\n");
+    return;
   }
-  GNUNET_assert (NULL == s->cadet);
-  /* pair them up */
-  in->s = s;
-  s->cadet = in;
-  if (s->client_received_element_count == s->total)
-    start_intersection (s);
-  return GNUNET_OK;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Both ready, launching intersection!\n");
+  start_intersection (s);
 }
 
 
@@ -887,30 +719,27 @@ handle_alices_computation_request (void *cls,
  * preliminary initialization, more happens after we get Alice's first
  * message.
  *
- * @param cls closure
+ * @param cls our `struct BobServiceSession`
  * @param channel new handle to the channel
  * @param initiator peer that started the channel
- * @param port unused
- * @param options unused
  * @return session associated with the channel
  */
 static void *
 cb_channel_incoming (void *cls,
                      struct GNUNET_CADET_Channel *channel,
-                     const struct GNUNET_PeerIdentity *initiator,
-                     const struct GNUNET_HashCode *port,
-                     enum GNUNET_CADET_ChannelOption options)
+                     const struct GNUNET_PeerIdentity *initiator)
 {
-  struct CadetIncomingSession *in;
+  struct BobServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New incoming channel from peer %s.\n",
               GNUNET_i2s (initiator));
-  in = GNUNET_new (struct CadetIncomingSession);
-  in->peer = *initiator;
-  in->channel = channel;
-  in->cadet_mq = GNUNET_CADET_mq_create (in->channel);
-  return in;
+  GNUNET_CADET_close_port (s->port);
+  s->port = NULL;
+  s->peer = *initiator;
+  s->channel = channel;
+  s->cadet_mq = GNUNET_CADET_get_mq (s->channel);
+  return s;
 }
 
 
@@ -992,13 +821,19 @@ handle_bob_client_message_multipart (void *cls,
   if (s->total != s->client_received_element_count)
   {
     /* more to come */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Request still partial, waiting for more client data!\n");
     return;
   }
-  if (NULL == s->cadet)
+  if (NULL == s->channel)
   {
     /* no Alice waiting for this request, wait for Alice */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client ready, still waiting for Alice!\n");
     return;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Both ready, launching intersection!\n");
   start_intersection (s);
 }
 
@@ -1037,11 +872,6 @@ check_bob_client_message (void *cls,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
-  if (NULL != find_matching_client_session (&msg->session_key))
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
   return GNUNET_OK;
 }
 
@@ -1059,7 +889,17 @@ handle_bob_client_message (void *cls,
                            const struct BobComputationMessage *msg)
 {
   struct BobServiceSession *s = cls;
-  struct CadetIncomingSession *in;
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (alices_computation_request,
+                             GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_SESSION_INITIALIZATION,
+                             struct EccServiceRequestMessage,
+                             s),
+    GNUNET_MQ_hd_var_size (alices_cryptodata_message,
+                           GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_ALICE_CRYPTODATA,
+                           struct EccAliceCryptodataMessage,
+                           s),
+    GNUNET_MQ_handler_end ()
+  };
   uint32_t contained_count;
   uint32_t total_count;
   const struct GNUNET_SCALARPRODUCT_Element *elements;
@@ -1073,21 +913,6 @@ handle_bob_client_message (void *cls,
   s->total = total_count;
   s->client_received_element_count = contained_count;
   s->session_id = msg->session_key;
-  s->port = GNUNET_CADET_open_port (my_cadet,
-                                    &msg->session_key,
-                                    &cb_channel_incoming,
-                                    s);
-  if (NULL == s->port)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (s->client);
-    return;
-  }
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_put (client_sessions,
-                                                   &s->session_id,
-                                                   s,
-                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   elements = (const struct GNUNET_SCALARPRODUCT_Element *) &msg[1];
   s->intersected_elements
     = GNUNET_CONTAINER_multihashmap_create (s->total,
@@ -1122,22 +947,22 @@ handle_bob_client_message (void *cls,
     s->used_element_count++;
   }
   GNUNET_SERVICE_client_continue (s->client);
-  if (s->total != s->client_received_element_count)
-  {
-    /* multipart msg */
-    return;
-  }
-  in = find_matching_cadet_session (&s->session_id);
-  if (NULL == in)
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received client request, opening port %s!\n",
+              GNUNET_h2s (&msg->session_key));
+  s->port = GNUNET_CADET_open_port (my_cadet,
+                                    &msg->session_key,
+                                    &cb_channel_incoming,
+                                    s,
+                                    NULL,
+                                    &cb_channel_destruction,
+                                    cadet_handlers);
+  if (NULL == s->port)
   {
-    /* nothing yet, wait for Alice */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (s->client);
     return;
   }
-  GNUNET_assert (NULL == in->s);
-  /* pair them up */
-  in->s = s;
-  s->cadet = in;
-  start_intersection (s);
 }
 
 
@@ -1162,10 +987,6 @@ shutdown_task (void *cls)
     GNUNET_CRYPTO_ecc_dlog_release (edc);
     edc = NULL;
   }
-  GNUNET_CONTAINER_multihashmap_destroy (client_sessions);
-  client_sessions = NULL;
-  GNUNET_CONTAINER_multihashmap_destroy (cadet_sessions);
-  cadet_sessions = NULL;
 }
 
 
@@ -1229,28 +1050,11 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &handle_alices_computation_request,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_SESSION_INITIALIZATION,
-      sizeof (struct EccServiceRequestMessage) },
-    { &handle_alices_cryptodata_message,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ECC_ALICE_CRYPTODATA,
-      0},
-    { NULL, 0, 0}
-  };
-
   cfg = c;
   /* We don't really do DLOG, so we can setup with very minimal resources */
   edc = GNUNET_CRYPTO_ecc_dlog_prepare (4 /* max value */,
                                         2 /* RAM */);
-  client_sessions = GNUNET_CONTAINER_multihashmap_create (128,
-                                                          GNUNET_YES);
-  cadet_sessions = GNUNET_CONTAINER_multihashmap_create (128,
-                                                         GNUNET_YES);
-  my_cadet = GNUNET_CADET_connect (cfg, 
-                                   NULL,
-                                   &cb_channel_destruction,
-                                   cadet_handlers);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
@@ -1277,10 +1081,10 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_SCALARPRODUCT_CLIENT_TO_BOB,
                         struct BobComputationMessage,
                         NULL),
-GNUNET_MQ_hd_var_size (bob_client_message_multipart,
-                       GNUNET_MESSAGE_TYPE_SCALARPRODUCT_CLIENT_MULTIPART_BOB,
-                       struct ComputationBobCryptodataMultipartMessage,
-                       NULL),
+ GNUNET_MQ_hd_var_size (bob_client_message_multipart,
+                        GNUNET_MESSAGE_TYPE_SCALARPRODUCT_CLIENT_MULTIPART_BOB,
+                        struct ComputationBobCryptodataMultipartMessage,
+                        NULL),
  GNUNET_MQ_handler_end ());
 
 
diff --git a/src/scalarproduct/gnunet-service-scalarproduct_alice.c b/src/scalarproduct/gnunet-service-scalarproduct_alice.c
index 45d1f4e2c..a55d03900 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct_alice.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct_alice.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013, 2014 GNUnet e.V.
+     Copyright (C) 2013, 2014, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -248,15 +248,10 @@ destroy_service_session (struct AliceServiceSession *s)
   if (GNUNET_YES == s->in_destroy)
     return;
   s->in_destroy = GNUNET_YES;
-  if (NULL != s->cadet_mq)
-  {
-    GNUNET_MQ_destroy (s->cadet_mq);
-    s->cadet_mq = NULL;
-  }
   if (NULL != s->client)
   {
     struct GNUNET_SERVICE_Client *c = s->client;
-    
+
     s->client = NULL;
     GNUNET_SERVICE_client_drop (c);
   }
@@ -428,17 +423,14 @@ transmit_client_response (struct AliceServiceSession *s)
  *
  * It must NOT call #GNUNET_CADET_channel_destroy() on the channel.
  *
- * @param cls closure (set from #GNUNET_CADET_connect())
+ * @param cls our `struct AliceServiceSession`
  * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
  */
 static void
 cb_channel_destruction (void *cls,
-                        const struct GNUNET_CADET_Channel *channel,
-                        void *channel_ctx)
+                        const struct GNUNET_CADET_Channel *channel)
 {
-  struct AliceServiceSession *s = channel_ctx;
+  struct AliceServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer disconnected, terminating session %s with peer %s\n",
@@ -450,11 +442,6 @@ cb_channel_destruction (void *cls,
     s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
     prepare_client_end_notification (s);
   }
-  if (NULL != s->cadet_mq)
-  {
-    GNUNET_MQ_destroy (s->cadet_mq);
-    s->cadet_mq = NULL;
-  }
   s->channel = NULL;
 }
 
@@ -633,42 +620,24 @@ compute_scalar_product (struct AliceServiceSession *session)
 
 
 /**
- * Handle a multipart chunk of a response we got from another service
+ * Check a multipart chunk of a response we got from another service
  * we wanted to calculate a scalarproduct with.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the @a channel
- * @param message the actual message
+ * @param cls the `struct AliceServiceSession`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_bobs_cryptodata_multipart (void *cls,
-                                  struct GNUNET_CADET_Channel *channel,
-                                  void **channel_ctx,
-                                  const struct GNUNET_MessageHeader *message)
+check_bobs_cryptodata_multipart (void *cls,
+                                 const struct BobCryptodataMultipartMessage *msg)
 {
-  struct AliceServiceSession *s = *channel_ctx;
-  const struct BobCryptodataMultipartMessage *msg;
-  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
-  size_t i;
+  struct AliceServiceSession *s = cls;
   uint32_t contained;
   size_t msg_size;
   size_t required_size;
 
-  if (NULL == s)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg_size = ntohs (message->size);
-  if (sizeof (struct BobCryptodataMultipartMessage) > msg_size)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct BobCryptodataMultipartMessage *) message;
+  msg_size = ntohs (msg->header.size);
   contained = ntohl (msg->contained_element_count);
   required_size = sizeof (struct BobCryptodataMultipartMessage)
     + 2 * contained * sizeof (struct GNUNET_CRYPTO_PaillierCiphertext);
@@ -678,7 +647,26 @@ handle_bobs_cryptodata_multipart (void *cls,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
+/**
+ * Handle a multipart chunk of a response we got from another service
+ * we wanted to calculate a scalarproduct with.
+ *
+ * @param cls the `struct AliceServiceSession`
+ * @param msg the actual message
+ */
+static void
+handle_bobs_cryptodata_multipart (void *cls,
+                                  const struct BobCryptodataMultipartMessage *msg)
+{
+  struct AliceServiceSession *s = cls;
+  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
+  size_t i;
+  uint32_t contained;
+
+  contained = ntohl (msg->contained_element_count);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received %u additional crypto values from Bob\n",
               (unsigned int) contained);
@@ -688,60 +676,41 @@ handle_bobs_cryptodata_multipart (void *cls,
   for (i = 0; i < contained; i++)
   {
     GNUNET_memcpy (&s->r[s->cadet_received_element_count + i],
-            &payload[2 * i],
-            sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                   &payload[2 * i],
+                   sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
     GNUNET_memcpy (&s->r_prime[s->cadet_received_element_count + i],
-            &payload[2 * i],
-            sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                   &payload[2 * i],
+                   sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
   }
   s->cadet_received_element_count += contained;
   GNUNET_CADET_receive_done (s->channel);
   if (s->cadet_received_element_count != s->used_element_count)
-    return GNUNET_OK;
+    return; /* more to come */
 
   s->product = compute_scalar_product (s);
   transmit_client_response (s);
-  return GNUNET_OK;
 }
 
 
 /**
- * Handle a response we got from another service we wanted to
+ * Check a response we got from another service we wanted to
  * calculate a scalarproduct with.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
+ * @param cls our `struct AliceServiceSession`
  * @param message the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (we are done)
  */
 static int
-handle_bobs_cryptodata_message (void *cls,
-                                struct GNUNET_CADET_Channel *channel,
-                                void **channel_ctx,
-                                const struct GNUNET_MessageHeader *message)
+check_bobs_cryptodata_message (void *cls,
+                               const struct BobCryptodataMessage *msg)
 {
-  struct AliceServiceSession *s = *channel_ctx;
-  const struct BobCryptodataMessage *msg;
-  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
-  uint32_t i;
+  struct AliceServiceSession *s = cls;
   uint32_t contained;
   uint16_t msg_size;
   size_t required_size;
 
-  if (NULL == s)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg_size = ntohs (message->size);
-  if (sizeof (struct BobCryptodataMessage) > msg_size)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct BobCryptodataMessage *) message;
+  msg_size = ntohs (msg->header.size);
   contained = ntohl (msg->contained_element_count);
   required_size = sizeof (struct BobCryptodataMessage)
     + 2 * contained * sizeof (struct GNUNET_CRYPTO_PaillierCiphertext)
@@ -765,29 +734,51 @@ handle_bobs_cryptodata_message (void *cls,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a response we got from another service we wanted to
+ * calculate a scalarproduct with.
+ *
+ * @param cls our `struct AliceServiceSession`
+ * @param msg the actual message
+ */
+static void
+handle_bobs_cryptodata_message (void *cls,
+                                const struct BobCryptodataMessage *msg)
+{
+  struct AliceServiceSession *s = cls;
+  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
+  uint32_t i;
+  uint32_t contained;
+
+  contained = ntohl (msg->contained_element_count);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received %u crypto values from Bob\n",
               (unsigned int) contained);
-
   payload = (const struct GNUNET_CRYPTO_PaillierCiphertext *) &msg[1];
   GNUNET_memcpy (&s->s,
-          &payload[0],
-          sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                 &payload[0],
+                 sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
   GNUNET_memcpy (&s->s_prime,
-          &payload[1],
-          sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                 &payload[1],
+                 sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
   payload = &payload[2];
 
-  s->r = GNUNET_malloc (sizeof (struct GNUNET_CRYPTO_PaillierCiphertext) * s->used_element_count);
-  s->r_prime = GNUNET_malloc (sizeof (struct GNUNET_CRYPTO_PaillierCiphertext) * s->used_element_count);
+  s->r = GNUNET_new_array (s->used_element_count,
+                           struct GNUNET_CRYPTO_PaillierCiphertext);
+  s->r_prime = GNUNET_new_array (s->used_element_count,
+                                 struct GNUNET_CRYPTO_PaillierCiphertext);
   for (i = 0; i < contained; i++)
   {
     GNUNET_memcpy (&s->r[i],
-            &payload[2 * i],
-            sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                   &payload[2 * i],
+                   sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
     GNUNET_memcpy (&s->r_prime[i],
-            &payload[2 * i + 1],
-            sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
+                   &payload[2 * i + 1],
+                   sizeof (struct GNUNET_CRYPTO_PaillierCiphertext));
   }
   s->cadet_received_element_count = contained;
   GNUNET_CADET_receive_done (s->channel);
@@ -795,12 +786,10 @@ handle_bobs_cryptodata_message (void *cls,
   if (s->cadet_received_element_count != s->used_element_count)
   {
     /* More to come */
-    return GNUNET_OK;
+    return;
   }
-
   s->product = compute_scalar_product (s);
   transmit_client_response (s);
-  return GNUNET_OK;
 }
 
 
@@ -935,11 +924,13 @@ send_alices_cryptodata_message (struct AliceServiceSession *s)
  *
  * @param cls closure with the `struct AliceServiceSession`
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status what has happened with the set intersection?
  */
 static void
 cb_intersection_element_removed (void *cls,
                                  const struct GNUNET_SET_Element *element,
+                                 uint64_t current_size,
                                  enum GNUNET_SET_Status status)
 {
   struct AliceServiceSession *s = cls;
@@ -1033,6 +1024,7 @@ cb_intersection_request_alice (void *cls,
   s->intersection_op
     = GNUNET_SET_accept (request,
                          GNUNET_SET_RESULT_REMOVED,
+                         (struct GNUNET_SET_Option[]) {{ 0 }},
                          &cb_intersection_element_removed,
                          s);
   if (NULL == s->intersection_op)
@@ -1066,6 +1058,17 @@ cb_intersection_request_alice (void *cls,
 static void
 client_request_complete_alice (struct AliceServiceSession *s)
 {
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_var_size (bobs_cryptodata_message,
+                           GNUNET_MESSAGE_TYPE_SCALARPRODUCT_BOB_CRYPTODATA,
+                           struct BobCryptodataMessage,
+                           s),
+    GNUNET_MQ_hd_var_size (bobs_cryptodata_multipart,
+                           GNUNET_MESSAGE_TYPE_SCALARPRODUCT_BOB_CRYPTODATA_MULTIPART,
+                           struct BobCryptodataMultipartMessage,
+                           s),
+    GNUNET_MQ_handler_end ()
+  };
   struct ServiceRequestMessage *msg;
   struct GNUNET_MQ_Envelope *e;
 
@@ -1077,14 +1080,17 @@ client_request_complete_alice (struct AliceServiceSession *s)
                                    s,
                                    &s->peer,
                                    &s->session_id,
-                                   GNUNET_CADET_OPTION_RELIABLE);
+                                   GNUNET_CADET_OPTION_RELIABLE,
+                                   NULL,
+                                   &cb_channel_destruction,
+                                   cadet_handlers);
   if (NULL == s->channel)
   {
     s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
     prepare_client_end_notification (s);
     return;
   }
-  s->cadet_mq = GNUNET_CADET_mq_create (s->channel);
+  s->cadet_mq = GNUNET_CADET_get_mq (s->channel);
   s->intersection_listen
     = GNUNET_SET_listen (cfg,
                          GNUNET_SET_OPERATION_INTERSECTION,
@@ -1110,7 +1116,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
 
 
 /**
- * We're receiving additional set data. Check if 
+ * We're receiving additional set data. Check if
  * @a msg is well-formed.
  *
  * @param cls client identification of the client
@@ -1118,7 +1124,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
  * @return #GNUNET_OK if @a msg is well-formed
  */
 static int
-check_alice_client_message_multipart (void *cls,                                       
+check_alice_client_message_multipart (void *cls,
                                       const struct ComputationBobCryptodataMultipartMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1148,7 +1154,7 @@ check_alice_client_message_multipart (void *cls,
  * @param msg the actual message
  */
 static void
-handle_alice_client_message_multipart (void *cls,                                      
+handle_alice_client_message_multipart (void *cls,
                                        const struct ComputationBobCryptodataMultipartMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1203,7 +1209,7 @@ handle_alice_client_message_multipart (void *cls,
  * @return #GNUNET_OK if @a msg is well-formed
  */
 static int
-check_alice_client_message (void *cls,                       
+check_alice_client_message (void *cls,
                             const struct AliceComputationMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1232,7 +1238,7 @@ check_alice_client_message (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handler for Alice's client request message.
  * We are doing request-initiation to compute a scalar product with a peer.
@@ -1241,7 +1247,7 @@ check_alice_client_message (void *cls,
  * @param msg the actual message
  */
 static void
-handle_alice_client_message (void *cls,                      
+handle_alice_client_message (void *cls,
                              const struct AliceComputationMessage *msg)
 {
   struct AliceServiceSession *s = cls;
@@ -1382,16 +1388,6 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &handle_bobs_cryptodata_message,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_BOB_CRYPTODATA,
-      0},
-    { &handle_bobs_cryptodata_multipart,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_BOB_CRYPTODATA_MULTIPART,
-      0},
-    { NULL, 0, 0}
-  };
-
   cfg = c;
   /*
     offset has to be sufficiently small to allow computation of:
@@ -1400,13 +1396,9 @@ run (void *cls,
   my_offset = gcry_mpi_new (GNUNET_CRYPTO_PAILLIER_BITS / 3);
   gcry_mpi_set_bit (my_offset,
                     GNUNET_CRYPTO_PAILLIER_BITS / 3);
-
   GNUNET_CRYPTO_paillier_create (&my_pubkey,
                                  &my_privkey);
-  my_cadet = GNUNET_CADET_connect (cfg,
-                                   NULL,
-                                   &cb_channel_destruction,
-                                   cadet_handlers);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
@@ -1437,7 +1429,7 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_SCALARPRODUCT_CLIENT_MULTIPART_ALICE,
                         struct ComputationBobCryptodataMultipartMessage,
                         NULL),
-  GNUNET_MQ_handler_end ());
+ GNUNET_MQ_handler_end ());
 
 
 /* end of gnunet-service-scalarproduct_alice.c */
diff --git a/src/scalarproduct/gnunet-service-scalarproduct_bob.c b/src/scalarproduct/gnunet-service-scalarproduct_bob.c
index 56cb91fe5..0c38cb426 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct_bob.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct_bob.c
@@ -59,12 +59,6 @@ struct MpiElement
 
 
 /**
- * An incoming session from CADET.
- */
-struct CadetIncomingSession;
-
-
-/**
  * A scalarproduct session which tracks an offer for a
  * multiplication service by a local client.
  */
@@ -187,20 +181,6 @@ struct BobServiceSession
    */
   int in_destroy;
 
-};
-
-
-/**
- * An incoming session from CADET.
- */
-struct CadetIncomingSession
-{
-
-  /**
-   * Associated client session, or NULL.
-   */
-  struct BobServiceSession *s;
-
   /**
    * The CADET channel.
    */
@@ -212,11 +192,6 @@ struct CadetIncomingSession
   struct GNUNET_PeerIdentity peer;
 
   /**
-   * (hopefully) unique transaction ID
-   */
-  struct GNUNET_HashCode session_id;
-
-  /**
    * Public key of the remote service.
    */
   struct GNUNET_CRYPTO_PaillierPublicKey remote_pubkey;
@@ -226,21 +201,10 @@ struct CadetIncomingSession
    */
   struct GNUNET_MQ_Handle *cadet_mq;
 
-  /**
-   * Has this CADET session been added to the map yet?
-   * #GNUNET_YES if so, in which case @e session_id is
-   * the key.
-   */
-  int in_map;
-
-  /**
-   * Are we already in #destroy_cadet_session()?
-   */
-  int in_destroy;
-
 };
 
 
+
 /**
  * GNUnet configuration handle
  */
@@ -262,51 +226,11 @@ static struct GNUNET_CRYPTO_PaillierPrivateKey my_privkey;
 static gcry_mpi_t my_offset;
 
 /**
- * Map of `struct BobServiceSession`, by session keys.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *client_sessions;
-
-/**
- * Map of `struct CadetIncomingSession`, by session keys.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *cadet_sessions;
-
-/**
  * Handle to the CADET service.
  */
 static struct GNUNET_CADET_Handle *my_cadet;
 
 
-
-/**
- * Finds a not terminated client session in the respective map based on
- * session key.
- *
- * @param key the session key we want to search for
- * @return the matching session, or NULL for none
- */
-static struct BobServiceSession *
-find_matching_client_session (const struct GNUNET_HashCode *key)
-{
-  return GNUNET_CONTAINER_multihashmap_get (client_sessions,
-                                            key);
-}
-
-
-/**
- * Finds a CADET session in the respective map based on session key.
- *
- * @param key the session key we want to search for
- * @return the matching session, or NULL for none
- */
-static struct CadetIncomingSession *
-find_matching_cadet_session (const struct GNUNET_HashCode *key)
-{
-  return GNUNET_CONTAINER_multihashmap_get (cadet_sessions,
-                                            key);
-}
-
-
 /**
  * Callback used to free the elements in the map.
  *
@@ -332,39 +256,20 @@ free_element_cb (void *cls,
  * @param session the session to free elements from
  */
 static void
-destroy_cadet_session (struct CadetIncomingSession *s);
-
-
-/**
- * Destroy session state, we are done with it.
- *
- * @param session the session to free elements from
- */
-static void
 destroy_service_session (struct BobServiceSession *s)
 {
-  struct CadetIncomingSession *in;
   unsigned int i;
 
   if (GNUNET_YES == s->in_destroy)
     return;
   s->in_destroy = GNUNET_YES;
-  if (NULL != (in = s->cadet))
-  {
-    s->cadet = NULL;
-    destroy_cadet_session (in);
-  }
   if (NULL != s->client)
   {
     struct GNUNET_SERVICE_Client *c = s->client;
-    
+
     s->client = NULL;
     GNUNET_SERVICE_client_drop (c);
   }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (client_sessions,
-                                                       &s->session_id,
-                                                       s));
   if (NULL != s->intersected_elements)
   {
     GNUNET_CONTAINER_multihashmap_iterate (s->intersected_elements,
@@ -405,48 +310,17 @@ destroy_service_session (struct BobServiceSession *s)
     GNUNET_free (s->r_prime);
     s->r_prime = NULL;
   }
-  GNUNET_CADET_close_port (s->port);
-  GNUNET_free (s);
-}
-
-
-/**
- * Destroy incoming CADET session state, we are done with it.
- *
- * @param in the session to free elements from
- */
-static void
-destroy_cadet_session (struct CadetIncomingSession *in)
-{
-  struct BobServiceSession *s;
-
-  if (GNUNET_YES == in->in_destroy)
-    return;
-  in->in_destroy = GNUNET_YES;
-  if (NULL != (s = in->s))
-  {
-    in->s = NULL;
-    destroy_service_session (s);
-  }
-  if (GNUNET_YES == in->in_map)
+  if (NULL != s->port)
   {
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (cadet_sessions,
-                                                         &in->session_id,
-                                                         in));
-    in->in_map = GNUNET_NO;
-  }
-  if (NULL != in->cadet_mq)
-  {
-    GNUNET_MQ_destroy (in->cadet_mq);
-    in->cadet_mq = NULL;
+    GNUNET_CADET_close_port (s->port);
+    s->port = NULL;
   }
-  if (NULL != in->channel)
+  if (NULL != s->channel)
   {
-    GNUNET_CADET_channel_destroy (in->channel);
-    in->channel = NULL;
+    GNUNET_CADET_channel_destroy (s->channel);
+    s->channel = NULL;
   }
-  GNUNET_free (in);
+  GNUNET_free (s);
 }
 
 
@@ -485,38 +359,26 @@ prepare_client_end_notification (struct BobServiceSession *session)
  *
  * It must NOT call #GNUNET_CADET_channel_destroy() on the channel.
  *
- * @param cls closure (set from #GNUNET_CADET_connect())
+ * @param cls the `struct BobServiceSession`
  * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
  */
 static void
 cb_channel_destruction (void *cls,
-                        const struct GNUNET_CADET_Channel *channel,
-                        void *channel_ctx)
+                        const struct GNUNET_CADET_Channel *channel)
 {
-  struct CadetIncomingSession *in = channel_ctx;
-  struct BobServiceSession *s;
+  struct BobServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer disconnected, terminating session %s with peer %s\n",
-              GNUNET_h2s (&in->session_id),
-              GNUNET_i2s (&in->peer));
-  if (NULL != (s = in->s))
-  {
-    if (GNUNET_SCALARPRODUCT_STATUS_ACTIVE == s->status)
-    {
-      s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
-      prepare_client_end_notification (s);
-    }
-  }
-  if (NULL != in->cadet_mq)
+              GNUNET_h2s (&s->session_id),
+              GNUNET_i2s (&s->peer));
+  if (GNUNET_SCALARPRODUCT_STATUS_ACTIVE == s->status)
   {
-    GNUNET_MQ_destroy (in->cadet_mq);
-    in->cadet_mq = NULL;
+    s->status = GNUNET_SCALARPRODUCT_STATUS_FAILURE;
+    prepare_client_end_notification (s);
   }
-  in->channel = NULL;
-  destroy_cadet_session (in);
+  s->channel = NULL;
+  destroy_service_session (s);
 }
 
 
@@ -585,7 +447,7 @@ transmit_bobs_cryptodata_message_multipart (struct BobServiceSession *s)
       GNUNET_MQ_notify_sent (e,
                              &bob_cadet_done_cb,
                              s);
-    GNUNET_MQ_send (s->cadet->cadet_mq,
+    GNUNET_MQ_send (s->cadet_mq,
                     e);
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -653,7 +515,7 @@ transmit_bobs_cryptodata_message (struct BobServiceSession *s)
     GNUNET_MQ_notify_sent (e,
                            &bob_cadet_done_cb,
                            s);
-  GNUNET_MQ_send (s->cadet->cadet_mq,
+  GNUNET_MQ_send (s->cadet_mq,
                   e);
   transmit_bobs_cryptodata_message_multipart (s);
 }
@@ -752,14 +614,14 @@ compute_service_response (struct BobServiceSession *session)
     gcry_mpi_sub (tmp, my_offset, rand[p[i]]);
     gcry_mpi_sub (tmp, tmp, b[p[i]].value);
     GNUNET_assert (2 ==
-                   GNUNET_CRYPTO_paillier_encrypt (&session->cadet->remote_pubkey,
+                   GNUNET_CRYPTO_paillier_encrypt (&session->remote_pubkey,
                                                    tmp,
                                                    2,
                                                    &r[i]));
 
     // E(S - r_pi - b_pi) * E(S + a_pi) ==  E(2*S + a - r - b)
     if (GNUNET_OK !=
-        GNUNET_CRYPTO_paillier_hom_add (&session->cadet->remote_pubkey,
+        GNUNET_CRYPTO_paillier_hom_add (&session->remote_pubkey,
                                         &r[i],
                                         &a[p[i]],
                                         &r[i]))
@@ -775,14 +637,14 @@ compute_service_response (struct BobServiceSession *session)
     // E(S - r_qi)
     gcry_mpi_sub (tmp, my_offset, rand[q[i]]);
     GNUNET_assert (2 ==
-                   GNUNET_CRYPTO_paillier_encrypt (&session->cadet->remote_pubkey,
+                   GNUNET_CRYPTO_paillier_encrypt (&session->remote_pubkey,
                                                    tmp,
                                                    2,
                                                    &r_prime[i]));
 
     // E(S - r_qi) * E(S + a_qi) == E(2*S + a_qi - r_qi)
     if (GNUNET_OK !=
-        GNUNET_CRYPTO_paillier_hom_add (&session->cadet->remote_pubkey,
+        GNUNET_CRYPTO_paillier_hom_add (&session->remote_pubkey,
                                         &r_prime[i],
                                         &a[q[i]],
                                         &r_prime[i]))
@@ -796,7 +658,7 @@ compute_service_response (struct BobServiceSession *session)
   // Calculate S' =  E(SUM( r_i^2 ))
   tmp = compute_square_sum (rand, count);
   GNUNET_assert (1 ==
-                 GNUNET_CRYPTO_paillier_encrypt (&session->cadet->remote_pubkey,
+                 GNUNET_CRYPTO_paillier_encrypt (&session->remote_pubkey,
                                                  tmp,
                                                  1,
                                                  &session->s_prime));
@@ -807,7 +669,7 @@ compute_service_response (struct BobServiceSession *session)
     gcry_mpi_add (rand[i], rand[i], b[i].value);
   tmp = compute_square_sum (rand, count);
   GNUNET_assert (1 ==
-                 GNUNET_CRYPTO_paillier_encrypt (&session->cadet->remote_pubkey,
+                 GNUNET_CRYPTO_paillier_encrypt (&session->remote_pubkey,
                                                  tmp,
                                                  1,
                                                  &session->s));
@@ -919,8 +781,8 @@ transmit_cryptographic_reply (struct BobServiceSession *s)
   if (GNUNET_OK !=
       compute_service_response (s))
   {
-    channel = s->cadet->channel;
-    s->cadet->channel = NULL;
+    channel = s->channel;
+    s->channel = NULL;
     GNUNET_CADET_channel_destroy (channel);
     return;
   }
@@ -929,49 +791,25 @@ transmit_cryptographic_reply (struct BobServiceSession *s)
 
 
 /**
- * Handle a multipart-chunk of a request from another service to
+ * Check a multipart-chunk of a request from another service to
  * calculate a scalarproduct with us.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the @a channel
- * @param message the actual message
+ * @param cls the `struct BobServiceSession *`
+ * @param msg the actual message
  * @return #GNUNET_OK to keep the connection open,
  *         #GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_alices_cryptodata_message (void *cls,
-                                  struct GNUNET_CADET_Channel *channel,
-                                  void **channel_ctx,
-                                  const struct GNUNET_MessageHeader *message)
+check_alices_cryptodata_message (void *cls,
+                                 const struct AliceCryptodataMessage *msg)
 {
-  struct CadetIncomingSession *in = *channel_ctx;
-  struct BobServiceSession *s;
-  const struct AliceCryptodataMessage *msg;
-  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
+  struct BobServiceSession *s = cls;
   uint32_t contained_elements;
   size_t msg_length;
   uint16_t msize;
   unsigned int max;
 
-  if (NULL == in)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  s = in->s;
-  if (NULL == s)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msize = ntohs (message->size);
-  if (msize <= sizeof (struct AliceCryptodataMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  msg = (const struct AliceCryptodataMessage *) message;
+  msize = ntohs (msg->header.size);
   contained_elements = ntohl (msg->contained_element_count);
   /* Our intersection may still be ongoing, but this is nevertheless
      an upper bound on the required array size */
@@ -986,14 +824,38 @@ handle_alices_cryptodata_message (void *cls,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a multipart-chunk of a request from another service to
+ * calculate a scalarproduct with us.
+ *
+ * @param cls the `struct BobServiceSession *`
+ * @param msg the actual message
+ */
+static void
+handle_alices_cryptodata_message (void *cls,
+                                  const struct AliceCryptodataMessage *msg)
+{
+  struct BobServiceSession *s = cls;
+  const struct GNUNET_CRYPTO_PaillierCiphertext *payload;
+  uint32_t contained_elements;
+  unsigned int max;
+
+  contained_elements = ntohl (msg->contained_element_count);
+  /* Our intersection may still be ongoing, but this is nevertheless
+     an upper bound on the required array size */
+  max = GNUNET_CONTAINER_multihashmap_size (s->intersected_elements);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received %u crypto values from Alice\n",
               (unsigned int) contained_elements);
 
   payload = (const struct GNUNET_CRYPTO_PaillierCiphertext *) &msg[1];
   if (NULL == s->e_a)
-    s->e_a = GNUNET_malloc (sizeof (struct GNUNET_CRYPTO_PaillierCiphertext) *
-                            max);
+    s->e_a = GNUNET_new_array (max,
+                               struct GNUNET_CRYPTO_PaillierCiphertext);
   GNUNET_memcpy (&s->e_a[s->cadet_received_element_count],
                  payload,
                  sizeof (struct GNUNET_CRYPTO_PaillierCiphertext) * contained_elements);
@@ -1007,8 +869,7 @@ handle_alices_cryptodata_message (void *cls,
        CADET response(s) */
     transmit_cryptographic_reply (s);
   }
-  GNUNET_CADET_receive_done (s->cadet->channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (s->channel);
 }
 
 
@@ -1018,11 +879,13 @@ handle_alices_cryptodata_message (void *cls,
  *
  * @param cls closure with the `struct BobServiceSession`
  * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
+ * @param current_size current set size
  * @param status what has happened with the set intersection?
  */
 static void
 cb_intersection_element_removed (void *cls,
                                  const struct GNUNET_SET_Element *element,
+                                 uint64_t current_size,
                                  enum GNUNET_SET_Status status)
 {
   struct BobServiceSession *s = cls;
@@ -1048,7 +911,7 @@ cb_intersection_element_removed (void *cls,
   case GNUNET_SET_STATUS_DONE:
     s->intersection_op = NULL;
     GNUNET_break (NULL == s->intersection_set);
-    GNUNET_CADET_receive_done (s->cadet->channel);
+    GNUNET_CADET_receive_done (s->channel);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Finished intersection, %d items remain\n",
          GNUNET_CONTAINER_multihashmap_size (s->intersected_elements));
@@ -1099,10 +962,11 @@ start_intersection (struct BobServiceSession *s)
               (unsigned int) s->total);
 
   s->intersection_op
-    = GNUNET_SET_prepare (&s->cadet->peer,
+    = GNUNET_SET_prepare (&s->peer,
                           &s->session_id,
                           NULL,
                           GNUNET_SET_RESULT_REMOVED,
+                          (struct GNUNET_SET_Option[]) {{ 0 }},
                           &cb_intersection_element_removed,
                           s);
   if (GNUNET_OK !=
@@ -1122,55 +986,19 @@ start_intersection (struct BobServiceSession *s)
 /**
  * Handle a request from Alice to calculate a scalarproduct with us (Bob).
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store the `struct CadetIncomingSession *`
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
+ * @param cls the `struct BobServiceSession *`
+ * @param msg the actual message
  */
-static int
+static void
 handle_alices_computation_request (void *cls,
-                                   struct GNUNET_CADET_Channel *channel,
-                                   void **channel_ctx,
-                                   const struct GNUNET_MessageHeader *message)
+                                   const struct ServiceRequestMessage *msg)
 {
-  struct CadetIncomingSession *in = *channel_ctx;
-  struct BobServiceSession *s;
-  const struct ServiceRequestMessage *msg;
+  struct BobServiceSession *s = cls;
 
-  msg = (const struct ServiceRequestMessage *) message;
-  if (GNUNET_YES == in->in_map)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL != find_matching_cadet_session (&msg->session_id))
-  {
-    /* not unique, got one like this already */
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  in->session_id = msg->session_id;
-  in->remote_pubkey = msg->public_key;
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_put (cadet_sessions,
-                                                    &in->session_id,
-                                                    in,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  s = find_matching_client_session (&in->session_id);
-  if (NULL == s)
-  {
-    /* no client waiting for this request, wait for client */
-    return GNUNET_OK;
-  }
-  GNUNET_assert (NULL == s->cadet);
-  /* pair them up */
-  in->s = s;
-  s->cadet = in;
+  s->session_id = msg->session_id; // ??
+  s->remote_pubkey = msg->public_key;
   if (s->client_received_element_count == s->total)
     start_intersection (s);
-  return GNUNET_OK;
 }
 
 
@@ -1182,31 +1010,27 @@ handle_alices_computation_request (void *cls,
  * @param cls closure with the `struct BobServiceSession`
  * @param channel new handle to the channel
  * @param initiator peer that started the channel
- * @param port unused
- * @param options unused
  * @return session associated with the channel
  */
 static void *
 cb_channel_incoming (void *cls,
                      struct GNUNET_CADET_Channel *channel,
-                     const struct GNUNET_PeerIdentity *initiator,
-                     const struct GNUNET_HashCode *port,
-                     enum GNUNET_CADET_ChannelOption options)
+                     const struct GNUNET_PeerIdentity *initiator)
 {
-  struct CadetIncomingSession *in;
+  struct BobServiceSession *s = cls;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New incoming channel from peer %s.\n",
               GNUNET_i2s (initiator));
-  in = GNUNET_new (struct CadetIncomingSession);
-  in->peer = *initiator;
-  in->channel = channel;
-  in->cadet_mq = GNUNET_CADET_mq_create (in->channel);
-  return in;
+  GNUNET_CADET_close_port (s->port);
+  s->port = NULL;
+  s->channel = channel;
+  s->peer = *initiator;
+  s->cadet_mq = GNUNET_CADET_get_mq (s->channel);
+  return s;
 }
 
 
-
 /**
  * We're receiving additional set data. Check it is well-formed.
  *
@@ -1287,7 +1111,7 @@ handle_bob_client_message_multipart (void *cls,
     /* more to come */
     return;
   }
-  if (NULL == s->cadet)
+  if (NULL == s->channel)
   {
     /* no Alice waiting for this request, wait for Alice */
     return;
@@ -1330,11 +1154,6 @@ check_bob_client_message (void *cls,
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
-  if (NULL != find_matching_client_session (&msg->session_key))
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
   return GNUNET_OK;
 }
 
@@ -1352,7 +1171,17 @@ handle_bob_client_message (void *cls,
                            const struct BobComputationMessage *msg)
 {
   struct BobServiceSession *s = cls;
-  struct CadetIncomingSession *in;
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (alices_computation_request,
+                             GNUNET_MESSAGE_TYPE_SCALARPRODUCT_SESSION_INITIALIZATION,
+                             struct ServiceRequestMessage,
+                             NULL),
+    GNUNET_MQ_hd_var_size (alices_cryptodata_message,
+                           GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ALICE_CRYPTODATA,
+                           struct AliceCryptodataMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
   uint32_t contained_count;
   uint32_t total_count;
   const struct GNUNET_SCALARPRODUCT_Element *elements;
@@ -1366,21 +1195,6 @@ handle_bob_client_message (void *cls,
   s->total = total_count;
   s->client_received_element_count = contained_count;
   s->session_id = msg->session_key;
-  s->port = GNUNET_CADET_open_port (my_cadet,
-                                    &msg->session_key,
-                                    &cb_channel_incoming,
-                                    s);
-  if (NULL == s->port)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (s->client);
-    return;
-  }
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_put (client_sessions,
-                                                   &s->session_id,
-                                                   s,
-                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   elements = (const struct GNUNET_SCALARPRODUCT_Element *) &msg[1];
   s->intersected_elements
     = GNUNET_CONTAINER_multihashmap_create (s->total,
@@ -1415,22 +1229,20 @@ handle_bob_client_message (void *cls,
     s->used_element_count++;
   }
   GNUNET_SERVICE_client_continue (s->client);
-  if (s->total != s->client_received_element_count)
-  {
-    /* multipart msg */
-    return;
-  }
-  in = find_matching_cadet_session (&s->session_id);
-  if (NULL == in)
+  /* We're ready, open the port */
+  s->port = GNUNET_CADET_open_port (my_cadet,
+                                    &msg->session_key,
+                                    &cb_channel_incoming,
+                                    s,
+                                    NULL,
+                                    &cb_channel_destruction,
+                                    cadet_handlers);
+  if (NULL == s->port)
   {
-    /* nothing yet, wait for Alice */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (s->client);
     return;
   }
-  GNUNET_assert (NULL == in->s);
-  /* pair them up */
-  in->s = s;
-  s->cadet = in;
-  start_intersection (s);
 }
 
 
@@ -1450,10 +1262,6 @@ shutdown_task (void *cls)
     GNUNET_CADET_disconnect (my_cadet);
     my_cadet = NULL;
   }
-  GNUNET_CONTAINER_multihashmap_destroy (client_sessions);
-  client_sessions = NULL;
-  GNUNET_CONTAINER_multihashmap_destroy (cadet_sessions);
-  cadet_sessions = NULL;
 }
 
 
@@ -1517,16 +1325,6 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &handle_alices_computation_request,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_SESSION_INITIALIZATION,
-      sizeof (struct ServiceRequestMessage) },
-    { &handle_alices_cryptodata_message,
-      GNUNET_MESSAGE_TYPE_SCALARPRODUCT_ALICE_CRYPTODATA,
-      0},
-    { NULL, 0, 0}
-  };
-
   cfg = c;
   /*
     offset has to be sufficiently small to allow computation of:
@@ -1538,13 +1336,7 @@ run (void *cls,
 
   GNUNET_CRYPTO_paillier_create (&my_pubkey,
                                  &my_privkey);
-  client_sessions = GNUNET_CONTAINER_multihashmap_create (128,
-                                                          GNUNET_YES);
-  cadet_sessions = GNUNET_CONTAINER_multihashmap_create (128,
-                                                         GNUNET_YES);
-  my_cadet = GNUNET_CADET_connect (cfg, NULL,
-                                   &cb_channel_destruction,
-                                   cadet_handlers);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
diff --git a/src/scalarproduct/scalarproduct_api.c b/src/scalarproduct/scalarproduct_api.c
index df9f8d196..05c122e74 100644
--- a/src/scalarproduct/scalarproduct_api.c
+++ b/src/scalarproduct/scalarproduct_api.c
@@ -268,7 +268,7 @@ GNUNET_SCALARPRODUCT_accept_computation (const struct GNUNET_CONFIGURATION_Handl
     GNUNET_free (h);
     return NULL;
   }
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (struct BobComputationMessage))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (struct BobComputationMessage))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   todo = GNUNET_MIN (possible,
                      element_count);
@@ -285,7 +285,7 @@ GNUNET_SCALARPRODUCT_accept_computation (const struct GNUNET_CONFIGURATION_Handl
   element_count_transfered = todo;
   GNUNET_MQ_send (h->mq,
                   env);
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   while (element_count_transfered < element_count)
   {
@@ -426,7 +426,7 @@ GNUNET_SCALARPRODUCT_start_computation (const struct GNUNET_CONFIGURATION_Handle
   h->cfg = cfg;
   h->key = *session_key;
 
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (struct AliceComputationMessage))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (struct AliceComputationMessage))
       / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   todo = GNUNET_MIN (possible,
                      element_count);
@@ -445,7 +445,7 @@ GNUNET_SCALARPRODUCT_start_computation (const struct GNUNET_CONFIGURATION_Handle
   GNUNET_MQ_send (h->mq,
                   env);
   element_count_transfered = todo;
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   while (element_count_transfered < element_count)
   {
diff --git a/src/scalarproduct/test_scalarproduct.conf b/src/scalarproduct/test_scalarproduct.conf
index 3ab47cfeb..73dbadbce 100644
--- a/src/scalarproduct/test_scalarproduct.conf
+++ b/src/scalarproduct/test_scalarproduct.conf
@@ -21,6 +21,3 @@ WAN_QUOTA_OUT = unlimited
 
 [peerinfo]
 USE_INCLUDED_HELLOS = NO
-
-[nat]
-RETURN_LOCAL_ADDRESSES = YES
diff --git a/src/secretsharing/.gitignore b/src/secretsharing/.gitignore
index bb169f0c4..fe9db53a4 100644
--- a/src/secretsharing/.gitignore
+++ b/src/secretsharing/.gitignore
@@ -1,2 +1,3 @@
 gnunet-service-secretsharing
 gnunet-secretsharing-profiler
+test_secretsharing_api
diff --git a/src/secretsharing/Makefile.am b/src/secretsharing/Makefile.am
index 3d0c73c3b..4b9a06c40 100644
--- a/src/secretsharing/Makefile.am
+++ b/src/secretsharing/Makefile.am
@@ -17,9 +17,6 @@ if USE_COVERAGE
 endif
 
 
-bin_PROGRAMS = \
- gnunet-secretsharing-profiler
-
 libexec_PROGRAMS = \
  gnunet-service-secretsharing
 
@@ -56,13 +53,18 @@ libgnunetsecretsharing_la_LIBADD = \
 libgnunetsecretsharing_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS)
 
+if HAVE_TESTING
+bin_PROGRAMS = \
+ gnunet-secretsharing-profiler
+
 check_PROGRAMS = \
  test_secretsharing_api
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
+endif
 
 test_secretsharing_api_SOURCES = \
  test_secretsharing_api.c
@@ -73,4 +75,3 @@ test_secretsharing_api_LDADD = \
 
 EXTRA_DIST = \
   test_secretsharing.conf
-
diff --git a/src/secretsharing/gnunet-secretsharing-profiler.c b/src/secretsharing/gnunet-secretsharing-profiler.c
index 3ff5d7fdd..a83dcd92a 100644
--- a/src/secretsharing/gnunet-secretsharing-profiler.c
+++ b/src/secretsharing/gnunet-secretsharing-profiler.c
@@ -41,7 +41,7 @@ static unsigned int threshold = 2;
 /**
  * Should we try to decrypt a value after the key generation?
  */
-static unsigned int decrypt = GNUNET_NO;
+static int decrypt = GNUNET_NO;
 
 /**
  * When would we like to see the operation finished?
@@ -88,7 +88,7 @@ static unsigned int num_decrypted;
 
 static struct GNUNET_HashCode session_id;
 
-static int verbose;
+static unsigned int verbose;
 
 static struct GNUNET_SECRETSHARING_Plaintext reference_plaintext;
 
@@ -602,26 +602,41 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'n', "num-peers", NULL,
-        gettext_noop ("number of peers in consensus"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers },
-      { 'D', "delay", NULL,
-        gettext_noop ("dkg start delay"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &delay },
-      { 't', "timeout", NULL,
-        gettext_noop ("dkg timeout"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &timeout },
-      { 'k', "threshold", NULL,
-        gettext_noop ("threshold"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &threshold },
-      { 'd', "decrypt", NULL,
-        gettext_noop ("also profile decryption"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &decrypt },
-      { 'V', "verbose", NULL,
-        gettext_noop ("be more verbose (print received values)"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &verbose },
-      GNUNET_GETOPT_OPTION_END
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('n',
+                                   "num-peers",
+                                   NULL,
+                                   gettext_noop ("number of peers in consensus"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_option_relative_time ('D',
+                                            "delay",
+                                            NULL,
+                                            gettext_noop ("dkg start delay"),
+                                            &delay),
+
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "timeout",
+                                            NULL,
+                                            gettext_noop ("dkg timeout"),
+                                            &timeout),
+
+    GNUNET_GETOPT_option_uint ('k',
+                                   "threshold",
+                                   NULL,
+                                   gettext_noop ("threshold"),
+                                   &threshold),
+    
+    GNUNET_GETOPT_option_flag ('d',
+                                  "descrypt",
+                                  gettext_noop ("also profile decryption"),
+                                  &decrypt),
+
+
+    GNUNET_GETOPT_option_verbose (&verbose),
+
+    GNUNET_GETOPT_OPTION_END
   };
   delay = GNUNET_TIME_UNIT_ZERO;
   timeout = GNUNET_TIME_UNIT_MINUTES;
diff --git a/src/secretsharing/gnunet-service-secretsharing.c b/src/secretsharing/gnunet-service-secretsharing.c
index ddbe81576..ccdba12c2 100644
--- a/src/secretsharing/gnunet-service-secretsharing.c
+++ b/src/secretsharing/gnunet-service-secretsharing.c
@@ -109,19 +109,16 @@ struct DecryptPeerInfo
 
 
 /**
+ * State we keep per client.
+ */
+struct ClientState;
+
+
+/**
  * Session to establish a threshold-shared secret.
  */
 struct KeygenSession
 {
-  /**
-   * Keygen sessions are held in a linked list.
-   */
-  struct KeygenSession *next;
-
-  /**
-   * Keygen sessions are held in a linked list.
-   */
-  struct KeygenSession *prev;
 
   /**
    * Current consensus, used for both DKG rounds.
@@ -129,15 +126,9 @@ struct KeygenSession
   struct GNUNET_CONSENSUS_Handle *consensus;
 
   /**
-   * Client that is interested in the result
-   * of this key generation session.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Message queue for 'client'
+   * Which client is this for?
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct ClientState *cs;
 
   /**
    * Randomly generated coefficients of the polynomial for sharing our
@@ -223,15 +214,6 @@ struct KeygenSession
  */
 struct DecryptSession
 {
-  /**
-   * Decrypt sessions are stored in a linked list.
-   */
-  struct DecryptSession *next;
-
-  /**
-   * Decrypt sessions are stored in a linked list.
-   */
-  struct DecryptSession *prev;
 
   /**
    * Handle to the consensus over partial decryptions.
@@ -239,14 +221,9 @@ struct DecryptSession
   struct GNUNET_CONSENSUS_Handle *consensus;
 
   /**
-   * Client connected to us.
+   * Which client is this for?
    */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Message queue for 'client'.
-   */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct ClientState *cs;
 
   /**
    * When should we start communicating for decryption?
@@ -279,24 +256,32 @@ struct DecryptSession
 
 
 /**
- * Decrypt sessions are held in a linked list.
+ * State we keep per client.
  */
-static struct DecryptSession *decrypt_sessions_head;
+struct ClientState
+{
+  /**
+   * Decrypt session of the client, if any.
+   */
+  struct DecryptSession *decrypt_session;
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct DecryptSession *decrypt_sessions_tail;
+  /**
+   * Keygen session of the client, if any.
+   */
+  struct KeygenSession *keygen_session;
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct KeygenSession *keygen_sessions_head;
+  /**
+   * Client this is about.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * MQ to talk to @a client.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+};
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct KeygenSession *keygen_sessions_tail;
 
 /**
  * The ElGamal prime field order as libgcrypt mpi.
@@ -331,11 +316,6 @@ static struct GNUNET_CRYPTO_EddsaPrivateKey *my_peer_private_key;
  */
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
-/**
- * Server for this service.
- */
-static struct GNUNET_SERVER_Handle *srv;
-
 
 /**
  * Get the peer info belonging to a peer identity in a keygen session.
@@ -468,7 +448,8 @@ normalize_peers (struct GNUNET_PeerIdentity *listed,
     n += 1;
   }
 
-  normalized = GNUNET_new_array (n, struct GNUNET_PeerIdentity);
+  normalized = GNUNET_new_array (n,
+                                 struct GNUNET_PeerIdentity);
 
   if (GNUNET_NO == local_peer_in_list)
     normalized[n - 1] = my_peer;
@@ -558,10 +539,13 @@ compute_lagrange_coefficient (gcry_mpi_t coeff, unsigned int j,
 static void
 decrypt_session_destroy (struct DecryptSession *ds)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying decrypt session\n");
-
-  GNUNET_CONTAINER_DLL_remove (decrypt_sessions_head, decrypt_sessions_tail, ds);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "destroying decrypt session\n");
+  if (NULL != ds->cs)
+  {
+    ds->cs->decrypt_session = NULL;
+    ds->cs = NULL;
+  }
   if (NULL != ds->consensus)
   {
     GNUNET_CONSENSUS_destroy (ds->consensus);
@@ -570,8 +554,7 @@ decrypt_session_destroy (struct DecryptSession *ds)
 
   if (NULL != ds->info)
   {
-    unsigned int i;
-    for (i = 0; i < ds->share->num_peers; i++)
+    for (unsigned int i = 0; i < ds->share->num_peers; i++)
     {
       if (NULL != ds->info[i].partial_decryption)
       {
@@ -582,26 +565,12 @@ decrypt_session_destroy (struct DecryptSession *ds)
     GNUNET_free (ds->info);
     ds->info = NULL;
   }
-
   if (NULL != ds->share)
   {
     GNUNET_SECRETSHARING_share_destroy (ds->share);
     ds->share = NULL;
   }
 
-  if (NULL != ds->client_mq)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying decrypt MQ\n");
-    GNUNET_MQ_destroy (ds->client_mq);
-    ds->client_mq = NULL;
-  }
-
-  if (NULL != ds->client)
-  {
-    GNUNET_SERVER_client_disconnect (ds->client);
-    ds->client = NULL;
-  }
-
   GNUNET_free (ds);
 }
 
@@ -630,14 +599,17 @@ keygen_info_destroy (struct KeygenPeerInfo *info)
 static void
 keygen_session_destroy (struct KeygenSession *ks)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying keygen session\n");
-
-  GNUNET_CONTAINER_DLL_remove (keygen_sessions_head, keygen_sessions_tail, ks);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "destroying keygen session\n");
 
+  if (NULL != ks->cs)
+  {
+    ks->cs->keygen_session = NULL;
+    ks->cs = NULL;
+  }
   if (NULL != ks->info)
   {
-    unsigned int i;
-    for (i = 0; i < ks->num_peers; i++)
+    for (unsigned int i = 0; i < ks->num_peers; i++)
       keygen_info_destroy (&ks->info[i]);
     GNUNET_free (ks->info);
     ks->info = NULL;
@@ -651,8 +623,7 @@ keygen_session_destroy (struct KeygenSession *ks)
 
   if (NULL != ks->presecret_polynomial)
   {
-    unsigned int i;
-    for (i = 0; i < ks->threshold; i++)
+    for (unsigned int i = 0; i < ks->threshold; i++)
     {
       GNUNET_assert (NULL != ks->presecret_polynomial[i]);
       gcry_mpi_release (ks->presecret_polynomial[i]);
@@ -661,38 +632,21 @@ keygen_session_destroy (struct KeygenSession *ks)
     GNUNET_free (ks->presecret_polynomial);
     ks->presecret_polynomial = NULL;
   }
-
-  if (NULL != ks->client_mq)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying keygen MQ\n");
-    GNUNET_MQ_destroy (ks->client_mq);
-    ks->client_mq = NULL;
-  }
-
   if (NULL != ks->my_share)
   {
     gcry_mpi_release (ks->my_share);
     ks->my_share = NULL;
   }
-
   if (NULL != ks->public_key)
   {
     gcry_mpi_release (ks->public_key);
     ks->public_key = NULL;
   }
-
   if (NULL != ks->peers)
   {
     GNUNET_free (ks->peers);
     ks->peers = NULL;
   }
-
-  if (NULL != ks->client)
-  {
-    GNUNET_SERVER_client_disconnect (ks->client);
-    ks->client = NULL;
-  }
-
   GNUNET_free (ks);
 }
 
@@ -706,11 +660,7 @@ keygen_session_destroy (struct KeygenSession *ks)
 static void
 cleanup_task (void *cls)
 {
-  while (NULL != decrypt_sessions_head)
-    decrypt_session_destroy (decrypt_sessions_head);
-
-  while (NULL != keygen_sessions_head)
-    keygen_session_destroy (keygen_sessions_head);
+  /* Nothing to do! */
 }
 
 
@@ -727,7 +677,8 @@ generate_presecret_polynomial (struct KeygenSession *ks)
   gcry_mpi_t v;
 
   GNUNET_assert (NULL == ks->presecret_polynomial);
-  ks->presecret_polynomial = GNUNET_new_array (ks->threshold, gcry_mpi_t);
+  ks->presecret_polynomial = GNUNET_new_array (ks->threshold,
+                                               gcry_mpi_t);
   for (i = 0; i < ks->threshold; i++)
   {
     v = ks->presecret_polynomial[i] = gcry_mpi_new (GNUNET_SECRETSHARING_ELGAMAL_BITS);
@@ -768,9 +719,9 @@ keygen_round1_new_element (void *cls,
   if (element->size != sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "keygen commit data with wrong size (%u) in consensus, "
-                " %lu expected\n",
-                element->size, sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData));
+                "keygen commit data with wrong size (%u) in consensus, %u expected\n",
+                (unsigned int) element->size,
+                (unsigned int) sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData));
     return;
   }
 
@@ -855,10 +806,13 @@ keygen_round2_conclude (void *cls)
     if (GNUNET_YES == ks->info[i].round2_valid)
       share->num_peers++;
 
-  share->peers = GNUNET_new_array (share->num_peers, struct GNUNET_PeerIdentity);
+  share->peers = GNUNET_new_array (share->num_peers,
+                                   struct GNUNET_PeerIdentity);
   share->sigmas =
-      GNUNET_new_array (share->num_peers, struct GNUNET_SECRETSHARING_FieldElement);
-  share->original_indices = GNUNET_new_array (share->num_peers, uint16_t);
+      GNUNET_new_array (share->num_peers,
+                        struct GNUNET_SECRETSHARING_FieldElement);
+  share->original_indices = GNUNET_new_array (share->num_peers,
+                                              uint16_t);
 
   /* maybe we're not even in the list of peers? */
   share->my_peer = share->num_peers;
@@ -907,7 +861,8 @@ keygen_round2_conclude (void *cls)
   GNUNET_SECRETSHARING_share_destroy (share);
   share = NULL;
 
-  GNUNET_MQ_send (ks->client_mq, ev);
+  GNUNET_MQ_send (ks->cs->mq,
+                  ev);
 }
 
 
@@ -1429,9 +1384,9 @@ keygen_round2_new_element (void *cls,
   if (element->size != expected_element_size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "keygen round2 data with wrong size (%u) in consensus, "
-                " %lu expected\n",
-                element->size, expected_element_size);
+                "keygen round2 data with wrong size (%u) in consensus, %u expected\n",
+                (unsigned int) element->size,
+                (unsigned int) expected_element_size);
     return;
   }
 
@@ -1669,73 +1624,102 @@ insert_round1_element (struct KeygenSession *ks)
 
 
 /**
+ * Check that @a msg is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+static int
+check_client_keygen (void *cls,
+                     const struct GNUNET_SECRETSHARING_CreateMessage *msg)
+{
+  unsigned int num_peers = ntohs (msg->num_peers);
+
+  if (ntohs (msg->header.size) - sizeof (*msg) !=
+      num_peers * sizeof (struct GNUNET_PeerIdentity))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Functions with this signature are called whenever a message is
  * received.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param msg the actual message
  */
-static void handle_client_keygen (void *cls,
-                                  struct GNUNET_SERVER_Client *client,
-                                  const struct GNUNET_MessageHeader
-                                  *message)
+static void
+handle_client_keygen (void *cls,
+                      const struct GNUNET_SECRETSHARING_CreateMessage *msg)
 {
-  const struct GNUNET_SECRETSHARING_CreateMessage *msg =
-      (const struct GNUNET_SECRETSHARING_CreateMessage *) message;
+  struct ClientState *cs = cls;
   struct KeygenSession *ks;
-  unsigned int i;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "client requested key generation\n");
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "client requested key generation\n");
+  if (NULL != cs->keygen_session)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
   ks = GNUNET_new (struct KeygenSession);
-
-  /* FIXME: check if client already has some session */
-
-  GNUNET_CONTAINER_DLL_insert (keygen_sessions_head, keygen_sessions_tail, ks);
-
-  ks->client = client;
-  ks->client_mq = GNUNET_MQ_queue_for_server_client (client);
-
+  ks->cs = cs;
+  cs->keygen_session = ks;
   ks->deadline = GNUNET_TIME_absolute_ntoh (msg->deadline);
   ks->threshold = ntohs (msg->threshold);
   ks->num_peers = ntohs (msg->num_peers);
 
-  ks->peers = normalize_peers ((struct GNUNET_PeerIdentity *) &msg[1], ks->num_peers,
-                               &ks->num_peers, &ks->local_peer_idx);
+  ks->peers = normalize_peers ((struct GNUNET_PeerIdentity *) &msg[1],
+                               ks->num_peers,
+                               &ks->num_peers,
+                               &ks->local_peer_idx);
 
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "first round of consensus with %u peers\n", ks->num_peers);
-  ks->consensus = GNUNET_CONSENSUS_create (cfg, ks->num_peers, ks->peers, &msg->session_id,
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "first round of consensus with %u peers\n",
+              ks->num_peers);
+  ks->consensus = GNUNET_CONSENSUS_create (cfg,
+                                           ks->num_peers,
+                                           ks->peers,
+                                           &msg->session_id,
                                            GNUNET_TIME_absolute_ntoh (msg->start),
                                            GNUNET_TIME_absolute_ntoh (msg->deadline),
-                                           keygen_round1_new_element, ks);
+                                           keygen_round1_new_element,
+                                           ks);
 
-  ks->info = GNUNET_new_array (ks->num_peers, struct KeygenPeerInfo);
+  ks->info = GNUNET_new_array (ks->num_peers,
+                               struct KeygenPeerInfo);
 
-  for (i = 0; i < ks->num_peers; i++)
+  for (unsigned int i = 0; i < ks->num_peers; i++)
     ks->info[i].peer = ks->peers[i];
 
   GNUNET_CRYPTO_paillier_create (&ks->info[ks->local_peer_idx].paillier_public_key,
                                  &ks->paillier_private_key);
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Generated paillier key pair\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Generated paillier key pair\n",
+              ks->local_peer_idx);
   generate_presecret_polynomial (ks);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Generated presecret polynomial\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Generated presecret polynomial\n",
+              ks->local_peer_idx);
   insert_round1_element (ks);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Concluding for round 1\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Concluding for round 1\n",
+              ks->local_peer_idx);
   GNUNET_CONSENSUS_conclude (ks->consensus,
                              keygen_round1_conclude,
                              ks);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Waiting for round 1 elements ...\n", ks->local_peer_idx);
+  GNUNET_SERVICE_client_continue (cs->client);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Waiting for round 1 elements ...\n",
+              ks->local_peer_idx);
 }
 
 
@@ -1771,20 +1755,24 @@ decrypt_conclude (void *cls)
     if (NULL != ds->info[i].partial_decryption)
       num++;
 
-  indices = GNUNET_malloc (num * sizeof (unsigned int));
+  indices = GNUNET_new_array (num,
+                              unsigned int);
   j = 0;
   for (i = 0; i < ds->share->num_peers; i++)
     if (NULL != ds->info[i].partial_decryption)
       indices[j++] = ds->info[i].original_index;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: decrypt conclude, with %u peers\n",
-              ds->share->my_peer, num);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "P%u: decrypt conclude, with %u peers\n",
+              ds->share->my_peer,
+              num);
 
   gcry_mpi_set_ui (prod, 1);
   for (i = 0; i < num; i++)
   {
 
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: index of %u: %u\n",
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "P%u: index of %u: %u\n",
                 ds->share->my_peer, i, indices[i]);
     compute_lagrange_coefficient (lagrange, indices[i], indices, num);
     // w_i^{\lambda_i}
@@ -1801,7 +1789,8 @@ decrypt_conclude (void *cls)
   ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT_DONE);
   GNUNET_CRYPTO_mpi_print_unsigned (&msg->plaintext, GNUNET_SECRETSHARING_ELGAMAL_BITS / 8, m);
   msg->success = htonl (1);
-  GNUNET_MQ_send (ds->client_mq, ev);
+  GNUNET_MQ_send (ds->cs->mq,
+                  ev);
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "sent decrypt done to client\n");
 
@@ -1945,10 +1934,15 @@ decrypt_new_element (void *cls,
   {
     char *tmp1_str;
     char *tmp2_str;
+
     tmp1_str = mpi_to_str (tmp1);
     tmp2_str = mpi_to_str (tmp2);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "P%u: Received invalid partial decryption from P%ld (eqn 1), expected %s got %s\n",
-                session->share->my_peer, info - session->info, tmp1_str, tmp2_str);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "P%u: Received invalid partial decryption from P%u (eqn 1), expected %s got %s\n",
+                session->share->my_peer,
+                (unsigned int) (info - session->info),
+                tmp1_str,
+                tmp2_str);
     GNUNET_free (tmp1_str);
     GNUNET_free (tmp2_str);
     goto cleanup;
@@ -1963,8 +1957,10 @@ decrypt_new_element (void *cls,
 
   if (0 != gcry_mpi_cmp (tmp1, tmp2))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "P%u: Received invalid partial decryption from P%ld (eqn 2)\n",
-                session->share->my_peer, info - session->info);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "P%u: Received invalid partial decryption from P%u (eqn 2)\n",
+                session->share->my_peer,
+                (unsigned int) (info - session->info));
     goto cleanup;
   }
 
@@ -2100,40 +2096,63 @@ insert_decrypt_element (struct DecryptSession *ds)
 
 
 /**
+ * Check that @a msg is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK (check deferred a bit)
+ */
+static int
+check_client_decrypt (void *cls,
+                      const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg)
+{
+  /* we check later, it's complicated */
+  return GNUNET_OK;
+}
+
+
+/**
  * Functions with this signature are called whenever a message is
  * received.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param msg the actual message
  */
-static void handle_client_decrypt (void *cls,
-                                   struct GNUNET_SERVER_Client *client,
-                                   const struct GNUNET_MessageHeader
-                                   *message)
+static void
+handle_client_decrypt (void *cls,
+                       const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg)
 {
-  const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg =
-      (const void *) message;
+  struct ClientState *cs = cls;
   struct DecryptSession *ds;
   struct GNUNET_HashCode session_id;
-  unsigned int i;
 
+  if (NULL != cs->decrypt_session)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
   ds = GNUNET_new (struct DecryptSession);
-  // FIXME: check if session already exists
-  GNUNET_CONTAINER_DLL_insert (decrypt_sessions_head, decrypt_sessions_tail, ds);
-  ds->client = client;
-  ds->client_mq = GNUNET_MQ_queue_for_server_client (client);
+  cs->decrypt_session = ds;
+  ds->cs = cs;
   ds->start = GNUNET_TIME_absolute_ntoh (msg->start);
   ds->deadline = GNUNET_TIME_absolute_ntoh (msg->deadline);
   ds->ciphertext = msg->ciphertext;
 
-  ds->share = GNUNET_SECRETSHARING_share_read (&msg[1], ntohs (msg->header.size) - sizeof *msg, NULL);
-  // FIXME: probably should be break rather than assert
-  GNUNET_assert (NULL != ds->share);
-
-  // FIXME: this is probably sufficient, but kdf/hash with all values would be nicer ...
-  GNUNET_CRYPTO_hash (&msg->ciphertext, sizeof (struct GNUNET_SECRETSHARING_Ciphertext), &session_id);
+  ds->share = GNUNET_SECRETSHARING_share_read (&msg[1],
+                                               ntohs (msg->header.size) - sizeof (*msg),
+                                               NULL);
+  if (NULL == ds->share)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
 
+  /* FIXME: this is probably sufficient, but kdf/hash with all values would be nicer ... */
+  GNUNET_CRYPTO_hash (&msg->ciphertext,
+                      sizeof (struct GNUNET_SECRETSHARING_Ciphertext),
+                      &session_id);
   ds->consensus = GNUNET_CONSENSUS_create (cfg,
                                            ds->share->num_peers,
                                            ds->share->peers,
@@ -2144,20 +2163,20 @@ static void handle_client_decrypt (void *cls,
                                            ds);
 
 
-  ds->info = GNUNET_new_array (ds->share->num_peers, struct DecryptPeerInfo);
-  for (i = 0; i < ds->share->num_peers; i++)
+  ds->info = GNUNET_new_array (ds->share->num_peers,
+                               struct DecryptPeerInfo);
+  for (unsigned int i = 0; i < ds->share->num_peers; i++)
   {
     ds->info[i].peer = ds->share->peers[i];
     ds->info[i].original_index = ds->share->original_indices[i];
   }
-
   insert_decrypt_element (ds);
-
-  GNUNET_CONSENSUS_conclude (ds->consensus, decrypt_conclude, ds);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "decrypting with %u peers\n",
+  GNUNET_CONSENSUS_conclude (ds->consensus,
+                             decrypt_conclude,
+                             ds);
+  GNUNET_SERVICE_client_continue (cs->client);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "decrypting with %u peers\n",
               ds->share->num_peers);
 }
 
@@ -2174,103 +2193,104 @@ init_crypto_constants (void)
 }
 
 
-static struct KeygenSession *
-keygen_session_get (struct GNUNET_SERVER_Client *client)
-{
-  struct KeygenSession *ks;
-  for (ks = keygen_sessions_head; NULL != ks; ks = ks->next)
-    if (ks->client == client)
-      return ks;
-  return NULL;
-}
-
-static struct DecryptSession *
-decrypt_session_get (struct GNUNET_SERVER_Client *client)
-{
-  struct DecryptSession *ds;
-  for (ds = decrypt_sessions_head; NULL != ds; ds = ds->next)
-    if (ds->client == client)
-      return ds;
-  return NULL;
-}
-
-
-/**
- * Clean up after a client has disconnected
- *
- * @param cls closure, unused
- * @param client the client to clean up after
- */
-static void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  struct KeygenSession *ks;
-  struct DecryptSession *ds;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handling client disconnect\n");
-
-  ks = keygen_session_get (client);
-  if (NULL != ks)
-    keygen_session_destroy (ks);
-
-  ds = decrypt_session_get (client);
-  if (NULL != ds)
-    decrypt_session_destroy (ds);
-}
-
-
 /**
- * Process template requests.
+ * Initialize secretsharing service.
  *
  * @param cls closure
- * @param server the initialized server
  * @param c configuration to use
+ * @param service the initialized service
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {handle_client_keygen, NULL, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_GENERATE, 0},
-    {handle_client_decrypt, NULL, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT, 0},
-    {NULL, NULL, 0, 0}
-  };
   cfg = c;
-  srv = server;
   my_peer_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
   if (NULL == my_peer_private_key)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not access host private key\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "could not access host private key\n");
     GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
   init_crypto_constants ();
-  if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &my_peer))
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_get_peer_identity (cfg,
+                                       &my_peer))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "could not retrieve host identity\n");
     GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &handle_client_disconnect, NULL);
   GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
                                  NULL);
 }
 
 
 /**
- * The main function for the template service.
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct ClientState *cs = GNUNET_new (struct ClientState);;
+
+  cs->client = c;
+  cs->mq = mq;
+  return cs;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
-int
-main (int argc, char *const *argv)
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
 {
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "secretsharing",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+  struct ClientState *cs = internal_cls;
+
+  if (NULL != cs->keygen_session)
+    keygen_session_destroy (cs->keygen_session);
+
+  if (NULL != cs->decrypt_session)
+    decrypt_session_destroy (cs->decrypt_session);
+  GNUNET_free (cs);
 }
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("secretsharing",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (client_keygen,
+                        GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_GENERATE,
+                        struct GNUNET_SECRETSHARING_CreateMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (client_decrypt,
+                        GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT,
+                        struct GNUNET_SECRETSHARING_DecryptRequestMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
diff --git a/src/set/.gitignore b/src/set/.gitignore
index 10f1da0d6..f1c958639 100644
--- a/src/set/.gitignore
+++ b/src/set/.gitignore
@@ -1,3 +1,7 @@
 gnunet-set-profiler
 gnunet-service-set
 gnunet-set-ibf-profiler
+test_set_api
+test_set_intersection_result_full
+test_set_union_copy
+test_set_union_result_symmetric
diff --git a/src/set/Makefile.am b/src/set/Makefile.am
index 4d990479c..14667d0ef 100644
--- a/src/set/Makefile.am
+++ b/src/set/Makefile.am
@@ -5,6 +5,8 @@ pkgcfgdir= $(pkgdatadir)/config.d/
 
 libexecdir= $(pkglibdir)/libexec/
 
+plugindir = $(libdir)/gnunet
+
 pkgcfg_DATA = \
   set.conf
 
@@ -49,7 +51,7 @@ gnunet_set_ibf_profiler_LDADD = \
 
 gnunet_service_set_SOURCES = \
  gnunet-service-set.c gnunet-service-set.h \
- gnunet-service-set_union.c \
+ gnunet-service-set_union.c gnunet-service-set_union.h \
  gnunet-service-set_intersection.c \
  ibf.c ibf.h \
  gnunet-service-set_union_strata_estimator.c gnunet-service-set_union_strata_estimator.h \
@@ -80,7 +82,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
@@ -112,5 +114,19 @@ test_set_union_copy_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   libgnunetset.la
 
+plugin_LTLIBRARIES = \
+  libgnunet_plugin_block_set_test.la
+
+libgnunet_plugin_block_set_test_la_SOURCES = \
+  plugin_block_set_test.c
+libgnunet_plugin_block_set_test_la_LIBADD = \
+  $(top_builddir)/src/block/libgnunetblock.la \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(LTLIBINTL)
+libgnunet_plugin_block_set_test_la_LDFLAGS = \
+ $(GN_PLUGIN_LDFLAGS)
+
+
 EXTRA_DIST = \
   test_set.conf
diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c
index e4e2535af..12af653c1 100644
--- a/src/set/gnunet-service-set.c
+++ b/src/set/gnunet-service-set.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -24,6 +24,8 @@
  * @author Christian Grothoff
  */
 #include "gnunet-service-set.h"
+#include "gnunet-service-set_union.h"
+#include "gnunet-service-set_intersection.h"
 #include "gnunet-service-set_protocol.h"
 #include "gnunet_statistics_service.h"
 
@@ -33,6 +35,35 @@
  */
 #define INCOMING_CHANNEL_TIMEOUT GNUNET_TIME_UNIT_MINUTES
 
+
+/**
+ * Lazy copy requests made by a client.
+ */
+struct LazyCopyRequest
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct LazyCopyRequest *prev;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct LazyCopyRequest *next;
+
+  /**
+   * Which set are we supposed to copy?
+   */
+  struct Set *source_set;
+
+  /**
+   * Cookie identifying the request.
+   */
+  uint32_t cookie;
+
+};
+
+
 /**
  * A listener is inhabited by a client, and waits for evaluation
  * requests from remote peers.
@@ -50,21 +81,24 @@ struct Listener
   struct Listener *prev;
 
   /**
-   * Client that owns the listener.
-   * Only one client may own a listener.
+   * Head of DLL of operations this listener is responsible for.
+   * Once the client has accepted/declined the operation, the
+   * operation is moved to the respective set's operation DLLS.
    */
-  struct GNUNET_SERVER_Client *client;
+  struct Operation *op_head;
 
   /**
-   * Message queue for the client
+   * Tail of DLL of operations this listener is responsible for.
+   * Once the client has accepted/declined the operation, the
+   * operation is moved to the respective set's operation DLLS.
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct Operation *op_tail;
 
   /**
-   * Application ID for the operation, used to distinguish
-   * multiple operations of the same type with the same peer.
+   * Client that owns the listener.
+   * Only one client may own a listener.
    */
-  struct GNUNET_HashCode app_id;
+  struct ClientState *cs;
 
   /**
    * The port we are listening on with CADET.
@@ -72,27 +106,18 @@ struct Listener
   struct GNUNET_CADET_Port *open_port;
 
   /**
+   * Application ID for the operation, used to distinguish
+   * multiple operations of the same type with the same peer.
+   */
+  struct GNUNET_HashCode app_id;
+
+  /**
    * The type of the operation.
    */
   enum GNUNET_SET_OperationType operation;
 };
 
 
-struct LazyCopyRequest
-{
-  struct Set *source_set;
-  uint32_t cookie;
-
-  struct LazyCopyRequest *prev;
-  struct LazyCopyRequest *next;
-};
-
-
-/**
- * Configuration of our local peer.
- */
-static const struct GNUNET_CONFIGURATION_Handle *configuration;
-
 /**
  * Handle to the cadet service, used to listen for and connect to
  * remote peers.
@@ -100,96 +125,48 @@ static const struct GNUNET_CONFIGURATION_Handle *configuration;
 static struct GNUNET_CADET_Handle *cadet;
 
 /**
- * Sets are held in a doubly linked list.
+ * DLL of lazy copy requests by this client.
  */
-static struct Set *sets_head;
+static struct LazyCopyRequest *lazy_copy_head;
 
 /**
- * Sets are held in a doubly linked list.
+ * DLL of lazy copy requests by this client.
  */
-static struct Set *sets_tail;
+static struct LazyCopyRequest *lazy_copy_tail;
 
 /**
- * Listeners are held in a doubly linked list.
+ * Generator for unique cookie we set per lazy copy request.
  */
-static struct Listener *listeners_head;
+static uint32_t lazy_copy_cookie;
 
 /**
- * Listeners are held in a doubly linked list.
+ * Statistics handle.
  */
-static struct Listener *listeners_tail;
+struct GNUNET_STATISTICS_Handle *_GSS_statistics;
 
 /**
- * Incoming sockets from remote peers are held in a doubly linked
- * list.
+ * Listeners are held in a doubly linked list.
  */
-static struct Operation *incoming_head;
+static struct Listener *listener_head;
 
 /**
- * Incoming sockets from remote peers are held in a doubly linked
- * list.
+ * Listeners are held in a doubly linked list.
  */
-static struct Operation *incoming_tail;
-
-static struct LazyCopyRequest *lazy_copy_head;
-static struct LazyCopyRequest *lazy_copy_tail;
-
-static uint32_t lazy_copy_cookie = 1;
+static struct Listener *listener_tail;
 
 /**
  * Counter for allocating unique IDs for clients, used to identify
  * incoming operation requests from remote peers, that the client can
- * choose to accept or refuse.
- */
-static uint32_t suggest_id = 1;
-
-/**
- * Statistics handle.
- */
-struct GNUNET_STATISTICS_Handle *_GSS_statistics;
-
-
-/**
- * Get set that is owned by the given client, if any.
- *
- * @param client client to look for
- * @return set that the client owns, NULL if the client
- *         does not own a set
+ * choose to accept or refuse.  0 must not be used (reserved for
+ * uninitialized).
  */
-static struct Set *
-set_get (struct GNUNET_SERVER_Client *client)
-{
-  struct Set *set;
-
-  for (set = sets_head; NULL != set; set = set->next)
-    if (set->client == client)
-      return set;
-  return NULL;
-}
-
-
-/**
- * Get the listener associated with the given client, if any.
- *
- * @param client the client
- * @return listener associated with the client, NULL
- *         if there isn't any
- */
-static struct Listener *
-listener_get (struct GNUNET_SERVER_Client *client)
-{
-  struct Listener *listener;
-
-  for (listener = listeners_head; NULL != listener; listener = listener->next)
-    if (listener->client == client)
-      return listener;
-  return NULL;
-}
+static uint32_t suggest_id;
 
 
 /**
  * Get the incoming socket associated with the given id.
  *
+ * @param listener the listener to look in
  * @param id id to look for
  * @return the incoming socket associated with the id,
  *         or NULL if there is none
@@ -197,48 +174,49 @@ listener_get (struct GNUNET_SERVER_Client *client)
 static struct Operation *
 get_incoming (uint32_t id)
 {
-  struct Operation *op;
-
-  for (op = incoming_head; NULL != op; op = op->next)
-    if (op->suggest_id == id)
-    {
-      GNUNET_assert (GNUNET_YES == op->is_incoming);
-      return op;
-    }
+  for (struct Listener *listener = listener_head;
+       NULL != listener;
+       listener = listener->next)
+  {
+    for (struct Operation *op = listener->op_head; NULL != op; op = op->next)
+      if (op->suggest_id == id)
+        return op;
+  }
   return NULL;
 }
 
 
 /**
- * Destroy a listener, free all resources associated with it.
+ * Destroy an incoming request from a remote peer
  *
- * @param listener listener to destroy
+ * @param op remote request to destroy
  */
 static void
-listener_destroy (struct Listener *listener)
+incoming_destroy (struct Operation *op)
 {
-  /* If the client is not dead yet, destroy it.
-   * The client's destroy callback will destroy the listener again. */
-  if (NULL != listener->client)
-  {
-    struct GNUNET_SERVER_Client *client = listener->client;
+  struct Listener *listener;
+  struct GNUNET_CADET_Channel *channel;
 
-    listener->client = NULL;
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Disconnecting listener client\n");
-    GNUNET_SERVER_client_disconnect (client);
-    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Destroying incoming operation %p\n",
+              op);
+  if (NULL != (listener = op->listener))
+  {
+    GNUNET_CONTAINER_DLL_remove (listener->op_head,
+                                 listener->op_tail,
+                                 op);
+    op->listener = NULL;
   }
-  if (NULL != listener->client_mq)
+  if (NULL != op->timeout_task)
   {
-    GNUNET_MQ_destroy (listener->client_mq);
-    listener->client_mq = NULL;
+    GNUNET_SCHEDULER_cancel (op->timeout_task);
+    op->timeout_task = NULL;
+  }
+  if (NULL != (channel = op->channel))
+  {
+    op->channel = NULL;
+    GNUNET_CADET_channel_destroy (channel);
   }
-  GNUNET_CADET_close_port (listener->open_port);
-  GNUNET_CONTAINER_DLL_remove (listeners_head,
-                               listeners_tail,
-                               listener);
-  GNUNET_free (listener);
 }
 
 
@@ -308,12 +286,11 @@ garbage_collect_cb (void *cls,
 static void
 collect_generation_garbage (struct Set *set)
 {
-  struct Operation *op;
   struct GarbageContext gc;
 
   gc.min_op_generation = UINT_MAX;
   gc.max_op_generation = 0;
-  for (op = set->ops_head; NULL != op; op = op->next)
+  for (struct Operation *op = set->ops_head; NULL != op; op = op->next)
   {
     gc.min_op_generation = GNUNET_MIN (gc.min_op_generation,
                                        op->generation_created);
@@ -327,23 +304,36 @@ collect_generation_garbage (struct Set *set)
 }
 
 
+/**
+ * Is @a generation in the range of exclusions?
+ *
+ * @param generation generation to query
+ * @param excluded array of generations where the element is excluded
+ * @param excluded_size length of the @a excluded array
+ * @return #GNUNET_YES if @a generation is in any of the ranges
+ */
 static int
 is_excluded_generation (unsigned int generation,
                         struct GenerationRange *excluded,
                         unsigned int excluded_size)
 {
-  unsigned int i;
-
-  for (i = 0; i < excluded_size; i++)
-  {
-    if ( (generation >= excluded[i].start) && (generation < excluded[i].end) )
+  for (unsigned int i = 0; i < excluded_size; i++)
+    if ( (generation >= excluded[i].start) &&
+         (generation < excluded[i].end) )
       return GNUNET_YES;
-  }
-
   return GNUNET_NO;
 }
 
 
+/**
+ * Is element @a ee part of the set during @a query_generation?
+ *
+ * @param ee element to test
+ * @param query_generation generation to query
+ * @param excluded array of generations where the element is excluded
+ * @param excluded_size length of the @a excluded array
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 static int
 is_element_of_generation (struct ElementEntry *ee,
                           unsigned int query_generation,
@@ -352,11 +342,12 @@ is_element_of_generation (struct ElementEntry *ee,
 {
   struct MutationEvent *mut;
   int is_present;
-  unsigned int i;
 
   GNUNET_assert (NULL != ee->mutations);
-
-  if (GNUNET_YES == is_excluded_generation (query_generation, excluded, excluded_size))
+  if (GNUNET_YES ==
+      is_excluded_generation (query_generation,
+                              excluded,
+                              excluded_size))
   {
     GNUNET_break (0);
     return GNUNET_NO;
@@ -366,7 +357,7 @@ is_element_of_generation (struct ElementEntry *ee,
 
   /* Could be made faster with binary search, but lists
      are small, so why bother. */
-  for (i = 0; i < ee->mutations_size; i++)
+  for (unsigned int i = 0; i < ee->mutations_size; i++)
   {
     mut = &ee->mutations[i];
 
@@ -378,7 +369,10 @@ is_element_of_generation (struct ElementEntry *ee,
       continue;
     }
 
-    if (GNUNET_YES == is_excluded_generation (mut->generation, excluded, excluded_size))
+    if (GNUNET_YES ==
+        is_excluded_generation (mut->generation,
+                                excluded,
+                                excluded_size))
     {
       /* The generation is excluded (because it belongs to another
          fork via a lazy copy) and thus mutations aren't considered
@@ -387,11 +381,12 @@ is_element_of_generation (struct ElementEntry *ee,
     }
 
     /* This would be an inconsistency in how we manage mutations. */
-    if ( (GNUNET_YES == is_present) && (GNUNET_YES == mut->added) )
+    if ( (GNUNET_YES == is_present) &&
+         (GNUNET_YES == mut->added) )
       GNUNET_assert (0);
-
     /* Likewise. */
-    if ( (GNUNET_NO == is_present) && (GNUNET_NO == mut->added) )
+    if ( (GNUNET_NO == is_present) &&
+         (GNUNET_NO == mut->added) )
       GNUNET_assert (0);
 
     is_present = mut->added;
@@ -401,44 +396,33 @@ is_element_of_generation (struct ElementEntry *ee,
 }
 
 
-int
-_GSS_is_element_of_set (struct ElementEntry *ee,
-                        struct Set *set)
-{
-  return is_element_of_generation (ee,
-                                   set->current_generation,
-                                   set->excluded_generations,
-                                   set->excluded_generations_size);
-}
-
-
-static int
-is_element_of_iteration (struct ElementEntry *ee,
-                         struct Set *set)
-{
-  return is_element_of_generation (ee,
-                                   set->iter_generation,
-                                   set->excluded_generations,
-                                   set->excluded_generations_size);
-}
-
-
+/**
+ * Is element @a ee part of the set used by @a op?
+ *
+ * @param ee element to test
+ * @param op operation the defines the set and its generation
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 int
 _GSS_is_element_of_operation (struct ElementEntry *ee,
                               struct Operation *op)
 {
   return is_element_of_generation (ee,
                                    op->generation_created,
-                                   op->spec->set->excluded_generations,
-                                   op->spec->set->excluded_generations_size);
+                                   op->set->excluded_generations,
+                                   op->set->excluded_generations_size);
 }
 
 
 /**
- * Destroy the given operation.  Call the implementation-specific
- * cancel function of the operation.  Disconnects from the remote
- * peer.  Does not disconnect the client, as there may be multiple
- * operations per set.
+ * Destroy the given operation.  Used for any operation where both
+ * peers were known and that thus actually had a vt and channel.  Must
+ * not be used for operations where 'listener' is still set and we do
+ * not know the other peer.
+ *
+ * Call the implementation-specific cancel function of the operation.
+ * Disconnects from the remote peer.  Does not disconnect the client,
+ * as there may be multiple operations per set.
  *
  * @param op operation to destroy
  * @param gc #GNUNET_YES to perform garbage collection on the set
@@ -447,43 +431,39 @@ void
 _GSS_operation_destroy (struct Operation *op,
                         int gc)
 {
-  struct Set *set;
+  struct Set *set = op->set;
   struct GNUNET_CADET_Channel *channel;
 
-  if (NULL == op->vt)
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Destroying operation %p\n",
+              op);
+  GNUNET_assert (NULL == op->listener);
+  if (NULL != op->state)
   {
-    /* already in #_GSS_operation_destroy() */
-    return;
+    set->vt->cancel (op);
+    op->state = NULL;
   }
-  GNUNET_assert (GNUNET_NO == op->is_incoming);
-  GNUNET_assert (NULL != op->spec);
-  set = op->spec->set;
-  GNUNET_CONTAINER_DLL_remove (set->ops_head,
-                               set->ops_tail,
-                               op);
-  op->vt->cancel (op);
-  op->vt = NULL;
-  if (NULL != op->spec)
+  if (NULL != set)
   {
-    if (NULL != op->spec->context_msg)
-    {
-      GNUNET_free (op->spec->context_msg);
-      op->spec->context_msg = NULL;
-    }
-    GNUNET_free (op->spec);
-    op->spec = NULL;
+    GNUNET_CONTAINER_DLL_remove (set->ops_head,
+                                 set->ops_tail,
+                                 op);
+    op->set = NULL;
   }
-  if (NULL != op->mq)
+  if (NULL != op->context_msg)
   {
-    GNUNET_MQ_destroy (op->mq);
-    op->mq = NULL;
+    GNUNET_free (op->context_msg);
+    op->context_msg = NULL;
   }
   if (NULL != (channel = op->channel))
   {
+    /* This will free op; called conditionally as this helper function
+       is also called from within the channel disconnect handler. */
     op->channel = NULL;
     GNUNET_CADET_channel_destroy (channel);
   }
-  if (GNUNET_YES == gc)
+  if ( (NULL != set) &&
+       (GNUNET_YES == gc) )
     collect_generation_garbage (set);
   /* We rely on the channel end handler to free 'op'. When 'op->channel' was NULL,
    * there was a channel end handler that will free 'op' on the call stack. */
@@ -491,6 +471,28 @@ _GSS_operation_destroy (struct Operation *op,
 
 
 /**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a `struct ClientState`
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct ClientState *cs;
+
+  cs = GNUNET_new (struct ClientState);
+  cs->client = c;
+  cs->mq = mq;
+  return cs;
+}
+
+
+/**
  * Iterator over hash map entries to free element entries.
  *
  * @param cls closure
@@ -506,71 +508,76 @@ destroy_elements_iterator (void *cls,
   struct ElementEntry *ee = value;
 
   GNUNET_free_non_null (ee->mutations);
-
   GNUNET_free (ee);
   return GNUNET_YES;
 }
 
 
 /**
- * Destroy a set, and free all resources and operations associated with it.
+ * Clean up after a client has disconnected
  *
- * @param set the set to destroy
+ * @param cls closure, unused
+ * @param client the client to clean up after
+ * @param internal_cls the `struct ClientState`
  */
 static void
-set_destroy (struct Set *set)
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
 {
-  if (NULL != set->client)
-  {
-    /* If the client is not dead yet, destroy it.  The client's destroy
-     * callback will call `set_destroy()` again in this case.  We do
-     * this so that the channel end handler still has a valid set handle
-     * to destroy. */
-    struct GNUNET_SERVER_Client *client = set->client;
-
-    set->client = NULL;
-    GNUNET_SERVER_client_disconnect (client);
-    return;
-  }
-  GNUNET_assert (NULL != set->state);
-  while (NULL != set->ops_head)
-    _GSS_operation_destroy (set->ops_head, GNUNET_NO);
-  set->vt->destroy_set (set->state);
-  set->state = NULL;
-  if (NULL != set->client_mq)
-  {
-    GNUNET_MQ_destroy (set->client_mq);
-    set->client_mq = NULL;
-  }
-  if (NULL != set->iter)
-  {
-    GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
-    set->iter = NULL;
-    set->iteration_id++;
-  }
+  struct ClientState *cs = internal_cls;
+  struct Operation *op;
+  struct Listener *listener;
+  struct Set *set;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client disconnected, cleaning up\n");
+  if (NULL != (set = cs->set))
   {
-    struct SetContent *content;
+    struct SetContent *content = set->content;
     struct PendingMutation *pm;
     struct PendingMutation *pm_current;
+    struct LazyCopyRequest *lcr;
 
-    content = set->content;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Destroying client's set\n");
+    /* Destroy pending set operations */
+    while (NULL != set->ops_head)
+      _GSS_operation_destroy (set->ops_head,
+                              GNUNET_NO);
+
+    /* Destroy operation-specific state */
+    GNUNET_assert (NULL != set->state);
+    set->vt->destroy_set (set->state);
+    set->state = NULL;
+
+    /* Clean up ongoing iterations */
+    if (NULL != set->iter)
+    {
+      GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
+      set->iter = NULL;
+      set->iteration_id++;
+    }
 
-    // discard any pending mutations that reference this set
+    /* discard any pending mutations that reference this set */
     pm = content->pending_mutations_head;
     while (NULL != pm)
     {
       pm_current = pm;
       pm = pm->next;
-      if (pm_current-> set == set)
+      if (pm_current->set == set)
+      {
         GNUNET_CONTAINER_DLL_remove (content->pending_mutations_head,
                                      content->pending_mutations_tail,
                                      pm_current);
-
+        GNUNET_free (pm_current);
+      }
     }
 
+    /* free set content (or at least decrement RC) */
     set->content = NULL;
     GNUNET_assert (0 != content->refcount);
-    content->refcount -= 1;
+    content->refcount--;
     if (0 == content->refcount)
     {
       GNUNET_assert (NULL != content->elements);
@@ -581,140 +588,85 @@ set_destroy (struct Set *set)
       content->elements = NULL;
       GNUNET_free (content);
     }
-  }
-  GNUNET_free_non_null (set->excluded_generations);
-  set->excluded_generations = NULL;
-  GNUNET_CONTAINER_DLL_remove (sets_head,
-                               sets_tail,
-                               set);
+    GNUNET_free_non_null (set->excluded_generations);
+    set->excluded_generations = NULL;
 
-  // remove set from pending copy requests
-  {
-    struct LazyCopyRequest *lcr;
+    /* remove set from pending copy requests */
     lcr = lazy_copy_head;
     while (NULL != lcr)
     {
-      struct LazyCopyRequest *lcr_current;
-      lcr_current = lcr;
+      struct LazyCopyRequest *lcr_current = lcr;
+
       lcr = lcr->next;
       if (lcr_current->source_set == set)
+      {
         GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
                                      lazy_copy_tail,
                                      lcr_current);
+        GNUNET_free (lcr_current);
+      }
     }
+    GNUNET_free (set);
   }
 
-  GNUNET_free (set);
-}
-
-
-/**
- * Clean up after a client has disconnected
- *
- * @param cls closure, unused
- * @param client the client to clean up after
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct Listener *listener;
-  struct Set *set;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "client disconnected, cleaning up\n");
-  set = set_get (client);
-  if (NULL != set)
+  if (NULL != (listener = cs->listener))
   {
-    set->client = NULL;
-    set_destroy (set);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client's set destroyed\n");
-  }
-  listener = listener_get (client);
-  if (NULL != listener)
-  {
-    listener->client = NULL;
-    listener_destroy (listener);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client's listener destroyed\n");
-  }
+                "Destroying client's listener\n");
+    GNUNET_CADET_close_port (listener->open_port);
+    listener->open_port = NULL;
+    while (NULL != (op = listener->op_head))
+      incoming_destroy (op);
+    GNUNET_CONTAINER_DLL_remove (listener_head,
+                                 listener_tail,
+                                 listener);
+    GNUNET_free (listener);
+  }
+  GNUNET_free (cs);
 }
 
 
 /**
- * Destroy an incoming request from a remote peer
+ * Check a request for a set operation from another peer.
  *
- * @param incoming remote request to destroy
+ * @param cls the operation state
+ * @param msg the received message
+ * @return #GNUNET_OK if the channel should be kept alive,
+ *         #GNUNET_SYSERR to destroy the channel
  */
-static void
-incoming_destroy (struct Operation *incoming)
+static int
+check_incoming_msg (void *cls,
+                    const struct OperationRequestMessage *msg)
 {
-  struct GNUNET_CADET_Channel *channel;
+  struct Operation *op = cls;
+  struct Listener *listener = op->listener;
+  const struct GNUNET_MessageHeader *nested_context;
 
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
-  GNUNET_CONTAINER_DLL_remove (incoming_head,
-                               incoming_tail,
-                               incoming);
-  if (NULL != incoming->timeout_task)
+  /* double operation request */
+  if (0 != op->suggest_id)
   {
-    GNUNET_SCHEDULER_cancel (incoming->timeout_task);
-    incoming->timeout_task = NULL;
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  /* make sure that the tunnel end handler will not destroy us again */
-  incoming->vt = NULL;
-  if (NULL != incoming->spec)
+  /* This should be equivalent to the previous condition, but can't hurt to check twice */
+  if (NULL == op->listener)
   {
-    GNUNET_free (incoming->spec);
-    incoming->spec = NULL;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
-  if (NULL != incoming->mq)
+  if (listener->operation != (enum GNUNET_SET_OperationType) ntohl (msg->operation))
   {
-    GNUNET_MQ_destroy (incoming->mq);
-    incoming->mq = NULL;
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  if (NULL != (channel = incoming->channel))
+  nested_context = GNUNET_MQ_extract_nested_mh (msg);
+  if ( (NULL != nested_context) &&
+       (ntohs (nested_context->size) > GNUNET_SET_CONTEXT_MESSAGE_MAX_SIZE) )
   {
-    incoming->channel = NULL;
-    GNUNET_CADET_channel_destroy (channel);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-}
-
-
-/**
- * Suggest the given request to the listener. The listening client can
- * then accept or reject the remote request.
- *
- * @param incoming the incoming peer with the request to suggest
- * @param listener the listener to suggest the request to
- */
-static void
-incoming_suggest (struct Operation *incoming,
-                  struct Listener *listener)
-{
-  struct GNUNET_MQ_Envelope *mqm;
-  struct GNUNET_SET_RequestMessage *cmsg;
-
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
-  GNUNET_assert (NULL != incoming->spec);
-  GNUNET_assert (0 == incoming->suggest_id);
-  incoming->suggest_id = suggest_id++;
-  if (0 == suggest_id)
-    suggest_id++;
-  GNUNET_assert (NULL != incoming->timeout_task);
-  GNUNET_SCHEDULER_cancel (incoming->timeout_task);
-  incoming->timeout_task = NULL;
-  mqm = GNUNET_MQ_msg_nested_mh (cmsg,
-                                 GNUNET_MESSAGE_TYPE_SET_REQUEST,
-                                 incoming->spec->context_msg);
-  GNUNET_assert (NULL != mqm);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Suggesting incoming request with accept id %u to listener\n",
-              incoming->suggest_id);
-  cmsg->accept_id = htonl (incoming->suggest_id);
-  cmsg->peer_id = incoming->spec->peer;
-  GNUNET_MQ_send (listener->client_mq,
-                  mqm);
+  return GNUNET_OK;
 }
 
 
@@ -730,93 +682,85 @@ incoming_suggest (struct Operation *incoming,
  * our virtual table and subsequent msgs would be routed differently (as
  * we then know what type of operation this is).
  *
- * @param op the operation state
- * @param mh the received message
+ * @param cls the operation state
+ * @param msg the received message
  * @return #GNUNET_OK if the channel should be kept alive,
  *         #GNUNET_SYSERR to destroy the channel
  */
-static int
-handle_incoming_msg (struct Operation *op,
-                     const struct GNUNET_MessageHeader *mh)
+static void
+handle_incoming_msg (void *cls,
+                     const struct OperationRequestMessage *msg)
 {
-  const struct OperationRequestMessage *msg;
+  struct Operation *op = cls;
   struct Listener *listener = op->listener;
-  struct OperationSpecification *spec;
   const struct GNUNET_MessageHeader *nested_context;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_SET_RequestMessage *cmsg;
 
-  msg = (const struct OperationRequestMessage *) mh;
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  if (GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST != ntohs (mh->type))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  /* double operation request */
-  if (NULL != op->spec)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  spec = GNUNET_new (struct OperationSpecification);
   nested_context = GNUNET_MQ_extract_nested_mh (msg);
-  if ( (NULL != nested_context) &&
-       (ntohs (nested_context->size) > GNUNET_SET_CONTEXT_MESSAGE_MAX_SIZE) )
-  {
-    GNUNET_break_op (0);
-    GNUNET_free (spec);
-    return GNUNET_SYSERR;
-  }
   /* Make a copy of the nested_context (application-specific context
      information that is opaque to set) so we can pass it to the
      listener later on */
   if (NULL != nested_context)
-    spec->context_msg = GNUNET_copy_message (nested_context);
-  spec->operation = ntohl (msg->operation);
-  spec->app_id = listener->app_id;
-  spec->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                         UINT32_MAX);
-  spec->peer = op->peer;
-  spec->remote_element_count = ntohl (msg->element_count);
-  op->spec = spec;
-
-  listener = op->listener;
+    op->context_msg = GNUNET_copy_message (nested_context);
+  op->remote_element_count = ntohl (msg->element_count);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received P2P operation request (op %u, port %s) for active listener\n",
-              ntohl (msg->operation),
-              GNUNET_h2s (&listener->app_id));
-  incoming_suggest (op,
-                    listener);
-  return GNUNET_OK;
+              (uint32_t) ntohl (msg->operation),
+              GNUNET_h2s (&op->listener->app_id));
+  GNUNET_assert (0 == op->suggest_id);
+  if (0 == suggest_id)
+    suggest_id++;
+  op->suggest_id = suggest_id++;
+  GNUNET_assert (NULL != op->timeout_task);
+  GNUNET_SCHEDULER_cancel (op->timeout_task);
+  op->timeout_task = NULL;
+  env = GNUNET_MQ_msg_nested_mh (cmsg,
+                                 GNUNET_MESSAGE_TYPE_SET_REQUEST,
+                                 op->context_msg);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Suggesting incoming request with accept id %u to listener %p of client %p\n",
+              op->suggest_id,
+              listener,
+              listener->cs);
+  cmsg->accept_id = htonl (op->suggest_id);
+  cmsg->peer_id = op->peer;
+  GNUNET_MQ_send (listener->cs->mq,
+                  env);
+  /* NOTE: GNUNET_CADET_receive_done() will be called in
+     #handle_client_accept() */
 }
 
 
+/**
+ * Add an element to @a set as specified by @a msg
+ *
+ * @param set set to manipulate
+ * @param msg message specifying the change
+ */
 static void
 execute_add (struct Set *set,
-             const struct GNUNET_MessageHeader *m)
+             const struct GNUNET_SET_ElementMessage *msg)
 {
-  const struct GNUNET_SET_ElementMessage *msg;
   struct GNUNET_SET_Element el;
   struct ElementEntry *ee;
   struct GNUNET_HashCode hash;
 
-  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (m->type));
-
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (msg->header.type));
+  el.size = ntohs (msg->header.size) - sizeof (*msg);
   el.data = &msg[1];
   el.element_type = ntohs (msg->element_type);
-  GNUNET_SET_element_hash (&el, &hash);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client inserts element %s of size %u\n",
-              GNUNET_h2s (&hash),
-              el.size);
-
+  GNUNET_SET_element_hash (&el,
+                           &hash);
   ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
                                           &hash);
-
   if (NULL == ee)
   {
-    ee = GNUNET_malloc (el.size + sizeof *ee);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client inserts element %s of size %u\n",
+                GNUNET_h2s (&hash),
+                el.size);
+    ee = GNUNET_malloc (el.size + sizeof (*ee));
     ee->element.size = el.size;
     GNUNET_memcpy (&ee[1],
             el.data,
@@ -833,8 +777,17 @@ execute_add (struct Set *set,
                                                      ee,
                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   }
-  else if (GNUNET_YES == _GSS_is_element_of_set (ee, set))
+  else if (GNUNET_YES ==
+           is_element_of_generation (ee,
+                                     set->current_generation,
+                                     set->excluded_generations,
+                                     set->excluded_generations_size))
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client inserted element %s of size %u twice (ignored)\n",
+                GNUNET_h2s (&hash),
+                el.size);
+
     /* same element inserted twice */
     return;
   }
@@ -844,29 +797,31 @@ execute_add (struct Set *set,
       .generation = set->current_generation,
       .added = GNUNET_YES
     };
-    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
+    GNUNET_array_append (ee->mutations,
+                         ee->mutations_size,
+                         mut);
   }
-
-  set->vt->add (set->state, ee);
+  set->vt->add (set->state,
+                ee);
 }
 
 
+/**
+ * Remove an element from @a set as specified by @a msg
+ *
+ * @param set set to manipulate
+ * @param msg message specifying the change
+ */
 static void
 execute_remove (struct Set *set,
-                const struct GNUNET_MessageHeader *m)
+                const struct GNUNET_SET_ElementMessage *msg)
 {
-  const struct GNUNET_SET_ElementMessage *msg;
   struct GNUNET_SET_Element el;
   struct ElementEntry *ee;
   struct GNUNET_HashCode hash;
 
-  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (m->type));
-
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client removes element of size %u\n",
-              el.size);
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (msg->header.type));
+  el.size = ntohs (msg->header.size) - sizeof (*msg);
   el.data = &msg[1];
   el.element_type = ntohs (msg->element_type);
   GNUNET_SET_element_hash (&el, &hash);
@@ -875,11 +830,21 @@ execute_remove (struct Set *set,
   if (NULL == ee)
   {
     /* Client tried to remove non-existing element. */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client removes non-existing element of size %u\n",
+                el.size);
     return;
   }
-  if (GNUNET_NO == _GSS_is_element_of_set (ee, set))
+  if (GNUNET_NO ==
+      is_element_of_generation (ee,
+                                set->current_generation,
+                                set->excluded_generations,
+                                set->excluded_generations_size))
   {
     /* Client tried to remove element twice */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client removed element of size %u twice (ignored)\n",
+                el.size);
     return;
   }
   else
@@ -888,24 +853,37 @@ execute_remove (struct Set *set,
       .generation = set->current_generation,
       .added = GNUNET_NO
     };
-    GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
+
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client removes element of size %u\n",
+                el.size);
+
+    GNUNET_array_append (ee->mutations,
+                         ee->mutations_size,
+                         mut);
   }
-  set->vt->remove (set->state, ee);
+  set->vt->remove (set->state,
+                   ee);
 }
 
 
-
+/**
+ * Perform a mutation on a set as specified by the @a msg
+ *
+ * @param set the set to mutate
+ * @param msg specification of what to change
+ */
 static void
 execute_mutation (struct Set *set,
-                  const struct GNUNET_MessageHeader *m)
+                  const struct GNUNET_SET_ElementMessage *msg)
 {
-  switch (ntohs (m->type))
+  switch (ntohs (msg->header.type))
   {
     case GNUNET_MESSAGE_TYPE_SET_ADD:
-      execute_add (set, m);
+      execute_add (set, msg);
       break;
     case GNUNET_MESSAGE_TYPE_SET_REMOVE:
-      execute_remove (set, m);
+      execute_remove (set, msg);
       break;
     default:
       GNUNET_break (0);
@@ -913,6 +891,34 @@ execute_mutation (struct Set *set,
 }
 
 
+/**
+ * Execute mutations that were delayed on a set because of
+ * pending operations.
+ *
+ * @param set the set to execute mutations on
+ */
+static void
+execute_delayed_mutations (struct Set *set)
+{
+  struct PendingMutation *pm;
+
+  if (0 != set->content->iterator_count)
+    return; /* still cannot do this */
+  while (NULL != (pm = set->content->pending_mutations_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
+                                 set->content->pending_mutations_tail,
+                                 pm);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Executing pending mutation on %p.\n",
+                pm->set);
+    execute_mutation (pm->set,
+                      pm->msg);
+    GNUNET_free (pm->msg);
+    GNUNET_free (pm);
+  }
+}
+
 
 /**
  * Send the next element of a set to the set's client.  The next element is given by
@@ -936,65 +942,45 @@ send_client_element (struct Set *set)
   struct GNUNET_SET_IterResponseMessage *msg;
 
   GNUNET_assert (NULL != set->iter);
-
-again:
-
-  ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
-                                                     NULL,
-                                                     (const void **) &ee);
-  if (GNUNET_NO == ret)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Iteration on %p done.\n",
-                (void *) set);
-    ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
-    GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
-    set->iter = NULL;
-    set->iteration_id++;
-
-    GNUNET_assert (set->content->iterator_count > 0);
-    set->content->iterator_count -= 1;
-
-    if (0 == set->content->iterator_count)
+  do {
+    ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
+                                                       NULL,
+                                                       (const void **) &ee);
+    if (GNUNET_NO == ret)
     {
-      while (NULL != set->content->pending_mutations_head)
-      {
-        struct PendingMutation *pm;
-
-        pm = set->content->pending_mutations_head;
-        GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
-                                     set->content->pending_mutations_tail,
-                                     pm);
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    "Executing pending mutation on %p.\n",
-                    (void *) pm->set);
-        execute_mutation (pm->set, pm->mutation_message);
-        GNUNET_free (pm->mutation_message);
-        GNUNET_free (pm);
-      }
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Iteration on %p done.\n",
+                  set);
+      ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
+      GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
+      set->iter = NULL;
+      set->iteration_id++;
+      GNUNET_assert (set->content->iterator_count > 0);
+      set->content->iterator_count--;
+      execute_delayed_mutations (set);
+      GNUNET_MQ_send (set->cs->mq,
+                      ev);
+      return;
     }
-
-  }
-  else
-  {
     GNUNET_assert (NULL != ee);
-
-    if (GNUNET_NO == is_element_of_iteration (ee, set))
-      goto again;
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sending iteration element on %p.\n",
-                (void *) set);
-    ev = GNUNET_MQ_msg_extra (msg,
-                              ee->element.size,
-                              GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
-    GNUNET_memcpy (&msg[1],
-            ee->element.data,
-            ee->element.size);
-    msg->element_type = htons (ee->element.element_type);
-    msg->iteration_id = htons (set->iteration_id);
-  }
-  GNUNET_MQ_send (set->client_mq, ev);
+  } while (GNUNET_NO ==
+           is_element_of_generation (ee,
+                                     set->iter_generation,
+                                     set->excluded_generations,
+                                     set->excluded_generations_size));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending iteration element on %p.\n",
+              set);
+  ev = GNUNET_MQ_msg_extra (msg,
+                            ee->element.size,
+                            GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
+  GNUNET_memcpy (&msg[1],
+                 ee->element.data,
+                 ee->element.size);
+  msg->element_type = htons (ee->element.element_type);
+  msg->iteration_id = htons (set->iteration_id);
+  GNUNET_MQ_send (set->cs->mq,
+                  ev);
 }
 
 
@@ -1004,30 +990,28 @@ again:
  * can right now start an iteration. If all checks out, starts
  * sending the elements of the set to the client.
  *
- * @param cls unused
- * @param client client that sent the message
+ * @param cls client that sent the message
  * @param m message sent by the client
  */
 static void
 handle_client_iterate (void *cls,
-                       struct GNUNET_SERVER_Client *client,
                        const struct GNUNET_MessageHeader *m)
 {
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* attempt to iterate over a non existing set */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   if (NULL != set->iter)
   {
     /* Only one concurrent iterate-action allowed per set */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1035,9 +1019,8 @@ handle_client_iterate (void *cls,
               (void *) set,
               set->current_generation,
               GNUNET_CONTAINER_multihashmap_size (set->content->elements));
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
-  set->content->iterator_count += 1;
+  GNUNET_SERVICE_client_continue (cs->client);
+  set->content->iterator_count++;
   set->iter = GNUNET_CONTAINER_multihashmap_iterator_create (set->content->elements);
   set->iter_generation = set->current_generation;
   send_client_element (set);
@@ -1049,27 +1032,24 @@ handle_client_iterate (void *cls,
  * the first request from a client, and includes the type of set
  * operation to be performed.
  *
- * @param cls unused
- * @param client client that sent the message
+ * @param cls client that sent the message
  * @param m message sent by the client
  */
 static void
 handle_client_create_set (void *cls,
-                          struct GNUNET_SERVER_Client *client,
-                          const struct GNUNET_MessageHeader *m)
+                          const struct GNUNET_SET_CreateMessage *msg)
 {
-  const struct GNUNET_SET_CreateMessage *msg;
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  msg = (const struct GNUNET_SET_CreateMessage *) m;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client created new set (operation %u)\n",
-              ntohl (msg->operation));
-  if (NULL != set_get (client))
+              (uint32_t) ntohl (msg->operation));
+  if (NULL != cs->set)
   {
     /* There can only be one set per client */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   set = GNUNET_new (struct Set);
@@ -1084,28 +1064,25 @@ handle_client_create_set (void *cls,
   default:
     GNUNET_free (set);
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  set->operation = ntohl (msg->operation);
+  set->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
   set->state = set->vt->create ();
   if (NULL == set->state)
   {
     /* initialization failed (i.e. out of memory) */
     GNUNET_free (set);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   set->content = GNUNET_new (struct SetContent);
   set->content->refcount = 1;
-  set->content->elements = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_YES);
-  set->client = client;
-  set->client_mq = GNUNET_MQ_queue_for_server_client (client);
-  GNUNET_CONTAINER_DLL_insert (sets_head,
-                               sets_tail,
-                               set);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  set->content->elements = GNUNET_CONTAINER_multihashmap_create (1,
+                                                                 GNUNET_YES);
+  set->cs = cs;
+  cs->set = set;
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1121,31 +1098,12 @@ handle_client_create_set (void *cls,
 static void
 incoming_timeout_cb (void *cls)
 {
-  struct Operation *incoming = cls;
+  struct Operation *op = cls;
 
-  incoming->timeout_task = NULL;
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
+  op->timeout_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Remote peer's incoming request timed out\n");
-  incoming_destroy (incoming);
-}
-
-
-/**
- * Terminates an incoming operation in case we have not yet received an
- * operation request. Called by the channel destruction handler.
- *
- * @param op the channel context
- */
-static void
-handle_incoming_disconnect (struct Operation *op)
-{
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  /* channel is already dead, incoming_destroy must not
-   * destroy it ... */
-  op->channel = NULL;
   incoming_destroy (op);
-  op->vt = NULL;
 }
 
 
@@ -1161,108 +1119,200 @@ handle_incoming_disconnect (struct Operation *op)
  *
  * @param cls closure
  * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port Port this channel is for.
- * @param options Unused.
+ * @param source peer that started the channel
  * @return initial channel context for the channel
  *         returns NULL on error
  */
 static void *
 channel_new_cb (void *cls,
                 struct GNUNET_CADET_Channel *channel,
-                const struct GNUNET_PeerIdentity *initiator,
-                const struct GNUNET_HashCode *port,
-                enum GNUNET_CADET_ChannelOption options)
+                const struct GNUNET_PeerIdentity *source)
 {
-  static const struct SetVT incoming_vt = {
-    .msg_handler = &handle_incoming_msg,
-    .peer_disconnect = &handle_incoming_disconnect
-  };
   struct Listener *listener = cls;
-  struct Operation *incoming;
+  struct Operation *op;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New incoming channel\n");
-  incoming = GNUNET_new (struct Operation);
-  incoming->listener = listener;
-  incoming->is_incoming = GNUNET_YES;
-  incoming->peer = *initiator;
-  incoming->channel = channel;
-  incoming->mq = GNUNET_CADET_mq_create (incoming->channel);
-  incoming->vt = &incoming_vt;
-  incoming->timeout_task
+  op = GNUNET_new (struct Operation);
+  op->listener = listener;
+  op->peer = *source;
+  op->channel = channel;
+  op->mq = GNUNET_CADET_get_mq (op->channel);
+  op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
+                                       UINT32_MAX);
+  op->timeout_task
     = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
                                     &incoming_timeout_cb,
-                                    incoming);
-  GNUNET_CONTAINER_DLL_insert_tail (incoming_head,
-                                    incoming_tail,
-                                    incoming);
-  // incoming_suggest (incoming,
-  //                  listener);
-  return incoming;
+                                    op);
+  GNUNET_CONTAINER_DLL_insert (listener->op_head,
+                               listener->op_tail,
+                               op);
+  return op;
+}
+
+
+/**
+ * Function called whenever a channel is destroyed.  Should clean up
+ * any associated state.  It must NOT call
+ * GNUNET_CADET_channel_destroy() on the channel.
+ *
+ * The peer_disconnect function is part of a a virtual table set initially either
+ * when a peer creates a new channel with us, or once we create
+ * a new channel ourselves (evaluate).
+ *
+ * Once we know the exact type of operation (union/intersection), the vt is
+ * replaced with an operation specific instance (_GSS_[op]_vt).
+ *
+ * @param channel_ctx place where local state associated
+ *                   with the channel is stored
+ * @param channel connection to the other end (henceforth invalid)
+ */
+static void
+channel_end_cb (void *channel_ctx,
+                const struct GNUNET_CADET_Channel *channel)
+{
+  struct Operation *op = channel_ctx;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "channel_end_cb called\n");
+  op->channel = NULL;
+  if (NULL != op->listener)
+    incoming_destroy (op);
+  else if (NULL != op->set)
+    op->set->vt->channel_death (op);
+  else
+    _GSS_operation_destroy (op,
+                            GNUNET_YES);
+  GNUNET_free (op);
+}
+
+
+/**
+ * Function called whenever an MQ-channel's transmission window size changes.
+ *
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative..
+ */
+static void
+channel_window_cb (void *cls,
+                   const struct GNUNET_CADET_Channel *channel,
+                   int window_size)
+{
+  /* FIXME: not implemented, we could do flow control here... */
 }
 
 
 /**
  * Called when a client wants to create a new listener.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client that sent the message
+ * @param msg message sent by the client
  */
 static void
 handle_client_listen (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *m)
+                      const struct GNUNET_SET_ListenMessage *msg)
 {
-  const struct GNUNET_SET_ListenMessage *msg;
+  struct ClientState *cs = cls;
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_var_size (incoming_msg,
+                           GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
+                           struct OperationRequestMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_ibf,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
+                           struct IBFMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_elements,
+                           GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
+                           struct GNUNET_SET_ElementMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_offer,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_inquiry,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
+                           struct InquiryMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_demand,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_hd_fixed_size (union_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
+                           struct StrataEstimatorMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
+                           struct StrataEstimatorMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_full_element,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
+                           struct GNUNET_SET_ElementMessage,
+                           NULL),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
+                             struct IntersectionElementInfoMessage,
+                             NULL),
+    GNUNET_MQ_hd_var_size (intersection_p2p_bf,
+                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
+                           struct BFMessage,
+                           NULL),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
+                             struct IntersectionDoneMessage,
+                             NULL),
+    GNUNET_MQ_handler_end ()
+  };
   struct Listener *listener;
-  struct Operation *op;
 
-  msg = (const struct GNUNET_SET_ListenMessage *) m;
-  if (NULL != listener_get (client))
+  if (NULL != cs->listener)
   {
     /* max. one active listener per client! */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   listener = GNUNET_new (struct Listener);
-  listener->client = client;
-  listener->client_mq = GNUNET_MQ_queue_for_server_client (client);
+  listener->cs = cs;
   listener->app_id = msg->app_id;
-  listener->operation = ntohl (msg->operation);
-  GNUNET_CONTAINER_DLL_insert_tail (listeners_head,
-                                    listeners_tail,
-                                    listener);
-  listener->open_port = GNUNET_CADET_open_port (cadet,
-                                                &msg->app_id,
-                                                &channel_new_cb,
-                                                listener);
+  listener->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
+  GNUNET_CONTAINER_DLL_insert (listener_head,
+                               listener_tail,
+                               listener);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New listener created (op %u, port %s)\n",
               listener->operation,
               GNUNET_h2s (&listener->app_id));
-
-  /* check for existing incoming requests the listener might be interested in */
-  for (op = incoming_head; NULL != op; op = op->next)
-  {
-    if (NULL == op->spec)
-      continue; /* no details available yet */
-    if (0 != op->suggest_id)
-      continue; /* this one has been already suggested to a listener */
-    if (listener->operation != op->spec->operation)
-      continue; /* incompatible operation */
-    if (0 != GNUNET_CRYPTO_hash_cmp (&listener->app_id,
-                                     &op->spec->app_id))
-      continue; /* incompatible appliation */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Found matching existing request\n");
-    incoming_suggest (op,
-                      listener);
-  }
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  listener->open_port
+    = GNUNET_CADET_open_port (cadet,
+                              &msg->app_id,
+                              &channel_new_cb,
+                              listener,
+                              &channel_window_cb,
+                              &channel_end_cb,
+                              cadet_handlers);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1270,64 +1320,73 @@ handle_client_listen (void *cls,
  * Called when the listening client rejects an operation
  * request by another peer.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client that sent the message
+ * @param msg message sent by the client
  */
 static void
 handle_client_reject (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *m)
+                      const struct GNUNET_SET_RejectMessage *msg)
 {
-  struct Operation *incoming;
-  const struct GNUNET_SET_RejectMessage *msg;
+  struct ClientState *cs = cls;
+  struct Operation *op;
 
-  msg = (const struct GNUNET_SET_RejectMessage *) m;
-  incoming = get_incoming (ntohl (msg->accept_reject_id));
-  if (NULL == incoming)
+  op = get_incoming (ntohl (msg->accept_reject_id));
+  if (NULL == op)
   {
-    /* no matching incoming operation for this reject */
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client,
-                                GNUNET_SYSERR);
+    /* no matching incoming operation for this reject;
+       could be that the other peer already disconnected... */
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client rejected unknown operation %u\n",
+                (unsigned int) ntohl (msg->accept_reject_id));
+    GNUNET_SERVICE_client_continue (cs->client);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer request (op %u, app %s) rejected by client\n",
-              incoming->spec->operation,
-              GNUNET_h2s (&incoming->spec->app_id));
-  GNUNET_CADET_channel_destroy (incoming->channel);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+              op->listener->operation,
+              GNUNET_h2s (&cs->listener->app_id));
+  GNUNET_CADET_channel_destroy (op->channel);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
+/**
+ * Called when a client wants to add or remove an element to a set it inhabits.
+ *
+ * @param cls client that sent the message
+ * @param msg message sent by the client
+ */
+static int
+check_client_mutation (void *cls,
+                       const struct GNUNET_SET_ElementMessage *msg)
+{
+  /* NOTE: Technically, we should probably check with the
+     block library whether the element we are given is well-formed */
+  return GNUNET_OK;
+}
+
 
 /**
  * Called when a client wants to add or remove an element to a set it inhabits.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client that sent the message
+ * @param msg message sent by the client
  */
 static void
 handle_client_mutation (void *cls,
-                        struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *m)
+                        const struct GNUNET_SET_ElementMessage *msg)
 {
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  GNUNET_SERVICE_client_continue (cs->client);
 
   if (0 != set->content->iterator_count)
   {
@@ -1335,17 +1394,18 @@ handle_client_mutation (void *cls,
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Scheduling mutation on set\n");
-
     pm = GNUNET_new (struct PendingMutation);
-    pm->mutation_message = GNUNET_copy_message (m);
+    pm->msg = (struct GNUNET_SET_ElementMessage *) GNUNET_copy_message (&msg->header);
     pm->set = set;
-    GNUNET_CONTAINER_DLL_insert (set->content->pending_mutations_head,
-                                 set->content->pending_mutations_tail,
-                                 pm);
+    GNUNET_CONTAINER_DLL_insert_tail (set->content->pending_mutations_head,
+                                      set->content->pending_mutations_tail,
+                                      pm);
     return;
   }
-
-  execute_mutation (set, m);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Executing mutation on set\n");
+  execute_mutation (set,
+                    msg);
 }
 
 
@@ -1362,8 +1422,8 @@ advance_generation (struct Set *set)
 
   if (set->current_generation == set->content->latest_generation)
   {
-    set->content->latest_generation += 1;
-    set->current_generation += 1;
+    set->content->latest_generation++;
+    set->current_generation++;
     return;
   }
 
@@ -1371,10 +1431,8 @@ advance_generation (struct Set *set)
 
   r.start = set->current_generation + 1;
   r.end = set->content->latest_generation + 1;
-
   set->content->latest_generation = r.end;
   set->current_generation = r.end;
-
   GNUNET_array_append (set->excluded_generations,
                        set->excluded_generations_size,
                        r);
@@ -1386,64 +1444,148 @@ advance_generation (struct Set *set)
  * peer.  Initiates the CADET connection to the listener and sends the
  * request.
  *
- * @param cls unused
- * @param client client that sent the message
- * @param m message sent by the client
+ * @param cls client that sent the message
+ * @param msg message sent by the client
+ * @return #GNUNET_OK if the message is well-formed
+ */
+static int
+check_client_evaluate (void *cls,
+                        const struct GNUNET_SET_EvaluateMessage *msg)
+{
+  /* FIXME: suboptimal, even if the context below could be NULL,
+     there are malformed messages this does not check for... */
+  return GNUNET_OK;
+}
+
+
+/**
+ * Called when a client wants to initiate a set operation with another
+ * peer.  Initiates the CADET connection to the listener and sends the
+ * request.
+ *
+ * @param cls client that sent the message
+ * @param msg message sent by the client
  */
 static void
 handle_client_evaluate (void *cls,
-                        struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *m)
+                        const struct GNUNET_SET_EvaluateMessage *msg)
 {
+  struct ClientState *cs = cls;
+  struct Operation *op = GNUNET_new (struct Operation);
+  const struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_var_size (incoming_msg,
+                           GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
+                           struct OperationRequestMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_ibf,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
+                           struct IBFMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_elements,
+                           GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
+                           struct GNUNET_SET_ElementMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_offer,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
+                           struct GNUNET_MessageHeader,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_inquiry,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
+                           struct InquiryMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_demand,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
+                           struct GNUNET_MessageHeader,
+                           op),
+    GNUNET_MQ_hd_fixed_size (union_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
+                           struct StrataEstimatorMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
+                           struct StrataEstimatorMessage,
+                           op),
+    GNUNET_MQ_hd_var_size (union_p2p_full_element,
+                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
+                           struct GNUNET_SET_ElementMessage,
+                           op),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
+                             struct IntersectionElementInfoMessage,
+                             op),
+    GNUNET_MQ_hd_var_size (intersection_p2p_bf,
+                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
+                           struct BFMessage,
+                           op),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
+                             struct IntersectionDoneMessage,
+                             op),
+    GNUNET_MQ_handler_end ()
+  };
   struct Set *set;
-  const struct GNUNET_SET_EvaluateMessage *msg;
-  struct OperationSpecification *spec;
-  struct Operation *op;
   const struct GNUNET_MessageHeader *context;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_free (op);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  msg = (const struct GNUNET_SET_EvaluateMessage *) m;
-  spec = GNUNET_new (struct OperationSpecification);
-  spec->operation = set->operation;
-  spec->app_id = msg->app_id;
-  spec->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                         UINT32_MAX);
-  spec->peer = msg->target_peer;
-  spec->set = set;
-  spec->result_mode = ntohl (msg->result_mode);
-  spec->client_request_id = ntohl (msg->request_id);
+  op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
+                                       UINT32_MAX);
+  op->peer = msg->target_peer;
+  op->result_mode = ntohl (msg->result_mode);
+  op->client_request_id = ntohl (msg->request_id);
+  op->byzantine = msg->byzantine;
+  op->byzantine_lower_bound = msg->byzantine_lower_bound;
+  op->force_full = msg->force_full;
+  op->force_delta = msg->force_delta;
   context = GNUNET_MQ_extract_nested_mh (msg);
-  op = GNUNET_new (struct Operation);
-  op->spec = spec;
 
-  // Advance generation values, so that
-  // mutations won't interfer with the running operation.
+  /* Advance generation values, so that
+     mutations won't interfer with the running operation. */
+  op->set = set;
   op->generation_created = set->current_generation;
   advance_generation (set);
-
-  op->vt = set->vt;
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                op);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Creating new CADET channel to port %s\n",
-              GNUNET_h2s (&msg->app_id));
+              "Creating new CADET channel to port %s for set operation type %u\n",
+              GNUNET_h2s (&msg->app_id),
+              set->operation);
   op->channel = GNUNET_CADET_channel_create (cadet,
                                              op,
                                              &msg->target_peer,
                                              &msg->app_id,
-                                             GNUNET_CADET_OPTION_RELIABLE);
-  op->mq = GNUNET_CADET_mq_create (op->channel);
-  set->vt->evaluate (op,
-                     context);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+                                             GNUNET_CADET_OPTION_RELIABLE,
+                                             &channel_window_cb,
+                                             &channel_end_cb,
+                                             cadet_handlers);
+  op->mq = GNUNET_CADET_get_mq (op->channel);
+  op->state = set->vt->evaluate (op,
+                                 context);
+  if (NULL == op->state)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1452,24 +1594,21 @@ handle_client_evaluate (void *cls,
  * that we only expect acks for set elements, not after the
  * #GNUNET_MESSAGE_TYPE_SET_ITER_DONE message.
  *
- * @param cls unused
- * @param client the client
- * @param m the message
+ * @param cls client the client
+ * @param ack the message
  */
 static void
 handle_client_iter_ack (void *cls,
-                        struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *m)
+                        const struct GNUNET_SET_IterAckMessage *ack)
 {
-  const struct GNUNET_SET_IterAckMessage *ack;
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set acknowledged receiving a value */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   if (NULL == set->iter)
@@ -1477,12 +1616,10 @@ handle_client_iter_ack (void *cls,
     /* client sent an ack, but we were not expecting one (as
        set iteration has finished) */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  ack = (const struct GNUNET_SET_IterAckMessage *) m;
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  GNUNET_SERVICE_client_continue (cs->client);
   if (ntohl (ack->send_more))
   {
     send_client_element (set);
@@ -1497,86 +1634,68 @@ handle_client_iter_ack (void *cls,
 
 
 /**
- * Handle a request from the client to
- * copy a set.
+ * Handle a request from the client to copy a set.
  *
- * @param cls unused
- * @param client the client
+ * @param cls the client
  * @param mh the message
  */
 static void
 handle_client_copy_lazy_prepare (void *cls,
-                                 struct GNUNET_SERVER_Client *client,
                                  const struct GNUNET_MessageHeader *mh)
 {
+  struct ClientState *cs = cls;
   struct Set *set;
   struct LazyCopyRequest *cr;
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_CopyLazyResponseMessage *resp_msg;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client requested creation of lazy copy\n");
   cr = GNUNET_new (struct LazyCopyRequest);
-
-  cr->cookie = lazy_copy_cookie;
-  lazy_copy_cookie += 1;
+  cr->cookie = ++lazy_copy_cookie;
   cr->source_set = set;
-
   GNUNET_CONTAINER_DLL_insert (lazy_copy_head,
                                lazy_copy_tail,
                                cr);
-
-
   ev = GNUNET_MQ_msg (resp_msg,
                       GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE);
   resp_msg->cookie = cr->cookie;
-  GNUNET_MQ_send (set->client_mq, ev);
-
-
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client requested lazy copy\n");
+  GNUNET_MQ_send (set->cs->mq,
+                  ev);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
 /**
- * Handle a request from the client to
- * connect to a copy of a set.
+ * Handle a request from the client to connect to a copy of a set.
  *
- * @param cls unused
- * @param client the client
- * @param mh the message
+ * @param cls the client
+ * @param msg the message
  */
 static void
 handle_client_copy_lazy_connect (void *cls,
-                                 struct GNUNET_SERVER_Client *client,
-                                 const struct GNUNET_MessageHeader *mh)
+                                 const struct GNUNET_SET_CopyLazyConnectMessage *msg)
 {
+  struct ClientState *cs = cls;
   struct LazyCopyRequest *cr;
-  const struct GNUNET_SET_CopyLazyConnectMessage *msg =
-      (const struct GNUNET_SET_CopyLazyConnectMessage *) mh;
   struct Set *set;
   int found;
 
-  if (NULL != set_get (client))
+  if (NULL != cs->set)
   {
     /* There can only be one set per client */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
   found = GNUNET_NO;
-
   for (cr = lazy_copy_head; NULL != cr; cr = cr->next)
   {
     if (cr->cookie == msg->cookie)
@@ -1585,21 +1704,20 @@ handle_client_copy_lazy_connect (void *cls,
       break;
     }
   }
-
   if (GNUNET_NO == found)
   {
     /* client asked for copy with cookie we don't know */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
   GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
                                lazy_copy_tail,
                                cr);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client %p requested use of lazy copy\n",
+              cs);
   set = GNUNET_new (struct Set);
-
   switch (cr->source_set->operation)
   {
   case GNUNET_SET_OPERATION_INTERSECTION:
@@ -1619,75 +1737,57 @@ handle_client_copy_lazy_connect (void *cls,
     GNUNET_break (0);
     GNUNET_free (set);
     GNUNET_free (cr);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
 
   set->operation = cr->source_set->operation;
-  set->state = set->vt->copy_state (cr->source_set);
+  set->state = set->vt->copy_state (cr->source_set->state);
   set->content = cr->source_set->content;
-  set->content->refcount += 1;
+  set->content->refcount++;
 
   set->current_generation = cr->source_set->current_generation;
   set->excluded_generations_size = cr->source_set->excluded_generations_size;
-  set->excluded_generations = GNUNET_memdup (cr->source_set->excluded_generations,
-                                             set->excluded_generations_size * sizeof (struct GenerationRange));
+  set->excluded_generations
+    = GNUNET_memdup (cr->source_set->excluded_generations,
+                     set->excluded_generations_size * sizeof (struct GenerationRange));
 
   /* Advance the generation of the new set, so that mutations to the
      of the cloned set and the source set are independent. */
   advance_generation (set);
-
-
-  set->client = client;
-  set->client_mq = GNUNET_MQ_queue_for_server_client (client);
-  GNUNET_CONTAINER_DLL_insert (sets_head,
-                               sets_tail,
-                               set);
-
+  set->cs = cs;
+  cs->set = set;
   GNUNET_free (cr);
-
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client connected to lazy set\n");
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
 /**
- * Handle a request from the client to
- * cancel a running set operation.
+ * Handle a request from the client to cancel a running set operation.
  *
- * @param cls unused
- * @param client the client
- * @param mh the message
+ * @param cls the client
+ * @param msg the message
  */
 static void
 handle_client_cancel (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *mh)
+                      const struct GNUNET_SET_CancelMessage *msg)
 {
-  const struct GNUNET_SET_CancelMessage *msg =
-      (const struct GNUNET_SET_CancelMessage *) mh;
+  struct ClientState *cs = cls;
   struct Set *set;
   struct Operation *op;
   int found;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client requested cancel for op %u\n",
-              ntohl (msg->request_id));
   found = GNUNET_NO;
   for (op = set->ops_head; NULL != op; op = op->next)
   {
-    if (op->spec->client_request_id == ntohl (msg->request_id))
+    if (op->client_request_id == ntohl (msg->request_id))
     {
       found = GNUNET_YES;
       break;
@@ -1700,16 +1800,19 @@ handle_client_cancel (void *cls,
      * yet and try to cancel the (just barely non-existent) operation.
      * So this is not a hard error.
      */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client canceled non-existent op\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client canceled non-existent op %u\n",
+                (uint32_t) ntohl (msg->request_id));
   }
   else
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client requested cancel for op %u\n",
+                (uint32_t) ntohl (msg->request_id));
     _GSS_operation_destroy (op,
                             GNUNET_YES);
   }
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1718,28 +1821,25 @@ handle_client_cancel (void *cls,
  * came from a remote peer.  We forward the accept to the associated
  * operation for handling
  *
- * @param cls unused
- * @param client the client
- * @param mh the message
+ * @param cls the client
+ * @param msg the message
  */
 static void
 handle_client_accept (void *cls,
-                      struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *mh)
+                      const struct GNUNET_SET_AcceptMessage *msg)
 {
+  struct ClientState *cs = cls;
   struct Set *set;
-  const struct GNUNET_SET_AcceptMessage *msg;
   struct Operation *op;
   struct GNUNET_SET_ResultMessage *result_message;
   struct GNUNET_MQ_Envelope *ev;
+  struct Listener *listener;
 
-  msg = (const struct GNUNET_SET_AcceptMessage *) mh;
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested to accept */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   op = get_incoming (ntohl (msg->accept_reject_id));
@@ -1747,270 +1847,170 @@ handle_client_accept (void *cls,
   {
     /* It is not an error if the set op does not exist -- it may
      * have been destroyed when the partner peer disconnected. */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client accepted request that is no longer active\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client %p accepted request %u of listener %p that is no longer active\n",
+                cs,
+                ntohl (msg->accept_reject_id),
+                cs->listener);
     ev = GNUNET_MQ_msg (result_message,
                         GNUNET_MESSAGE_TYPE_SET_RESULT);
     result_message->request_id = msg->request_id;
-    result_message->element_type = 0;
     result_message->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-    GNUNET_MQ_send (set->client_mq, ev);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
+    GNUNET_MQ_send (set->cs->mq,
+                    ev);
+    GNUNET_SERVICE_client_continue (cs->client);
     return;
   }
-
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client accepting request %u\n",
-              ntohl (msg->accept_reject_id));
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  op->is_incoming = GNUNET_NO;
-  GNUNET_CONTAINER_DLL_remove (incoming_head,
-                               incoming_tail,
+              (uint32_t) ntohl (msg->accept_reject_id));
+  listener = op->listener;
+  op->listener = NULL;
+  GNUNET_CONTAINER_DLL_remove (listener->op_head,
+                               listener->op_tail,
                                op);
-  op->spec->set = set;
+  op->set = set;
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                op);
-  op->spec->client_request_id = ntohl (msg->request_id);
-  op->spec->result_mode = ntohl (msg->result_mode);
-
-  // Advance generation values, so that
-  // mutations won't interfer with the running operation.
+  op->client_request_id = ntohl (msg->request_id);
+  op->result_mode = ntohl (msg->result_mode);
+  op->byzantine = msg->byzantine;
+  op->byzantine_lower_bound = msg->byzantine_lower_bound;
+  op->force_full = msg->force_full;
+  op->force_delta = msg->force_delta;
+
+  /* Advance generation values, so that future mutations do not
+     interfer with the running operation. */
   op->generation_created = set->current_generation;
   advance_generation (set);
-
-  op->vt = set->vt;
-  op->vt->accept (op);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_OK);
+  GNUNET_assert (NULL == op->state);
+  op->state = set->vt->accept (op);
+  if (NULL == op->state)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
+  /* Now allow CADET to continue, as we did not do this in
+     #handle_incoming_msg (as we wanted to first see if the
+     local client would accept the request). */
+  GNUNET_CADET_receive_done (op->channel);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
 /**
  * Called to clean up, after a shutdown has been requested.
  *
- * @param cls closure
+ * @param cls closure, NULL
  */
 static void
 shutdown_task (void *cls)
 {
-  while (NULL != incoming_head)
-    incoming_destroy (incoming_head);
-  while (NULL != listeners_head)
-    listener_destroy (listeners_head);
-  while (NULL != sets_head)
-    set_destroy (sets_head);
-
-  /* it's important to destroy cadet at the end, as all channels
-   * must be destroyed before the cadet handle! */
+  /* Delay actual shutdown to allow service to disconnect clients */
   if (NULL != cadet)
   {
     GNUNET_CADET_disconnect (cadet);
     cadet = NULL;
   }
-  GNUNET_STATISTICS_destroy (_GSS_statistics, GNUNET_YES);
+  GNUNET_STATISTICS_destroy (_GSS_statistics,
+                             GNUNET_YES);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "handled shutdown request\n");
 }
 
 
 /**
- * Function called whenever a channel is destroyed.  Should clean up
- * any associated state.  It must NOT call
- * GNUNET_CADET_channel_destroy() on the channel.
- *
- * The peer_disconnect function is part of a a virtual table set initially either
- * when a peer creates a new channel with us, or once we create
- * a new channel ourselves (evaluate).
- *
- * Once we know the exact type of operation (union/intersection), the vt is
- * replaced with an operation specific instance (_GSS_[op]_vt).
- *
- * @param cls closure (set from GNUNET_CADET_connect())
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
- */
-static void
-channel_end_cb (void *cls,
-                const struct GNUNET_CADET_Channel *channel,
-                void *channel_ctx)
-{
-  struct Operation *op = channel_ctx;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "channel_end_cb called\n");
-  op->channel = NULL;
-  op->keep++;
-  /* the vt can be null if a client already requested canceling op. */
-  if (NULL != op->vt)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "calling peer disconnect due to channel end\n");
-    op->vt->peer_disconnect (op);
-  }
-  op->keep--;
-  if (0 == op->keep)
-  {
-    /* cadet will never call us with the context again! */
-    GNUNET_free (op);
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "channel_end_cb finished\n");
-}
-
-
-/**
- * Functions with this signature are called whenever a message is
- * received via a cadet channel.
- *
- * The msg_handler is a virtual table set in initially either when a peer
- * creates a new channel with us, or once we create a new channel
- * ourselves (evaluate).
- *
- * Once we know the exact type of operation (union/intersection), the vt is
- * replaced with an operation specific instance (_GSS_[op]_vt).
- *
- * @param cls Closure (set from GNUNET_CADET_connect()).
- * @param channel Connection to the other end.
- * @param channel_ctx Place to store local state associated with the channel.
- * @param message The actual message.
- * @return #GNUNET_OK to keep the channel open,
- *         #GNUNET_SYSERR to close it (signal serious error).
- */
-static int
-dispatch_p2p_message (void *cls,
-                      struct GNUNET_CADET_Channel *channel,
-                      void **channel_ctx,
-                      const struct GNUNET_MessageHeader *message)
-{
-  struct Operation *op = *channel_ctx;
-  int ret;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Dispatching cadet message (type: %u)\n",
-              ntohs (message->type));
-  /* do this before the handler, as the handler might kill the channel */
-  GNUNET_CADET_receive_done (channel);
-  if (NULL != op->vt)
-    ret = op->vt->msg_handler (op,
-                               message);
-  else
-    ret = GNUNET_SYSERR;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Handled cadet message (type: %u)\n",
-              ntohs (message->type));
-  return ret;
-}
-
-
-/**
  * Function called by the service's run
  * method to run service-specific setup code.
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
 run (void *cls,
-     struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *cfg)
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler server_handlers[] = {
-    { &handle_client_accept, NULL,
-      GNUNET_MESSAGE_TYPE_SET_ACCEPT,
-      sizeof (struct GNUNET_SET_AcceptMessage)},
-    { &handle_client_iter_ack, NULL,
-      GNUNET_MESSAGE_TYPE_SET_ITER_ACK,
-      sizeof (struct GNUNET_SET_IterAckMessage) },
-    { &handle_client_mutation, NULL,
-      GNUNET_MESSAGE_TYPE_SET_ADD,
-      0},
-    { &handle_client_create_set, NULL,
-      GNUNET_MESSAGE_TYPE_SET_CREATE,
-      sizeof (struct GNUNET_SET_CreateMessage)},
-    { &handle_client_iterate, NULL,
-      GNUNET_MESSAGE_TYPE_SET_ITER_REQUEST,
-      sizeof (struct GNUNET_MessageHeader)},
-    { &handle_client_evaluate, NULL,
-      GNUNET_MESSAGE_TYPE_SET_EVALUATE,
-      0},
-    { &handle_client_listen, NULL,
-      GNUNET_MESSAGE_TYPE_SET_LISTEN,
-      sizeof (struct GNUNET_SET_ListenMessage)},
-    { &handle_client_reject, NULL,
-      GNUNET_MESSAGE_TYPE_SET_REJECT,
-      sizeof (struct GNUNET_SET_RejectMessage)},
-    { &handle_client_mutation, NULL,
-      GNUNET_MESSAGE_TYPE_SET_REMOVE,
-      0},
-    { &handle_client_cancel, NULL,
-      GNUNET_MESSAGE_TYPE_SET_CANCEL,
-      sizeof (struct GNUNET_SET_CancelMessage)},
-    { &handle_client_copy_lazy_prepare, NULL,
-      GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE,
-      sizeof (struct GNUNET_MessageHeader)},
-    { &handle_client_copy_lazy_connect, NULL,
-      GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT,
-      sizeof (struct GNUNET_SET_CopyLazyConnectMessage)},
-    { NULL, NULL, 0, 0}
-  };
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF, 0},
-    { &dispatch_p2p_message, GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE, 0},
-    {NULL, 0, 0}
-  };
-
-  configuration = cfg;
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-  GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect,
-                                   NULL);
-  GNUNET_SERVER_add_handlers (server,
-                              server_handlers);
-  _GSS_statistics = GNUNET_STATISTICS_create ("set", cfg);
-  cadet = GNUNET_CADET_connect (cfg,
-                                NULL,
-                                &channel_end_cb,
-                                cadet_handlers);
+  /* FIXME: need to modify SERVICE (!) API to allow
+     us to run a shutdown task *after* clients were
+     forcefully disconnected! */
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  _GSS_statistics = GNUNET_STATISTICS_create ("set",
+                                              cfg);
+  cadet = GNUNET_CADET_connect (cfg);
   if (NULL == cadet)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                _("Could not connect to cadet service\n"));
+                _("Could not connect to CADET service\n"));
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
 }
 
 
 /**
- * The main function for the set service.
- *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * Define "main" method using service macro.
  */
-int
-main (int argc,
-      char *const *argv)
-{
-  int ret;
+GNUNET_SERVICE_MAIN
+("set",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (client_accept,
+                          GNUNET_MESSAGE_TYPE_SET_ACCEPT,
+                          struct GNUNET_SET_AcceptMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_iter_ack,
+                          GNUNET_MESSAGE_TYPE_SET_ITER_ACK,
+                          struct GNUNET_SET_IterAckMessage,
+                          NULL),
+ GNUNET_MQ_hd_var_size (client_mutation,
+                        GNUNET_MESSAGE_TYPE_SET_ADD,
+                        struct GNUNET_SET_ElementMessage,
+                        NULL),
+ GNUNET_MQ_hd_fixed_size (client_create_set,
+                          GNUNET_MESSAGE_TYPE_SET_CREATE,
+                          struct GNUNET_SET_CreateMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_iterate,
+                          GNUNET_MESSAGE_TYPE_SET_ITER_REQUEST,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_var_size (client_evaluate,
+                        GNUNET_MESSAGE_TYPE_SET_EVALUATE,
+                        struct GNUNET_SET_EvaluateMessage,
+                        NULL),
+ GNUNET_MQ_hd_fixed_size (client_listen,
+                          GNUNET_MESSAGE_TYPE_SET_LISTEN,
+                          struct GNUNET_SET_ListenMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_reject,
+                          GNUNET_MESSAGE_TYPE_SET_REJECT,
+                          struct GNUNET_SET_RejectMessage,
+                          NULL),
+ GNUNET_MQ_hd_var_size (client_mutation,
+                        GNUNET_MESSAGE_TYPE_SET_REMOVE,
+                        struct GNUNET_SET_ElementMessage,
+                        NULL),
+ GNUNET_MQ_hd_fixed_size (client_cancel,
+                          GNUNET_MESSAGE_TYPE_SET_CANCEL,
+                          struct GNUNET_SET_CancelMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_copy_lazy_prepare,
+                          GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (client_copy_lazy_connect,
+                          GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT,
+                          struct GNUNET_SET_CopyLazyConnectMessage,
+                          NULL),
+ GNUNET_MQ_handler_end ());
 
-  ret = GNUNET_SERVICE_run (argc, argv, "set",
-                            GNUNET_SERVICE_OPTION_NONE,
-                            &run, NULL);
-  return (GNUNET_OK == ret) ? 0 : 1;
-}
 
 /* end of gnunet-service-set.c */
diff --git a/src/set/gnunet-service-set.h b/src/set/gnunet-service-set.h
index 9e1ffd01a..19413fd30 100644
--- a/src/set/gnunet-service-set.h
+++ b/src/set/gnunet-service-set.h
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -68,68 +68,13 @@ struct Operation;
 
 
 /**
- * Detail information about an operation.
- */
-struct OperationSpecification
-{
-
-  /**
-   * The remove peer we evaluate the operation with.
-   */
-  struct GNUNET_PeerIdentity peer;
-
-  /**
-   * Application ID for the operation, used to distinguish
-   * multiple operations of the same type with the same peer.
-   */
-  struct GNUNET_HashCode app_id;
-
-  /**
-   * Context message, may be NULL.
-   */
-  struct GNUNET_MessageHeader *context_msg;
-
-  /**
-   * Set associated with the operation, NULL until the spec has been
-   * associated with a set.
-   */
-  struct Set *set;
-
-  /**
-   * Salt to use for the operation.
-   */
-  uint32_t salt;
-
-  /**
-   * Remote peers element count
-   */
-  uint32_t remote_element_count;
-
-  /**
-   * ID used to identify an operation between service and client
-   */
-  uint32_t client_request_id;
-
-  /**
-   * The type of the operation.
-   */
-  enum GNUNET_SET_OperationType operation;
-
-  /**
-   * When are elements sent to the client, and which elements are sent?
-   */
-  enum GNUNET_SET_ResultMode result_mode;
-};
-
-
-/**
  * Signature of functions that create the implementation-specific
  * state for a set supporting a specific operation.
  *
  * @return a set state specific to the supported operation, NULL on error
  */
 typedef struct SetState *
-(*CreateImpl) (void);
+(*SetCreateImpl) (void);
 
 
 /**
@@ -140,18 +85,18 @@ typedef struct SetState *
  * @param ee element message from the client
  */
 typedef void
-(*AddRemoveImpl) (struct SetState *state,
+(*SetAddRemoveImpl) (struct SetState *state,
                   struct ElementEntry *ee);
 
 
 /**
- * Signature of functions that handle disconnection of the remote
- * peer.
+ * Make a copy of a set's internal state.
  *
- * @param op the set operation, contains implementation-specific data
+ * @param state set state to copy
+ * @return copy of the internal state
  */
-typedef void
-(*PeerDisconnectImpl) (struct Operation *op);
+typedef struct SetState *
+(*SetCopyStateImpl) (struct SetState *state);
 
 
 /**
@@ -161,7 +106,7 @@ typedef void
  * @param state the set state, contains implementation-specific data
  */
 typedef void
-(*DestroySetImpl) (struct SetState *state);
+(*SetDestroyImpl) (struct SetState *state);
 
 
 /**
@@ -169,8 +114,9 @@ typedef void
  *
  * @param op operation that is created by accepting the operation,
  *        should be initialized by the implementation
+ * @return operation-specific state to keep in @a op
  */
-typedef void
+typedef struct OperationState *
 (*OpAcceptImpl) (struct Operation *op);
 
 
@@ -182,38 +128,32 @@ typedef void
  *        begin the evaluation
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
+ * @return operation-specific state to keep in @a op
  */
-typedef void
+typedef struct OperationState *
 (*OpEvaluateImpl) (struct Operation *op,
                    const struct GNUNET_MessageHeader *opaque_context);
 
-
 /**
- * Signature of functions that implement the message handling for
- * the different set operations.
+ * Signature of functions that implement operation cancelation.
+ * This includes notifying the client about the operation's final
+ * state.
  *
  * @param op operation state
- * @param msg received message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to
- *         destroy the operation and the tunnel
  */
-typedef int
-(*MsgHandlerImpl) (struct Operation *op,
-                   const struct GNUNET_MessageHeader *msg);
+typedef void
+(*OpCancelImpl) (struct Operation *op);
 
 
 /**
- * Signature of functions that implement operation cancellation
+ * Signature of functions called when the CADET channel died.
  *
  * @param op operation state
  */
 typedef void
-(*CancelImpl) (struct Operation *op);
+(*OpChannelDeathImpl) (struct Operation *op);
 
 
-typedef struct SetState *
-(*CopyStateImpl) (struct Set *op);
-
 
 /**
  * Dispatch table for a specific set operation.  Every set operation
@@ -224,49 +164,48 @@ struct SetVT
   /**
    * Callback for the set creation.
    */
-  CreateImpl create;
+  SetCreateImpl create;
 
   /**
    * Callback for element insertion
    */
-  AddRemoveImpl add;
+  SetAddRemoveImpl add;
 
   /**
    * Callback for element removal.
    */
-  AddRemoveImpl remove;
+  SetAddRemoveImpl remove;
 
   /**
-   * Callback for accepting a set operation request
+   * Callback for making a copy of a set's internal state.
    */
-  OpAcceptImpl accept;
+  SetCopyStateImpl copy_state;
 
   /**
-   * Callback for starting evaluation with a remote peer.
+   * Callback for destruction of the set state.
    */
-  OpEvaluateImpl evaluate;
+  SetDestroyImpl destroy_set;
 
   /**
-   * Callback for destruction of the set state.
+   * Callback for accepting a set operation request
    */
-  DestroySetImpl destroy_set;
+  OpAcceptImpl accept;
 
   /**
-   * Callback for handling operation-specific messages.
+   * Callback for starting evaluation with a remote peer.
    */
-  MsgHandlerImpl msg_handler;
+  OpEvaluateImpl evaluate;
 
   /**
-   * Callback for handling the remote peer's disconnect.
+   * Callback for canceling an operation.
    */
-  PeerDisconnectImpl peer_disconnect;
+  OpCancelImpl cancel;
 
   /**
-   * Callback for canceling an operation by its ID.
+   * Callback called in case the CADET channel died.
    */
-  CancelImpl cancel;
+  OpChannelDeathImpl channel_death;
 
-  CopyStateImpl copy_state;
 };
 
 
@@ -336,20 +275,56 @@ struct ElementEntry
 };
 
 
+/**
+ * A listener is inhabited by a client, and waits for evaluation
+ * requests from remote peers.
+ */
 struct Listener;
 
 
 /**
+ * State we keep per client.
+ */
+struct ClientState
+{
+  /**
+   * Set, if associated with the client, otherwise NULL.
+   */
+  struct Set *set;
+
+  /**
+   * Listener, if associated with the client, otherwise NULL.
+   */
+  struct Listener *listener;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Message queue.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+};
+
+
+/**
  * Operation context used to execute a set operation.
  */
 struct Operation
 {
+
   /**
-   * V-Table for the operation belonging to the tunnel contest.
-   *
-   * Used for all operation specific operations after receiving the ops request
+   * Kept in a DLL of the listener, if @e listener is non-NULL.
    */
-  const struct SetVT *vt;
+  struct Operation *next;
+
+  /**
+   * Kept in a DLL of the listener, if @e listener is non-NULL.
+   */
+  struct Operation *prev;
 
   /**
    * Channel to the peer.
@@ -367,11 +342,15 @@ struct Operation
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * Detail information about the set operation, including the set to
-   * use.  When 'spec' is NULL, the operation is not yet entirely
-   * initialized.
+   * Context message, may be NULL.
+   */
+  struct GNUNET_MessageHeader *context_msg;
+
+  /**
+   * Set associated with the operation, NULL until the spec has been
+   * associated with a set.
    */
-  struct OperationSpecification *spec;
+  struct Set *set;
 
   /**
    * Operation-specific operation state.  Note that the exact
@@ -381,16 +360,6 @@ struct Operation
   struct OperationState *state;
 
   /**
-   * Evaluate operations are held in a linked list.
-   */
-  struct Operation *next;
-
-  /**
-   * Evaluate operations are held in a linked list.
-   */
-  struct Operation *prev;
-
-  /**
    * The identity of the requesting peer.  Needs to
    * be stored here as the op spec might not have been created yet.
    */
@@ -403,6 +372,50 @@ struct Operation
   struct GNUNET_SCHEDULER_Task *timeout_task;
 
   /**
+   * Salt to use for the operation.
+   */
+  uint32_t salt;
+
+  /**
+   * Remote peers element count
+   */
+  uint32_t remote_element_count;
+
+  /**
+   * ID used to identify an operation between service and client
+   */
+  uint32_t client_request_id;
+
+  /**
+   * When are elements sent to the client, and which elements are sent?
+   */
+  enum GNUNET_SET_ResultMode result_mode;
+
+  /**
+   * Always use delta operation instead of sending full sets,
+   * even it it's less efficient.
+   */
+  int force_delta;
+
+  /**
+   * Always send full sets, even if delta operations would
+   * be more efficient.
+   */
+  int force_full;
+
+  /**
+   * #GNUNET_YES to fail operations where Byzantine faults
+   * are suspected
+   */
+  int byzantine;
+
+  /**
+   * Lower bound for the set size, used only when
+   * byzantine mode is enabled.
+   */
+  int byzantine_lower_bound;
+
+  /**
    * Unique request id for the request from a remote peer, sent to the
    * client, which will accept or reject the request.  Set to '0' iff
    * the request has not been suggested yet.
@@ -410,45 +423,26 @@ struct Operation
   uint32_t suggest_id;
 
   /**
-   * #GNUNET_YES if this is not a "real" set operation yet, and we still
-   * need to wait for the other peer to give us more details.
-   */
-  int is_incoming;
-
-  /**
    * Generation in which the operation handle
    * was created.
    */
   unsigned int generation_created;
 
-  /**
-   * Incremented whenever (during shutdown) some component still
-   * needs to do something with this before the operation is freed.
-   * (Used as a reference counter, but only during termination.)
-   */
-  unsigned int keep;
 };
 
 
 /**
- * SetContent stores the actual set elements,
- * which may be shared by multiple generations derived
- * from one set.
+ * SetContent stores the actual set elements, which may be shared by
+ * multiple generations derived from one set.
  */
 struct SetContent
 {
-  /**
-   * Number of references to the content.
-   */
-  unsigned int refcount;
 
   /**
    * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *`.
    */
   struct GNUNET_CONTAINER_MultiHashMap *elements;
 
-  unsigned int latest_generation;
-
   /**
    * Mutations requested by the client that we're
    * unable to execute right now because we're iterating
@@ -464,6 +458,16 @@ struct SetContent
   struct PendingMutation *pending_mutations_tail;
 
   /**
+   * Number of references to the content.
+   */
+  unsigned int refcount;
+
+  /**
+   * FIXME: document!
+   */
+  unsigned int latest_generation;
+
+  /**
    * Number of concurrently active iterators.
    */
   int iterator_count;
@@ -484,19 +488,32 @@ struct GenerationRange
 };
 
 
+/**
+ * Information about a mutation to apply to a set.
+ */
 struct PendingMutation
 {
+  /**
+   * Mutations are kept in a DLL.
+   */
   struct PendingMutation *prev;
+
+  /**
+   * Mutations are kept in a DLL.
+   */
   struct PendingMutation *next;
 
+  /**
+   * Set this mutation is about.
+   */
   struct Set *set;
 
   /**
    * Message that describes the desired mutation.
-   * May only be a GNUNET_MESSAGE_TYPE_SET_ADD or
-   * GNUNET_MESSAGE_TYPE_SET_REMOVE.
+   * May only be a #GNUNET_MESSAGE_TYPE_SET_ADD or
+   * #GNUNET_MESSAGE_TYPE_SET_REMOVE.
    */
-  struct GNUNET_MessageHeader *mutation_message;
+  struct GNUNET_SET_ElementMessage *msg;
 };
 
 
@@ -520,12 +537,13 @@ struct Set
    * Client that owns the set.  Only one client may own a set,
    * and there can only be one set per client.
    */
-  struct GNUNET_SERVER_Client *client;
+  struct ClientState *cs;
 
   /**
-   * Message queue for the client.
+   * Content, possibly shared by multiple sets,
+   * and thus reference counted.
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct SetContent *content;
 
   /**
    * Virtual table for this set.  Determined by the operation type of
@@ -558,15 +576,15 @@ struct Set
   struct Operation *ops_tail;
 
   /**
-   * Current generation, that is, number of previously executed
-   * operations and lazy copies on the underlying set content.
+   * List of generations we have to exclude, due to lazy copies.
    */
-  unsigned int current_generation;
+  struct GenerationRange *excluded_generations;
 
   /**
-   * List of generations we have to exclude, due to lazy copies.
+   * Current generation, that is, number of previously executed
+   * operations and lazy copies on the underlying set content.
    */
-  struct GenerationRange *excluded_generations;
+  unsigned int current_generation;
 
   /**
    * Number of elements in array @a excluded_generations.
@@ -579,21 +597,16 @@ struct Set
   enum GNUNET_SET_OperationType operation;
 
   /**
-   * Each @e iter is assigned a unique number, so that the client
-   * can distinguish iterations.
-   */
-  uint16_t iteration_id;
-
-  /**
    * Generation we're currently iteration over.
    */
   unsigned int iter_generation;
 
   /**
-   * Content, possibly shared by multiple sets,
-   * and thus reference counted.
+   * Each @e iter is assigned a unique number, so that the client
+   * can distinguish iterations.
    */
-  struct SetContent *content;
+  uint16_t iteration_id;
+
 };
 
 
@@ -601,10 +614,14 @@ extern struct GNUNET_STATISTICS_Handle *_GSS_statistics;
 
 
 /**
- * Destroy the given operation.  Call the implementation-specific
- * cancel function of the operation.  Disconnects from the remote
- * peer.  Does not disconnect the client, as there may be multiple
- * operations per set.
+ * Destroy the given operation.   Used for any operation where both
+ * peers were known and that thus actually had a vt and channel.  Must
+ * not be used for operations where 'listener' is still set and we do
+ * not know the other peer.
+ *
+ * Call the implementation-specific cancel function of the operation.
+ * Disconnects from the remote peer.  Does not disconnect the client,
+ * as there may be multiple operations per set.
  *
  * @param op operation to destroy
  * @param gc #GNUNET_YES to perform garbage collection on the set
@@ -632,10 +649,13 @@ const struct SetVT *
 _GSS_intersection_vt (void);
 
 
-int
-_GSS_is_element_of_set (struct ElementEntry *ee,
-                        struct Set *set);
-
+/**
+ * Is element @a ee part of the set used by @a op?
+ *
+ * @param ee element to test
+ * @param op operation the defines the set and its generation
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 int
 _GSS_is_element_of_operation (struct ElementEntry *ee,
                               struct Operation *op);
diff --git a/src/set/gnunet-service-set_intersection.c b/src/set/gnunet-service-set_intersection.c
index 258ad6443..9dc421792 100644
--- a/src/set/gnunet-service-set_intersection.c
+++ b/src/set/gnunet-service-set_intersection.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -28,6 +28,7 @@
 #include "gnunet-service-set.h"
 #include "gnunet_block_lib.h"
 #include "gnunet-service-set_protocol.h"
+#include "gnunet-service-set_intersection.h"
 #include <gcrypt.h>
 
 
@@ -55,6 +56,18 @@ enum IntersectionOperationPhase
   PHASE_BF_EXCHANGE,
 
   /**
+   * We must next send the P2P DONE message (after finishing mostly
+   * with the local client).  Then we will wait for the channel to close.
+   */
+  PHASE_MUST_SEND_DONE,
+
+  /**
+   * We have received the P2P DONE message, and must finish with the
+   * local client before terminating the channel.
+   */
+  PHASE_DONE_RECEIVED,
+
+  /**
    * The protocol is over.  Results may still have to be sent to the
    * client.
    */
@@ -161,6 +174,13 @@ struct OperationState
    * Did we send the client that we are done?
    */
   int client_done_sent;
+
+  /**
+   * Set whenever we reach the state where the death of the
+   * channel is perfectly find and should NOT result in the
+   * operation being cancelled.
+   */
+  int channel_death_expected;
 };
 
 
@@ -192,12 +212,12 @@ send_client_removed_element (struct Operation *op,
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
-  if (GNUNET_SET_RESULT_REMOVED != op->spec->result_mode)
+  if (GNUNET_SET_RESULT_REMOVED != op->result_mode)
     return; /* Wrong mode for transmitting removed elements */
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Sending removed element (size %u) to client\n",
               element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm,
                             element->size,
                             GNUNET_MESSAGE_TYPE_SET_RESULT);
@@ -207,12 +227,12 @@ send_client_removed_element (struct Operation *op,
     return;
   }
   rm->result_status = htons (GNUNET_SET_STATUS_OK);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->element_type = element->element_type;
   GNUNET_memcpy (&rm[1],
-          element->data,
-          element->size);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+                 element->data,
+                 element->size);
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
 }
 
@@ -396,9 +416,9 @@ fail_intersection_operation (struct Operation *op)
   ev = GNUNET_MQ_msg (msg,
                       GNUNET_MESSAGE_TYPE_SET_RESULT);
   msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-  msg->request_id = htonl (op->spec->client_request_id);
+  msg->request_id = htonl (op->client_request_id);
   msg->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
   _GSS_operation_destroy (op,
                           GNUNET_YES);
@@ -427,8 +447,8 @@ send_bloomfilter (struct Operation *op)
      should use more bits to maximize its set reduction
      potential and minimize overall bandwidth consumption. */
   bf_elementbits = 2 + ceil (log2((double)
-                             (op->spec->remote_element_count /
-                                   (double) op->state->my_element_count)));
+                             (op->remote_element_count /
+                              (double) op->state->my_element_count)));
   if (bf_elementbits < 1)
     bf_elementbits = 1; /* make sure k is not 0 */
   /* optimize BF-size to ~50% of bits set */
@@ -514,12 +534,14 @@ send_client_done_and_destroy (void *cls)
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Intersection succeeded, sending DONE to local client\n");
   ev = GNUNET_MQ_msg (rm,
                       GNUNET_MESSAGE_TYPE_SET_RESULT);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->result_status = htons (GNUNET_SET_STATUS_DONE);
   rm->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
   _GSS_operation_destroy (op,
                           GNUNET_YES);
@@ -527,6 +549,53 @@ send_client_done_and_destroy (void *cls)
 
 
 /**
+ * Remember that we are done dealing with the local client
+ * AND have sent the other peer our message that we are done,
+ * so we are not just waiting for the channel to die before
+ * telling the local client that we are done as our last act.
+ *
+ * @param cls the `struct Operation`.
+ */
+static void
+finished_local_operations (void *cls)
+{
+  struct Operation *op = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "DONE sent to other peer, now waiting for other end to close the channel\n");
+  op->state->phase = PHASE_FINISHED;
+  op->state->channel_death_expected = GNUNET_YES;
+}
+
+
+/**
+ * Notify the other peer that we are done.  Once this message
+ * is out, we still need to notify the local client that we
+ * are done.
+ *
+ * @param op operation to notify for.
+ */
+static void
+send_p2p_done (struct Operation *op)
+{
+  struct GNUNET_MQ_Envelope *ev;
+  struct IntersectionDoneMessage *idm;
+
+  GNUNET_assert (PHASE_MUST_SEND_DONE == op->state->phase);
+  GNUNET_assert (GNUNET_NO == op->state->channel_death_expected);
+  ev = GNUNET_MQ_msg (idm,
+                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE);
+  idm->final_element_count = htonl (op->state->my_element_count);
+  idm->element_xor_hash = op->state->my_xor;
+  GNUNET_MQ_notify_sent (ev,
+                         &finished_local_operations,
+                         op);
+  GNUNET_MQ_send (op->mq,
+                  ev);
+}
+
+
+/**
  * Send all elements in the full result iterator.
  *
  * @param cls the `struct Operation *`
@@ -549,8 +618,21 @@ send_remaining_elements (void *cls)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sending done and destroy because iterator ran out\n");
-    op->keep--;
-    send_client_done_and_destroy (op);
+    GNUNET_CONTAINER_multihashmap_iterator_destroy (op->state->full_result_iter);
+    op->state->full_result_iter = NULL;
+    if (PHASE_DONE_RECEIVED == op->state->phase)
+    {
+      op->state->phase = PHASE_FINISHED;
+      send_client_done_and_destroy (op);
+    }
+    else if (PHASE_MUST_SEND_DONE == op->state->phase)
+    {
+      send_p2p_done (op);
+    }
+    else
+    {
+      GNUNET_assert (0);
+    }
     return;
   }
   ee = nxt;
@@ -559,48 +641,136 @@ send_remaining_elements (void *cls)
               "Sending element %s:%u to client (full set)\n",
               GNUNET_h2s (&ee->element_hash),
               element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm,
                             element->size,
                             GNUNET_MESSAGE_TYPE_SET_RESULT);
   GNUNET_assert (NULL != ev);
   rm->result_status = htons (GNUNET_SET_STATUS_OK);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->element_type = element->element_type;
   GNUNET_memcpy (&rm[1],
-          element->data,
-          element->size);
+                 element->data,
+                 element->size);
   GNUNET_MQ_notify_sent (ev,
                          &send_remaining_elements,
                          op);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
 }
 
 
 /**
- * Inform the peer that this operation is complete.
+ * Fills the "my_elements" hashmap with the initial set of
+ * (non-deleted) elements from the set of the specification.
  *
- * @param op the intersection operation to fail
+ * @param cls closure with the `struct Operation *`
+ * @param key current key code for the element
+ * @param value value in the hash map with the `struct ElementEntry *`
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+initialize_map_unfiltered (void *cls,
+                           const struct GNUNET_HashCode *key,
+                           void *value)
+{
+  struct ElementEntry *ee = value;
+  struct Operation *op = cls;
+
+  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
+    return GNUNET_YES; /* element not live in operation's generation */
+  GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
+                          &ee->element_hash,
+                          &op->state->my_xor);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initial full initialization of my_elements, adding %s:%u\n",
+              GNUNET_h2s (&ee->element_hash),
+              ee->element.size);
+  GNUNET_break (GNUNET_YES ==
+                GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
+                                                   &ee->element_hash,
+                                                   ee,
+                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  return GNUNET_YES;
+}
+
+
+/**
+ * Send our element count to the peer, in case our element count is
+ * lower than his.
+ *
+ * @param op intersection operation
  */
 static void
-send_peer_done (struct Operation *op)
+send_element_count (struct Operation *op)
 {
   struct GNUNET_MQ_Envelope *ev;
-  struct IntersectionDoneMessage *idm;
+  struct IntersectionElementInfoMessage *msg;
 
-  op->state->phase = PHASE_FINISHED;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Intersection succeeded, sending DONE\n");
-  GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
-  op->state->local_bf = NULL;
+              "Sending our element count (%u)\n",
+              op->state->my_element_count);
+  ev = GNUNET_MQ_msg (msg,
+                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO);
+  msg->sender_element_count = htonl (op->state->my_element_count);
+  GNUNET_MQ_send (op->mq, ev);
+}
 
-  ev = GNUNET_MQ_msg (idm,
-                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE);
-  idm->final_element_count = htonl (op->state->my_element_count);
-  idm->element_xor_hash = op->state->my_xor;
-  GNUNET_MQ_send (op->mq,
-                  ev);
+
+/**
+ * We go first, initialize our map with all elements and
+ * send the first Bloom filter.
+ *
+ * @param op operation to start exchange for
+ */
+static void
+begin_bf_exchange (struct Operation *op)
+{
+  op->state->phase = PHASE_BF_EXCHANGE;
+  GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                         &initialize_map_unfiltered,
+                                         op);
+  send_bloomfilter (op);
+}
+
+
+/**
+ * Handle the initial `struct IntersectionElementInfoMessage` from a
+ * remote peer.
+ *
+ * @param cls the intersection operation
+ * @param mh the header of the message
+ */
+void
+handle_intersection_p2p_element_info (void *cls,
+                                      const struct IntersectionElementInfoMessage *msg)
+{
+  struct Operation *op = cls;
+
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  }
+  op->remote_element_count = ntohl (msg->sender_element_count);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received remote element count (%u), I have %u\n",
+              op->remote_element_count,
+              op->state->my_element_count);
+  if ( ( (PHASE_INITIAL != op->state->phase) &&
+         (PHASE_COUNT_SENT != op->state->phase) ) ||
+       (op->state->my_element_count > op->remote_element_count) ||
+       (0 == op->state->my_element_count) ||
+       (0 == op->remote_element_count) )
+  {
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  }
+  GNUNET_break (NULL == op->state->remote_bf);
+  begin_bf_exchange (op);
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -615,9 +785,9 @@ process_bf (struct Operation *op)
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received BF in phase %u, foreign count is %u, my element count is %u/%u\n",
               op->state->phase,
-              op->spec->remote_element_count,
+              op->remote_element_count,
               op->state->my_element_count,
-              GNUNET_CONTAINER_multihashmap_size (op->spec->set->content->elements));
+              GNUNET_CONTAINER_multihashmap_size (op->set->content->elements));
   switch (op->state->phase)
   {
   case PHASE_INITIAL:
@@ -627,11 +797,8 @@ process_bf (struct Operation *op)
   case PHASE_COUNT_SENT:
     /* This is the first BF being sent, build our initial map with
        filtering in place */
-    op->state->my_elements
-      = GNUNET_CONTAINER_multihashmap_create (op->spec->remote_element_count,
-                                              GNUNET_YES);
     op->state->my_element_count = 0;
-    GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
+    GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
                                            &filtered_map_initialization,
                                            op);
     break;
@@ -641,6 +808,14 @@ process_bf (struct Operation *op)
                                            &iterator_bf_reduce,
                                            op);
     break;
+  case PHASE_MUST_SEND_DONE:
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  case PHASE_DONE_RECEIVED:
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
   case PHASE_FINISHED:
     GNUNET_break_op (0);
     fail_intersection_operation(op);
@@ -650,13 +825,28 @@ process_bf (struct Operation *op)
   op->state->remote_bf = NULL;
 
   if ( (0 == op->state->my_element_count) || /* fully disjoint */
-       ( (op->state->my_element_count == op->spec->remote_element_count) &&
+       ( (op->state->my_element_count == op->remote_element_count) &&
          (0 == memcmp (&op->state->my_xor,
                        &op->state->other_xor,
                        sizeof (struct GNUNET_HashCode))) ) )
   {
     /* we are done */
-    send_peer_done (op);
+    op->state->phase = PHASE_MUST_SEND_DONE;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Intersection succeeded, sending DONE to other peer\n");
+    GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
+    op->state->local_bf = NULL;
+    if (GNUNET_SET_RESULT_FULL == op->result_mode)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending full result set (%u elements)\n",
+                  GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
+      op->state->full_result_iter
+        = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
+      send_remaining_elements (op);
+      return;
+    }
+    send_p2p_done (op);
     return;
   }
   op->state->phase = PHASE_BF_EXCHANGE;
@@ -665,41 +855,53 @@ process_bf (struct Operation *op)
 
 
 /**
+ * Check an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_intersection_p2p_bf (void *cls,
+                           const struct BFMessage *msg)
+{
+  struct Operation *op = cls;
+
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Handle an BF message from a remote peer.
  *
  * @param cls the intersection operation
- * @param mh the header of the message
+ * @param msg the header of the message
  */
-static void
-handle_p2p_bf (void *cls,
-               const struct GNUNET_MessageHeader *mh)
+void
+handle_intersection_p2p_bf (void *cls,
+                            const struct BFMessage *msg)
 {
   struct Operation *op = cls;
-  const struct BFMessage *msg;
   uint32_t bf_size;
   uint32_t chunk_size;
   uint32_t bf_bits_per_element;
-  uint16_t msize;
 
-  msize = htons (mh->size);
-  if (msize < sizeof (struct BFMessage))
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation (op);
-    return;
-  }
-  msg = (const struct BFMessage *) mh;
   switch (op->state->phase)
   {
   case PHASE_INITIAL:
     GNUNET_break_op (0);
     fail_intersection_operation (op);
-    break;
+    return;
   case PHASE_COUNT_SENT:
   case PHASE_BF_EXCHANGE:
     bf_size = ntohl (msg->bloomfilter_total_length);
     bf_bits_per_element = ntohl (msg->bits_per_element);
-    chunk_size = msize - sizeof (struct BFMessage);
+    chunk_size = htons (msg->header.size) - sizeof (struct BFMessage);
     op->state->other_xor = msg->element_xor_hash;
     if (bf_size == chunk_size)
     {
@@ -715,9 +917,9 @@ handle_p2p_bf (void *cls,
                                              bf_size,
                                              bf_bits_per_element);
       op->state->salt = ntohl (msg->sender_mutator);
-      op->spec->remote_element_count = ntohl (msg->sender_element_count);
+      op->remote_element_count = ntohl (msg->sender_element_count);
       process_bf (op);
-      return;
+      break;
     }
     /* multipart chunk */
     if (NULL == op->state->bf_data)
@@ -728,7 +930,7 @@ handle_p2p_bf (void *cls,
       op->state->bf_bits_per_element = bf_bits_per_element;
       op->state->bf_data_offset = 0;
       op->state->salt = ntohl (msg->sender_mutator);
-      op->spec->remote_element_count = ntohl (msg->sender_element_count);
+      op->remote_element_count = ntohl (msg->sender_element_count);
     }
     else
     {
@@ -737,7 +939,7 @@ handle_p2p_bf (void *cls,
            (op->state->bf_bits_per_element != bf_bits_per_element) ||
            (op->state->bf_data_offset + chunk_size > bf_size) ||
            (op->state->salt != ntohl (msg->sender_mutator)) ||
-           (op->spec->remote_element_count != ntohl (msg->sender_element_count)) )
+           (op->remote_element_count != ntohl (msg->sender_element_count)) )
       {
         GNUNET_break_op (0);
         fail_intersection_operation (op);
@@ -764,153 +966,9 @@ handle_p2p_bf (void *cls,
   default:
     GNUNET_break_op (0);
     fail_intersection_operation (op);
-    break;
-  }
-}
-
-
-/**
- * Fills the "my_elements" hashmap with the initial set of
- * (non-deleted) elements from the set of the specification.
- *
- * @param cls closure with the `struct Operation *`
- * @param key current key code for the element
- * @param value value in the hash map with the `struct ElementEntry *`
- * @return #GNUNET_YES (we should continue to iterate)
- */
-static int
-initialize_map_unfiltered (void *cls,
-                           const struct GNUNET_HashCode *key,
-                           void *value)
-{
-  struct ElementEntry *ee = value;
-  struct Operation *op = cls;
-
-  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
-    return GNUNET_YES; /* element not live in operation's generation */
-  GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
-                          &ee->element_hash,
-                          &op->state->my_xor);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Initial full initialization of my_elements, adding %s:%u\n",
-              GNUNET_h2s (&ee->element_hash),
-              ee->element.size);
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
-                                                   &ee->element_hash,
-                                                   ee,
-                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  return GNUNET_YES;
-}
-
-
-/**
- * Send our element count to the peer, in case our element count is
- * lower than his.
- *
- * @param op intersection operation
- */
-static void
-send_element_count (struct Operation *op)
-{
-  struct GNUNET_MQ_Envelope *ev;
-  struct IntersectionElementInfoMessage *msg;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending our element count (%u)\n",
-              op->state->my_element_count);
-  ev = GNUNET_MQ_msg (msg,
-                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO);
-  msg->sender_element_count = htonl (op->state->my_element_count);
-  GNUNET_MQ_send (op->mq, ev);
-}
-
-
-/**
- * We go first, initialize our map with all elements and
- * send the first Bloom filter.
- *
- * @param op operation to start exchange for
- */
-static void
-begin_bf_exchange (struct Operation *op)
-{
-  op->state->phase = PHASE_BF_EXCHANGE;
-  op->state->my_elements
-    = GNUNET_CONTAINER_multihashmap_create (op->state->my_element_count,
-                                            GNUNET_YES);
-  GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
-                                         &initialize_map_unfiltered,
-                                         op);
-  send_bloomfilter (op);
-}
-
-
-/**
- * Handle the initial `struct IntersectionElementInfoMessage` from a
- * remote peer.
- *
- * @param cls the intersection operation
- * @param mh the header of the message
- */
-static void
-handle_p2p_element_info (void *cls,
-                         const struct GNUNET_MessageHeader *mh)
-{
-  struct Operation *op = cls;
-  const struct IntersectionElementInfoMessage *msg;
-
-  if (ntohs (mh->size) != sizeof (struct IntersectionElementInfoMessage))
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation(op);
-    return;
-  }
-  msg = (const struct IntersectionElementInfoMessage *) mh;
-  op->spec->remote_element_count = ntohl (msg->sender_element_count);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received remote element count (%u), I have %u\n",
-              op->spec->remote_element_count,
-              op->state->my_element_count);
-  if ( ( (PHASE_INITIAL != op->state->phase) &&
-         (PHASE_COUNT_SENT != op->state->phase) ) ||
-       (op->state->my_element_count > op->spec->remote_element_count) ||
-       (0 == op->state->my_element_count) ||
-       (0 == op->spec->remote_element_count) )
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation(op);
-    return;
-  }
-  GNUNET_break (NULL == op->state->remote_bf);
-  begin_bf_exchange (op);
-}
-
-
-/**
- * Send a result message to the client indicating that the operation
- * is over.  After the result done message has been sent to the
- * client, destroy the evaluate operation.
- *
- * @param op intersection operation
- */
-static void
-finish_and_destroy (struct Operation *op)
-{
-  GNUNET_assert (GNUNET_NO == op->state->client_done_sent);
-
-  if (GNUNET_SET_RESULT_FULL == op->spec->result_mode)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sending full result set (%u elements)\n",
-                GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
-    op->state->full_result_iter
-      = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
-    op->keep++;
-    send_remaining_elements (op);
     return;
   }
-  send_client_done_and_destroy (op);
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -955,28 +1013,26 @@ filter_all (void *cls,
  * @param cls the intersection operation
  * @param mh the message
  */
-static void
-handle_p2p_done (void *cls,
-                 const struct GNUNET_MessageHeader *mh)
+void
+handle_intersection_p2p_done (void *cls,
+                              const struct IntersectionDoneMessage *idm)
 {
   struct Operation *op = cls;
-  const struct IntersectionDoneMessage *idm;
 
-  if (PHASE_BF_EXCHANGE != op->state->phase)
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
   {
-    /* wrong phase to conclude? FIXME: Or should we allow this
-       if the other peer has _initially_ already an empty set? */
     GNUNET_break_op (0);
     fail_intersection_operation (op);
     return;
   }
-  if (ntohs (mh->size) != sizeof (struct IntersectionDoneMessage))
+  if (PHASE_BF_EXCHANGE != op->state->phase)
   {
+    /* wrong phase to conclude? FIXME: Or should we allow this
+       if the other peer has _initially_ already an empty set? */
     GNUNET_break_op (0);
     fail_intersection_operation (op);
     return;
   }
-  idm = (const struct IntersectionDoneMessage *) mh;
   if (0 == ntohl (idm->final_element_count))
   {
     /* other peer determined empty set is the intersection,
@@ -998,8 +1054,22 @@ handle_p2p_done (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Got IntersectionDoneMessage, have %u elements in intersection\n",
               op->state->my_element_count);
+  op->state->phase = PHASE_DONE_RECEIVED;
+  GNUNET_CADET_receive_done (op->channel);
+
+  GNUNET_assert (GNUNET_NO == op->state->client_done_sent);
+  if (GNUNET_SET_RESULT_FULL == op->result_mode)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Sending full result set to client (%u elements)\n",
+                GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
+    op->state->full_result_iter
+      = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
+    send_remaining_elements (op);
+    return;
+  }
   op->state->phase = PHASE_FINISHED;
-  finish_and_destroy (op);
+  send_client_done_and_destroy (op);
 }
 
 
@@ -1010,21 +1080,16 @@ handle_p2p_done (void *cls,
  *        begin the evaluation
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
+ * @return operation-specific state to keep in @a op
  */
-static void
+static struct OperationState *
 intersection_evaluate (struct Operation *op,
                        const struct GNUNET_MessageHeader *opaque_context)
 {
+  struct OperationState *state;
   struct GNUNET_MQ_Envelope *ev;
   struct OperationRequestMessage *msg;
 
-  op->state = GNUNET_new (struct OperationState);
-  /* we started the operation, thus we have to send the operation request */
-  op->state->phase = PHASE_INITIAL;
-  op->state->my_element_count = op->spec->set->state->current_set_element_count;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Initiating intersection operation evaluation\n");
   ev = GNUNET_MQ_msg_nested_mh (msg,
                                 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
                                 opaque_context);
@@ -1032,20 +1097,30 @@ intersection_evaluate (struct Operation *op,
   {
     /* the context message is too large!? */
     GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (op->spec->set->client);
-    return;
+    return NULL;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initiating intersection operation evaluation\n");
+  state = GNUNET_new (struct OperationState);
+  /* we started the operation, thus we have to send the operation request */
+  state->phase = PHASE_INITIAL;
+  state->my_element_count = op->set->state->current_set_element_count;
+  state->my_elements
+    = GNUNET_CONTAINER_multihashmap_create (state->my_element_count,
+                                            GNUNET_YES);
+
   msg->operation = htonl (GNUNET_SET_OPERATION_INTERSECTION);
-  msg->element_count = htonl (op->state->my_element_count);
+  msg->element_count = htonl (state->my_element_count);
   GNUNET_MQ_send (op->mq,
                   ev);
-  op->state->phase = PHASE_COUNT_SENT;
+  state->phase = PHASE_COUNT_SENT;
   if (NULL != opaque_context)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sent op request with context message\n");
   else
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sent op request without context message\n");
+  return state;
 }
 
 
@@ -1055,90 +1130,33 @@ intersection_evaluate (struct Operation *op,
  *
  * @param op operation that will be accepted as an intersection operation
  */
-static void
+static struct OperationState *
 intersection_accept (struct Operation *op)
 {
+  struct OperationState *state;
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Accepting set intersection operation\n");
-  op->state = GNUNET_new (struct OperationState);
-  op->state->phase = PHASE_INITIAL;
-  op->state->my_element_count
-    = op->spec->set->state->current_set_element_count;
-  op->state->my_elements
-    = GNUNET_CONTAINER_multihashmap_create
-    (GNUNET_MIN (op->state->my_element_count,
-                 op->spec->remote_element_count),
-     GNUNET_YES);
-  if (op->spec->remote_element_count < op->state->my_element_count)
+  state = GNUNET_new (struct OperationState);
+  state->phase = PHASE_INITIAL;
+  state->my_element_count
+    = op->set->state->current_set_element_count;
+  state->my_elements
+    = GNUNET_CONTAINER_multihashmap_create (GNUNET_MIN (state->my_element_count,
+                                                        op->remote_element_count),
+                                            GNUNET_YES);
+  op->state = state;
+  if (op->remote_element_count < state->my_element_count)
   {
     /* If the other peer (Alice) has fewer elements than us (Bob),
        we just send the count as Alice should send the first BF */
     send_element_count (op);
-    op->state->phase = PHASE_COUNT_SENT;
-    return;
+    state->phase = PHASE_COUNT_SENT;
+    return state;
   }
   /* We have fewer elements, so we start with the BF */
   begin_bf_exchange (op);
-}
-
-
-/**
- * Dispatch messages for a intersection operation.
- *
- * @param op the state of the intersection evaluate operation
- * @param mh the received message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
- */
-static int
-intersection_handle_p2p_message (struct Operation *op,
-                                 const struct GNUNET_MessageHeader *mh)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received p2p message (t: %u, s: %u)\n",
-              ntohs (mh->type), ntohs (mh->size));
-  switch (ntohs (mh->type))
-  {
-    /* this message handler is not active until after we received an
-     * operation request message, thus the ops request is not handled here
-     */
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO:
-    handle_p2p_element_info (op, mh);
-    break;
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF:
-    handle_p2p_bf (op, mh);
-    break;
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE:
-    handle_p2p_done (op, mh);
-    break;
-  default:
-    /* something wrong with cadet's message handlers? */
-    GNUNET_assert (0);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for peer-disconnects, notifies the client about the aborted
- * operation.  If we did not expect anything from the other peer, we
- * gracefully terminate the operation.
- *
- * @param op the destroyed operation
- */
-static void
-intersection_peer_disconnect (struct Operation *op)
-{
-  if (PHASE_FINISHED != op->state->phase)
-  {
-    fail_intersection_operation (op);
-    return;
-  }
-  /* the session has already been concluded */
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Other peer disconnected (finished)\n");
-  if (GNUNET_NO == op->state->client_done_sent)
-    finish_and_destroy (op);
+  return state;
 }
 
 
@@ -1168,6 +1186,11 @@ intersection_op_cancel (struct Operation *op)
     GNUNET_CONTAINER_multihashmap_destroy (op->state->my_elements);
     op->state->my_elements = NULL;
   }
+  if (NULL != op->state->full_result_iter)
+  {
+    GNUNET_CONTAINER_multihashmap_iterator_destroy (op->state->full_result_iter);
+    op->state->full_result_iter = NULL;
+  }
   GNUNET_free (op->state);
   op->state = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1236,6 +1259,28 @@ intersection_remove (struct SetState *set_state,
 
 
 /**
+ * Callback for channel death for the intersection operation.
+ *
+ * @param op operation that lost the channel
+ */
+static void
+intersection_channel_death (struct Operation *op)
+{
+  if (GNUNET_YES == op->state->channel_death_expected)
+  {
+    /* oh goodie, we are done! */
+    send_client_done_and_destroy (op);
+  }
+  else
+  {
+    /* sorry, channel went down early, too bad. */
+    _GSS_operation_destroy (op,
+                            GNUNET_YES);
+  }
+}
+
+
+/**
  * Get the table with implementing functions for set intersection.
  *
  * @return the operation specific VTable
@@ -1245,14 +1290,13 @@ _GSS_intersection_vt ()
 {
   static const struct SetVT intersection_vt = {
     .create = &intersection_set_create,
-    .msg_handler = &intersection_handle_p2p_message,
     .add = &intersection_add,
     .remove = &intersection_remove,
     .destroy_set = &intersection_set_destroy,
     .evaluate = &intersection_evaluate,
     .accept = &intersection_accept,
-    .peer_disconnect = &intersection_peer_disconnect,
     .cancel = &intersection_op_cancel,
+    .channel_death = &intersection_channel_death,
   };
 
   return &intersection_vt;
diff --git a/src/set/gnunet-service-set_intersection.h b/src/set/gnunet-service-set_intersection.h
new file mode 100644
index 000000000..3bd255142
--- /dev/null
+++ b/src/set/gnunet-service-set_intersection.h
@@ -0,0 +1,79 @@
+
+/*
+      This file is part of GNUnet
+      Copyright (C) 2013-2017 GNUnet e.V.
+
+      GNUnet is free software; you can redistribute it and/or modify
+      it under the terms of the GNU General Public License as published
+      by the Free Software Foundation; either version 3, or (at your
+      option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      General Public License for more details.
+
+      You should have received a copy of the GNU General Public License
+      along with GNUnet; see the file COPYING.  If not, write to the
+      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+      Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file set/gnunet-service-set_intersection.h
+ * @brief two-peer set operations
+ * @author Florian Dold
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_SET_INTERSECTION_H
+#define GNUNET_SERVICE_SET_INTERSECTION_H
+
+#include "gnunet-service-set.h"
+
+
+/**
+ * Check an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_intersection_p2p_bf (void *cls,
+                           const struct BFMessage *msg);
+
+
+/**
+ * Handle an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ */
+void
+handle_intersection_p2p_bf (void *cls,
+                            const struct BFMessage *msg);
+
+
+/**
+ * Handle the initial `struct IntersectionElementInfoMessage` from a
+ * remote peer.
+ *
+ * @param cls the intersection operation
+ * @param mh the header of the message
+ */
+void
+handle_intersection_p2p_element_info (void *cls,
+                                      const struct IntersectionElementInfoMessage *msg);
+
+
+/**
+ * Handle a done message from a remote peer
+ *
+ * @param cls the intersection operation
+ * @param mh the message
+ */
+void
+handle_intersection_p2p_done (void *cls,
+                              const struct IntersectionDoneMessage *idm);
+
+
+#endif
diff --git a/src/set/gnunet-service-set_protocol.h b/src/set/gnunet-service-set_protocol.h
index 748da15fc..0138b21c7 100644
--- a/src/set/gnunet-service-set_protocol.h
+++ b/src/set/gnunet-service-set_protocol.h
@@ -208,6 +208,20 @@ struct IntersectionDoneMessage
   struct GNUNET_HashCode element_xor_hash;
 };
 
+
+/**
+ * Strata estimator together with the peer's overall set size.
+ */
+struct StrataEstimatorMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE(C)
+   */
+  struct GNUNET_MessageHeader header;
+
+  uint64_t set_size;
+};
+
 GNUNET_NETWORK_STRUCT_END
 
 #endif
diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c
index e22465fd3..fc7e578e6 100644
--- a/src/set/gnunet-service-set_union.c
+++ b/src/set/gnunet-service-set_union.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013-2016 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -19,15 +19,16 @@
 */
 /**
  * @file set/gnunet-service-set_union.c
-
  * @brief two-peer set operations
  * @author Florian Dold
+ * @author Christian Grothoff
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet-service-set.h"
 #include "ibf.h"
+#include "gnunet-service-set_union.h"
 #include "gnunet-service-set_union_strata_estimator.h"
 #include "gnunet-service-set_protocol.h"
 #include <gcrypt.h>
@@ -85,6 +86,7 @@ enum UnionOperationPhase
    * upon initialization and later via #PHASE_EXPECT_ELEMENTS_AND_REQUESTS.
    *
    * XXX: could use better wording.
+   * XXX: repurposed to also expect a "request full set" message, should be renamed
    *
    * After receiving the complete IBF, we enter #PHASE_EXPECT_ELEMENTS
    */
@@ -115,14 +117,22 @@ enum UnionOperationPhase
    * In the penultimate phase,
    * we wait until all our demands
    * are satisfied.  Then we send a done
-   * message, and wait for another done message.*/
+   * message, and wait for another done message.
+   */
   PHASE_FINISH_WAITING,
 
   /**
    * In the ultimate phase, we wait until
    * our demands are satisfied and then
-   * quit (sending another DONE message). */
-  PHASE_DONE
+   * quit (sending another DONE message).
+   */
+  PHASE_DONE,
+
+  /**
+   * After sending the full set, wait for responses with the elements
+   * that the local peer is missing.
+   */
+  PHASE_FULL_SENDING,
 };
 
 
@@ -148,7 +158,7 @@ struct OperationState
   struct InvertibleBloomFilter *local_ibf;
 
   /**
-   * Maps IBF-Keys (specific to the current salt) to elements.
+   * Maps unsalted IBF-Keys to elements.
    * Used as a multihashmap, the keys being the lower 32bit of the IBF-Key.
    * Colliding IBF-Keys are linked.
    */
@@ -183,6 +193,23 @@ struct OperationState
    * Salt for the IBF we've received and that we're currently decoding.
    */
   uint32_t salt_receive;
+
+  /**
+   * Number of elements we received from the other peer
+   * that were not in the local set yet.
+   */
+  uint32_t received_fresh;
+
+  /**
+   * Total number of elements received from the other peer.
+   */
+  uint32_t received_total;
+
+  /**
+   * Initial size of our set, just before
+   * the operation started.
+   */
+  uint64_t initial_size;
 };
 
 
@@ -203,6 +230,14 @@ struct KeyEntry
    * is #GNUNET_YES.
    */
   struct ElementEntry *element;
+
+  /**
+   * Did we receive this element?
+   * Even if element->is_foreign is false, we might
+   * have received the element, so this indicates that
+   * the other peer has it.
+   */
+  int received;
 };
 
 
@@ -333,9 +368,10 @@ fail_union_operation (struct Operation *op)
        "union operation failed\n");
   ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_RESULT);
   msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-  msg->request_id = htonl (op->spec->client_request_id);
+  msg->request_id = htonl (op->client_request_id);
   msg->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
   _GSS_operation_destroy (op, GNUNET_YES);
 }
 
@@ -362,6 +398,23 @@ get_ibf_key (const struct GNUNET_HashCode *src)
 
 
 /**
+ * Context for #op_get_element_iterator
+ */
+struct GetElementContext
+{
+  /**
+   * FIXME.
+   */
+  struct GNUNET_HashCode hash;
+
+  /**
+   * FIXME.
+   */
+  struct KeyEntry *k;
+};
+
+
+/**
  * Iterator over the mapping from IBF keys to element entries.  Checks if we
  * have an element with a given GNUNET_HashCode.
  *
@@ -372,17 +425,20 @@ get_ibf_key (const struct GNUNET_HashCode *src)
  *         #GNUNET_NO if we've found the element.
  */
 static int
-op_has_element_iterator (void *cls,
+op_get_element_iterator (void *cls,
                          uint32_t key,
                          void *value)
 {
-  struct GNUNET_HashCode *element_hash = cls;
+  struct GetElementContext *ctx = cls;
   struct KeyEntry *k = value;
 
   GNUNET_assert (NULL != k);
   if (0 == GNUNET_CRYPTO_hash_cmp (&k->element->element_hash,
-                                   element_hash))
+                                   &ctx->hash))
+  {
+    ctx->k = k;
     return GNUNET_NO;
+  }
   return GNUNET_YES;
 }
 
@@ -395,23 +451,29 @@ op_has_element_iterator (void *cls,
  * @param element_hash hash of the element to look for
  * @return #GNUNET_YES if the element has been found, #GNUNET_NO otherwise
  */
-static int
-op_has_element (struct Operation *op,
+static struct KeyEntry *
+op_get_element (struct Operation *op,
                 const struct GNUNET_HashCode *element_hash)
 {
   int ret;
   struct IBF_Key ibf_key;
+  struct GetElementContext ctx = {{{ 0 }} , 0};
+
+  ctx.hash = *element_hash;
 
   ibf_key = get_ibf_key (element_hash);
   ret = GNUNET_CONTAINER_multihashmap32_get_multiple (op->state->key_to_element,
                                                       (uint32_t) ibf_key.key_val,
-                                                      op_has_element_iterator,
-                                                      (void *) element_hash);
+                                                      op_get_element_iterator,
+                                                      &ctx);
 
   /* was the iteration aborted because we found the element? */
   if (GNUNET_SYSERR == ret)
-    return GNUNET_YES;
-  return GNUNET_NO;
+  {
+    GNUNET_assert (NULL != ctx.k);
+    return ctx.k;
+  }
+  return NULL;
 }
 
 
@@ -427,10 +489,12 @@ op_has_element (struct Operation *op,
  *
  * @param op the union operation
  * @param ee the element entry
+ * @parem received was this element received from the remote peer?
  */
 static void
 op_register_element (struct Operation *op,
-                     struct ElementEntry *ee)
+                     struct ElementEntry *ee,
+                     int received)
 {
   struct IBF_Key ibf_key;
   struct KeyEntry *k;
@@ -439,6 +503,7 @@ op_register_element (struct Operation *op,
   k = GNUNET_new (struct KeyEntry);
   k->element = ee;
   k->ibf_key = ibf_key;
+  k->received = received;
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap32_put (op->state->key_to_element,
                                                       (uint32_t) ibf_key.key_val,
@@ -447,6 +512,9 @@ op_register_element (struct Operation *op,
 }
 
 
+/**
+ * FIXME.
+ */
 static void
 salt_key (const struct IBF_Key *k_in,
           uint32_t salt,
@@ -460,6 +528,9 @@ salt_key (const struct IBF_Key *k_in,
 }
 
 
+/**
+ * FIXME.
+ */
 static void
 unsalt_key (const struct IBF_Key *k_in,
             uint32_t salt,
@@ -493,7 +564,9 @@ prepare_ibf_iterator (void *cls,
        (void *) op,
        (unsigned long) ke->ibf_key.key_val,
        GNUNET_h2s (&ke->element->element_hash));
-  salt_key (&ke->ibf_key, op->state->salt_send, &salted_key);
+  salt_key (&ke->ibf_key,
+            op->state->salt_send,
+            &salted_key);
   ibf_insert (op->state->local_ibf, salted_key);
   return GNUNET_YES;
 }
@@ -519,17 +592,39 @@ init_key_to_element_iterator (void *cls,
 
   /* make sure that the element belongs to the set at the time
    * of creating the operation */
-  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
+  if (GNUNET_NO ==
+      _GSS_is_element_of_operation (ee,
+                                    op))
     return GNUNET_YES;
-
   GNUNET_assert (GNUNET_NO == ee->remote);
-
-  op_register_element (op, ee);
+  op_register_element (op,
+                       ee,
+                       GNUNET_NO);
   return GNUNET_YES;
 }
 
 
 /**
+ * Initialize the IBF key to element mapping local to this set
+ * operation.
+ *
+ * @param op the set union operation
+ */
+static void
+initialize_key_to_element (struct Operation *op)
+{
+  unsigned int len;
+
+  GNUNET_assert (NULL == op->state->key_to_element);
+  len = GNUNET_CONTAINER_multihashmap_size (op->set->content->elements);
+  op->state->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
+  GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                         &init_key_to_element_iterator,
+                                         op);
+}
+
+
+/**
  * Create an ibf with the operation's elements
  * of the specified size
  *
@@ -541,15 +636,8 @@ static int
 prepare_ibf (struct Operation *op,
              uint32_t size)
 {
-  if (NULL == op->state->key_to_element)
-  {
-    unsigned int len;
+  GNUNET_assert (NULL != op->state->key_to_element);
 
-    len = GNUNET_CONTAINER_multihashmap_size (op->spec->set->content->elements);
-    op->state->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
-    GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
-                                           init_key_to_element_iterator, op);
-  }
   if (NULL != op->state->local_ibf)
     ibf_destroy (op->state->local_ibf);
   op->state->local_ibf = ibf_create (size, SE_IBF_HASH_NUM);
@@ -639,43 +727,6 @@ send_ibf (struct Operation *op,
 
 
 /**
- * Send a strata estimator to the remote peer.
- *
- * @param op the union operation with the remote peer
- */
-static void
-send_strata_estimator (struct Operation *op)
-{
-  const struct StrataEstimator *se = op->state->se;
-  struct GNUNET_MQ_Envelope *ev;
-  struct GNUNET_MessageHeader *strata_msg;
-  char *buf;
-  size_t len;
-  uint16_t type;
-
-  buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
-  len = strata_estimator_write (op->state->se,
-                                buf);
-  if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
-    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
-  else
-    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
-  ev = GNUNET_MQ_msg_header_extra (strata_msg,
-                                   len,
-                                   type);
-  GNUNET_memcpy (&strata_msg[1],
-          buf,
-          len);
-  GNUNET_free (buf);
-  GNUNET_MQ_send (op->mq,
-                  ev);
-  op->state->phase = PHASE_EXPECT_IBF;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "sent SE, expecting IBF\n");
-}
-
-
-/**
  * Compute the necessary order of an ibf
  * from the size of the symmetric set difference.
  *
@@ -693,7 +744,68 @@ get_order_from_difference (unsigned int diff)
     ibf_order++;
   if (ibf_order > MAX_IBF_ORDER)
     ibf_order = MAX_IBF_ORDER;
-  return ibf_order;
+  // add one for correction
+  return ibf_order + 1;
+}
+
+
+/**
+ * Send a set element.
+ *
+ * @param cls the union operation `struct Operation *`
+ * @param key unused
+ * @param value the `struct ElementEntry *` to insert
+ *        into the key-to-element mapping
+ * @return #GNUNET_YES (to continue iterating)
+ */
+static int
+send_full_element_iterator (void *cls,
+                       const struct GNUNET_HashCode *key,
+                       void *value)
+{
+  struct Operation *op = cls;
+  struct GNUNET_SET_ElementMessage *emsg;
+  struct ElementEntry *ee = value;
+  struct GNUNET_SET_Element *el = &ee->element;
+  struct GNUNET_MQ_Envelope *ev;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Sending element %s\n",
+       GNUNET_h2s (key));
+  ev = GNUNET_MQ_msg_extra (emsg,
+                            el->size,
+                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
+  emsg->element_type = htons (el->element_type);
+  GNUNET_memcpy (&emsg[1],
+                 el->data,
+                 el->size);
+  GNUNET_MQ_send (op->mq,
+                  ev);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Switch to full set transmission for @a op.
+ *
+ * @param op operation to switch to full set transmission.
+ */
+static void
+send_full_set (struct Operation *op)
+{
+  struct GNUNET_MQ_Envelope *ev;
+
+  op->state->phase = PHASE_FULL_SENDING;
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Dedicing to transmit the full set\n");
+  /* FIXME: use a more memory-friendly way of doing this with an
+     iterator, just as we do in the non-full case! */
+  (void) GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                                &send_full_element_iterator,
+                                                op);
+  ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
+  GNUNET_MQ_send (op->mq,
+                  ev);
 }
 
 
@@ -701,40 +813,57 @@ get_order_from_difference (unsigned int diff)
  * Handle a strata estimator from a remote peer
  *
  * @param cls the union operation
- * @param mh the message
- * @param is_compressed #GNUNET_YES if the estimator is compressed
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param msg the message
  */
-static int
-handle_p2p_strata_estimator (void *cls,
-                             const struct GNUNET_MessageHeader *mh,
-                             int is_compressed)
+int
+check_union_p2p_strata_estimator (void *cls,
+                                  const struct StrataEstimatorMessage *msg)
 {
   struct Operation *op = cls;
-  struct StrataEstimator *remote_se;
-  int diff;
+  int is_compressed;
   size_t len;
 
-  GNUNET_STATISTICS_update (_GSS_statistics,
-                            "# bytes of SE received",
-                            ntohs (mh->size),
-                            GNUNET_NO);
-
   if (op->state->phase != PHASE_EXPECT_SE)
   {
-    fail_union_operation (op);
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  len = ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader);
+  is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (msg->header.type));
+  len = ntohs (msg->header.size) - sizeof (struct StrataEstimatorMessage);
   if ( (GNUNET_NO == is_compressed) &&
        (len != SE_STRATA_COUNT * SE_IBF_SIZE * IBF_BUCKET_SIZE) )
   {
-    fail_union_operation (op);
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_strata_estimator (void *cls,
+                                   const struct StrataEstimatorMessage *msg)
+{
+  struct Operation *op = cls;
+  struct StrataEstimator *remote_se;
+  unsigned int diff;
+  uint64_t other_size;
+  size_t len;
+  int is_compressed;
+
+  is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (msg->header.type));
+  GNUNET_STATISTICS_update (_GSS_statistics,
+                            "# bytes of SE received",
+                            ntohs (msg->header.size),
+                            GNUNET_NO);
+  len = ntohs (msg->header.size) - sizeof (struct StrataEstimatorMessage);
+  other_size = GNUNET_ntohll (msg->set_size);
   remote_se = strata_estimator_create (SE_STRATA_COUNT,
                                        SE_IBF_SIZE,
                                        SE_IBF_HASH_NUM);
@@ -742,40 +871,102 @@ handle_p2p_strata_estimator (void *cls,
   {
     /* insufficient resources, fail */
     fail_union_operation (op);
-    return GNUNET_SYSERR;
+    return;
   }
   if (GNUNET_OK !=
-      strata_estimator_read (&mh[1],
+      strata_estimator_read (&msg[1],
                              len,
                              is_compressed,
                              remote_se))
   {
     /* decompression failed */
-    fail_union_operation (op);
     strata_estimator_destroy (remote_se);
-    return GNUNET_SYSERR;
+    fail_union_operation (op);
+    return;
   }
   GNUNET_assert (NULL != op->state->se);
   diff = strata_estimator_difference (remote_se,
                                       op->state->se);
+
+  if (diff > 200)
+    diff = diff * 3 / 2;
+
   strata_estimator_destroy (remote_se);
   strata_estimator_destroy (op->state->se);
   op->state->se = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "got se diff=%d, using ibf size %d\n",
        diff,
-       1<<get_order_from_difference (diff));
-  if (GNUNET_OK !=
-      send_ibf (op,
-                get_order_from_difference (diff)))
+       1U << get_order_from_difference (diff));
+
   {
-    /* Internal error, best we can do is shut the connection */
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Failed to send IBF, closing connection\n");
+    char *set_debug;
+
+    set_debug = getenv ("GNUNET_SET_BENCHMARK");
+    if ( (NULL != set_debug) &&
+         (0 == strcmp (set_debug, "1")) )
+    {
+      FILE *f = fopen ("set.log", "a");
+      fprintf (f, "%llu\n", (unsigned long long) diff);
+      fclose (f);
+    }
+  }
+
+  if ( (GNUNET_YES == op->byzantine) &&
+       (other_size < op->byzantine_lower_bound) )
+  {
+    GNUNET_break (0);
     fail_union_operation (op);
-    return GNUNET_SYSERR;
+    return;
   }
-  return GNUNET_OK;
+
+  if ( (GNUNET_YES == op->force_full) ||
+       (diff > op->state->initial_size / 4) ||
+       (0 == other_size) )
+  {
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Deciding to go for full set transmission (diff=%d, own set=%u)\n",
+         diff,
+         op->state->initial_size);
+    GNUNET_STATISTICS_update (_GSS_statistics,
+                              "# of full sends",
+                              1,
+                              GNUNET_NO);
+    if ( (op->state->initial_size <= other_size) ||
+         (0 == other_size) )
+    {
+      send_full_set (op);
+    }
+    else
+    {
+      struct GNUNET_MQ_Envelope *ev;
+
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           "Telling other peer that we expect its full set\n");
+      op->state->phase = PHASE_EXPECT_IBF;
+      ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL);
+      GNUNET_MQ_send (op->mq,
+                      ev);
+    }
+  }
+  else
+  {
+    GNUNET_STATISTICS_update (_GSS_statistics,
+                              "# of ibf sends",
+                              1,
+                              GNUNET_NO);
+    if (GNUNET_OK !=
+        send_ibf (op,
+                  get_order_from_difference (diff)))
+    {
+      /* Internal error, best we can do is shut the connection */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to send IBF, closing connection\n");
+      fail_union_operation (op);
+      return;
+    }
+  }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -856,14 +1047,16 @@ decode_and_send (struct Operation *op)
   GNUNET_assert (PHASE_INVENTORY_ACTIVE == op->state->phase);
 
   if (GNUNET_OK !=
-      prepare_ibf (op, op->state->remote_ibf->size))
+      prepare_ibf (op,
+                   op->state->remote_ibf->size))
   {
     GNUNET_break (0);
     /* allocation failed */
     return GNUNET_SYSERR;
   }
   diff_ibf = ibf_dup (op->state->local_ibf);
-  ibf_subtract (diff_ibf, op->state->remote_ibf);
+  ibf_subtract (diff_ibf,
+                op->state->remote_ibf);
 
   ibf_destroy (op->state->remote_ibf);
   op->state->remote_ibf = NULL;
@@ -960,8 +1153,12 @@ decode_and_send (struct Operation *op)
     if (1 == side)
     {
       struct IBF_Key unsalted_key;
-      unsalt_key (&key, op->state->salt_receive, &unsalted_key);
-      send_offers_for_key (op, unsalted_key);
+
+      unsalt_key (&key,
+                  op->state->salt_receive,
+                  &unsalted_key);
+      send_offers_for_key (op,
+                           unsalted_key);
     }
     else if (-1 == side)
     {
@@ -993,99 +1190,118 @@ decode_and_send (struct Operation *op)
 
 
 /**
- * Handle an IBF message from a remote peer.
+ * Check an IBF message from a remote peer.
  *
  * Reassemble the IBF from multiple pieces, and
  * process the whole IBF once possible.
  *
  * @param cls the union operation
- * @param mh the header of the message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
  */
-static int
-handle_p2p_ibf (void *cls,
-                const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_ibf (void *cls,
+                     const struct IBFMessage *msg)
 {
   struct Operation *op = cls;
-  const struct IBFMessage *msg;
   unsigned int buckets_in_message;
 
-  if (ntohs (mh->size) < sizeof (struct IBFMessage))
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
     return GNUNET_SYSERR;
   }
-  msg = (const struct IBFMessage *) mh;
-  if ( (op->state->phase == PHASE_INVENTORY_PASSIVE) ||
-       (op->state->phase == PHASE_EXPECT_IBF) )
+  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  if (0 == buckets_in_message)
   {
-    op->state->phase = PHASE_EXPECT_IBF_CONT;
-    GNUNET_assert (NULL == op->state->remote_ibf);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Creating new ibf of size %u\n",
-         1 << msg->order);
-    op->state->remote_ibf = ibf_create (1<<msg->order, SE_IBF_HASH_NUM);
-    op->state->salt_receive = ntohl (msg->salt);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Receiving new IBF with salt %u\n", op->state->salt_receive);
-    if (NULL == op->state->remote_ibf)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  "Failed to parse remote IBF, closing connection\n");
-      fail_union_operation (op);
-      return GNUNET_SYSERR;
-    }
-    op->state->ibf_buckets_received = 0;
-    if (0 != ntohl (msg->offset))
-    {
-      GNUNET_break_op (0);
-      fail_union_operation (op);
-      return GNUNET_SYSERR;
-    }
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message * IBF_BUCKET_SIZE)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  else if (op->state->phase == PHASE_EXPECT_IBF_CONT)
+  if (op->state->phase == PHASE_EXPECT_IBF_CONT)
   {
     if (ntohl (msg->offset) != op->state->ibf_buckets_received)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
     if (1<<msg->order != op->state->remote_ibf->size)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
     if (ntohl (msg->salt) != op->state->salt_receive)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
   }
-  else
+  else if ( (op->state->phase != PHASE_INVENTORY_PASSIVE) &&
+            (op->state->phase != PHASE_EXPECT_IBF) )
   {
-    GNUNET_assert (0);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
 
-  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  return GNUNET_OK;
+}
 
-  if (0 == buckets_in_message)
+
+/**
+ * Handle an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ */
+void
+handle_union_p2p_ibf (void *cls,
+                      const struct IBFMessage *msg)
+{
+  struct Operation *op = cls;
+  unsigned int buckets_in_message;
+
+  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  if ( (op->state->phase == PHASE_INVENTORY_PASSIVE) ||
+       (op->state->phase == PHASE_EXPECT_IBF) )
   {
-    GNUNET_break_op (0);
-    fail_union_operation (op);
-    return GNUNET_SYSERR;
+    op->state->phase = PHASE_EXPECT_IBF_CONT;
+    GNUNET_assert (NULL == op->state->remote_ibf);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Creating new ibf of size %u\n",
+         1 << msg->order);
+    op->state->remote_ibf = ibf_create (1<<msg->order, SE_IBF_HASH_NUM);
+    op->state->salt_receive = ntohl (msg->salt);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Receiving new IBF with salt %u\n",
+         op->state->salt_receive);
+    if (NULL == op->state->remote_ibf)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to parse remote IBF, closing connection\n");
+      fail_union_operation (op);
+      return;
+    }
+    op->state->ibf_buckets_received = 0;
+    if (0 != ntohl (msg->offset))
+    {
+      GNUNET_break_op (0);
+      fail_union_operation (op);
+      return;
+    }
   }
-
-  if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message * IBF_BUCKET_SIZE)
+  else
   {
-    GNUNET_break_op (0);
-    fail_union_operation (op);
-    return GNUNET_SYSERR;
+    GNUNET_assert (op->state->phase == PHASE_EXPECT_IBF_CONT);
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Received more of IBF\n");
   }
-
   GNUNET_assert (NULL != op->state->remote_ibf);
 
   ibf_read_slice (&msg[1],
@@ -1105,10 +1321,11 @@ handle_p2p_ibf (void *cls,
       /* Internal error, best we can do is shut down */
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Failed to decode IBF, closing connection\n");
-      return GNUNET_SYSERR;
+      fail_union_operation (op);
+      return;
     }
   }
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1131,7 +1348,7 @@ send_client_element (struct Operation *op,
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "sending element (size %u) to client\n",
        element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm, element->size, GNUNET_MESSAGE_TYPE_SET_RESULT);
   if (NULL == ev)
   {
@@ -1140,10 +1357,14 @@ send_client_element (struct Operation *op,
     return;
   }
   rm->result_status = htons (status);
-  rm->request_id = htonl (op->spec->client_request_id);
-  rm->element_type = element->element_type;
-  GNUNET_memcpy (&rm[1], element->data, element->size);
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  rm->request_id = htonl (op->client_request_id);
+  rm->element_type = htons (element->element_type);
+  rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element));
+  GNUNET_memcpy (&rm[1],
+                 element->data,
+                 element->size);
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
 }
 
 
@@ -1160,16 +1381,27 @@ send_done_and_destroy (void *cls)
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
-  ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SET_RESULT);
-  rm->request_id = htonl (op->spec->client_request_id);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Signalling client that union operation is done\n");
+  ev = GNUNET_MQ_msg (rm,
+                      GNUNET_MESSAGE_TYPE_SET_RESULT);
+  rm->request_id = htonl (op->client_request_id);
   rm->result_status = htons (GNUNET_SET_STATUS_DONE);
   rm->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element));
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
   /* Will also call the union-specific cancel function. */
-  _GSS_operation_destroy (op, GNUNET_YES);
+  _GSS_operation_destroy (op,
+                          GNUNET_YES);
 }
 
 
+/**
+ * Tests if the operation is finished, and if so notify.
+ *
+ * @param op operation to check
+ */
 static void
 maybe_finish (struct Operation *op)
 {
@@ -1188,8 +1420,8 @@ maybe_finish (struct Operation *op)
 
       op->state->phase = PHASE_DONE;
       ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE);
-      GNUNET_MQ_send (op->mq, ev);
-
+      GNUNET_MQ_send (op->mq,
+                      ev);
       /* We now wait until the other peer closes the channel
        * after it got all elements from us. */
     }
@@ -1209,44 +1441,59 @@ maybe_finish (struct Operation *op)
 
 
 /**
- * Handle an element message from a remote peer.
+ * Check an element message from a remote peer.
  *
  * @param cls the union operation
- * @param mh the message
+ * @param emsg the message
  */
-static void
-handle_p2p_elements (void *cls,
-                     const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_elements (void *cls,
+                          const struct GNUNET_SET_ElementMessage *emsg)
 {
   struct Operation *op = cls;
-  struct ElementEntry *ee;
-  const struct GNUNET_SET_ElementMessage *emsg;
-  uint16_t element_size;
 
-  if (0 == GNUNET_CONTAINER_multihashmap_size (op->state->demanded_hashes))
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
-  if (ntohs (mh->size) < sizeof (struct GNUNET_SET_ElementMessage))
+  if (0 == GNUNET_CONTAINER_multihashmap_size (op->state->demanded_hashes))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  emsg = (const struct GNUNET_SET_ElementMessage *) mh;
 
-  element_size = ntohs (mh->size) - sizeof (struct GNUNET_SET_ElementMessage);
+/**
+ * Handle an element message from a remote peer.
+ * Sent by the other peer either because we decoded an IBF and placed a demand,
+ * or because the other peer switched to full set transmission.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_elements (void *cls,
+                           const struct GNUNET_SET_ElementMessage *emsg)
+{
+  struct Operation *op = cls;
+  struct ElementEntry *ee;
+  struct KeyEntry *ke;
+  uint16_t element_size;
+
+  element_size = ntohs (emsg->header.size) - sizeof (struct GNUNET_SET_ElementMessage);
   ee = GNUNET_malloc (sizeof (struct ElementEntry) + element_size);
-  GNUNET_memcpy (&ee[1], &emsg[1], element_size);
+  GNUNET_memcpy (&ee[1],
+                 &emsg[1],
+                 element_size);
   ee->element.size = element_size;
   ee->element.data = &ee[1];
   ee->element.element_type = ntohs (emsg->element_type);
   ee->remote = GNUNET_YES;
-  GNUNET_SET_element_hash (&ee->element, &ee->element_hash);
-
+  GNUNET_SET_element_hash (&ee->element,
+                           &ee->element_hash);
   if (GNUNET_NO ==
       GNUNET_CONTAINER_multihashmap_remove (op->state->demanded_hashes,
                                             &ee->element_hash,
@@ -1254,7 +1501,6 @@ handle_p2p_elements (void *cls,
   {
     /* We got something we didn't demand, since it's not in our map. */
     GNUNET_break_op (0);
-    GNUNET_free (ee);
     fail_union_operation (op);
     return;
   }
@@ -1273,7 +1519,10 @@ handle_p2p_elements (void *cls,
                             1,
                             GNUNET_NO);
 
-  if (GNUNET_YES == op_has_element (op, &ee->element_hash))
+  op->state->received_total++;
+
+  ke = op_get_element (op, &ee->element_hash);
+  if (NULL != ke)
   {
     /* Got repeated element.  Should not happen since
      * we track demands. */
@@ -1281,15 +1530,17 @@ handle_p2p_elements (void *cls,
                               "# repeated elements",
                               1,
                               GNUNET_NO);
+    ke->received = GNUNET_YES;
     GNUNET_free (ee);
   }
   else
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Registering new element from remote peer\n");
-    op_register_element (op, ee);
+    op->state->received_fresh++;
+    op_register_element (op, ee, GNUNET_YES);
     /* only send results immediately if the client wants it */
-    switch (op->spec->result_mode)
+    switch (op->result_mode)
     {
       case GNUNET_SET_RESULT_ADDED:
         send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
@@ -1304,85 +1555,390 @@ handle_p2p_elements (void *cls,
     }
   }
 
+  if ( (op->state->received_total > 8) &&
+       (op->state->received_fresh < op->state->received_total / 3) )
+  {
+    /* The other peer gave us lots of old elements, there's something wrong. */
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  GNUNET_CADET_receive_done (op->channel);
   maybe_finish (op);
 }
 
 
 /**
+ * Check a full element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+int
+check_union_p2p_full_element (void *cls,
+                              const struct GNUNET_SET_ElementMessage *emsg)
+{
+  struct Operation *op = cls;
+
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  // FIXME: check that we expect full elements here?
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_full_element (void *cls,
+                               const struct GNUNET_SET_ElementMessage *emsg)
+{
+  struct Operation *op = cls;
+  struct ElementEntry *ee;
+  struct KeyEntry *ke;
+  uint16_t element_size;
+
+  element_size = ntohs (emsg->header.size) - sizeof (struct GNUNET_SET_ElementMessage);
+  ee = GNUNET_malloc (sizeof (struct ElementEntry) + element_size);
+  GNUNET_memcpy (&ee[1], &emsg[1], element_size);
+  ee->element.size = element_size;
+  ee->element.data = &ee[1];
+  ee->element.element_type = ntohs (emsg->element_type);
+  ee->remote = GNUNET_YES;
+  GNUNET_SET_element_hash (&ee->element, &ee->element_hash);
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got element (full diff, size %u, hash %s) from peer\n",
+       (unsigned int) element_size,
+       GNUNET_h2s (&ee->element_hash));
+
+  GNUNET_STATISTICS_update (_GSS_statistics,
+                            "# received elements",
+                            1,
+                            GNUNET_NO);
+  GNUNET_STATISTICS_update (_GSS_statistics,
+                            "# exchanged elements",
+                            1,
+                            GNUNET_NO);
+
+  op->state->received_total++;
+
+  ke = op_get_element (op, &ee->element_hash);
+  if (NULL != ke)
+  {
+    /* Got repeated element.  Should not happen since
+     * we track demands. */
+    GNUNET_STATISTICS_update (_GSS_statistics,
+                              "# repeated elements",
+                              1,
+                              GNUNET_NO);
+    ke->received = GNUNET_YES;
+    GNUNET_free (ee);
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Registering new element from remote peer\n");
+    op->state->received_fresh++;
+    op_register_element (op, ee, GNUNET_YES);
+    /* only send results immediately if the client wants it */
+    switch (op->result_mode)
+    {
+      case GNUNET_SET_RESULT_ADDED:
+        send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
+        break;
+      case GNUNET_SET_RESULT_SYMMETRIC:
+        send_client_element (op, &ee->element, GNUNET_SET_STATUS_ADD_LOCAL);
+        break;
+      default:
+        /* Result mode not supported, should have been caught earlier. */
+        GNUNET_break (0);
+        break;
+    }
+  }
+
+  if ( (GNUNET_YES == op->byzantine) &&
+       (op->state->received_total > 384 + op->state->received_fresh * 4) &&
+       (op->state->received_fresh < op->state->received_total / 6) )
+  {
+    /* The other peer gave us lots of old elements, there's something wrong. */
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         "Other peer sent only %llu/%llu fresh elements, failing operation\n",
+         (unsigned long long) op->state->received_fresh,
+         (unsigned long long) op->state->received_total);
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  GNUNET_CADET_receive_done (op->channel);
+}
+
+
+/**
  * Send offers (for GNUNET_Hash-es) in response
  * to inquiries (for IBF_Key-s).
  *
  * @param cls the union operation
- * @param mh the message
+ * @param msg the message
  */
-static void
-handle_p2p_inquiry (void *cls,
-                    const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_inquiry (void *cls,
+                         const struct InquiryMessage *msg)
 {
   struct Operation *op = cls;
-  const struct IBF_Key *ibf_key;
   unsigned int num_keys;
-  struct InquiryMessage *msg;
 
-  /* look up elements and send them */
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
   if (op->state->phase != PHASE_INVENTORY_PASSIVE)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
-  num_keys = (ntohs (mh->size) - sizeof (struct InquiryMessage))
-      / sizeof (struct IBF_Key);
-  if ((ntohs (mh->size) - sizeof (struct InquiryMessage))
+  num_keys = (ntohs (msg->header.size) - sizeof (struct InquiryMessage))
+    / sizeof (struct IBF_Key);
+  if ((ntohs (msg->header.size) - sizeof (struct InquiryMessage))
       != num_keys * sizeof (struct IBF_Key))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  msg = (struct InquiryMessage *) mh;
 
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_inquiry (void *cls,
+                          const struct InquiryMessage *msg)
+{
+  struct Operation *op = cls;
+  const struct IBF_Key *ibf_key;
+  unsigned int num_keys;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received union inquiry\n");
+  num_keys = (ntohs (msg->header.size) - sizeof (struct InquiryMessage))
+    / sizeof (struct IBF_Key);
   ibf_key = (const struct IBF_Key *) &msg[1];
   while (0 != num_keys--)
   {
     struct IBF_Key unsalted_key;
-    unsalt_key (ibf_key, ntohl (msg->salt), &unsalted_key);
-    send_offers_for_key (op, unsalted_key);
+
+    unsalt_key (ibf_key,
+                ntohl (msg->salt),
+                &unsalted_key);
+    send_offers_for_key (op,
+                         unsalted_key);
     ibf_key++;
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
 /**
- * FIXME
+ * Iterator over hash map entries, called to
+ * destroy the linked list of colliding ibf key entries.
+ *
+ * @param cls closure
+ * @param key current key code
+ * @param value value in the hash map
+ * @return #GNUNET_YES if we should continue to iterate,
+ *         #GNUNET_NO if not.
  */
-static void
-handle_p2p_demand (void *cls,
-                   const struct GNUNET_MessageHeader *mh)
+static int
+send_missing_full_elements_iter (void *cls,
+                                 uint32_t key,
+                                 void *value)
 {
   struct Operation *op = cls;
-  struct ElementEntry *ee;
+  struct KeyEntry *ke = value;
+  struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ElementMessage *emsg;
-  const struct GNUNET_HashCode *hash;
+  struct ElementEntry *ee = ke->element;
+
+  if (GNUNET_YES == ke->received)
+    return GNUNET_YES;
+  ev = GNUNET_MQ_msg_extra (emsg,
+                            ee->element.size,
+                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
+  GNUNET_memcpy (&emsg[1],
+                 ee->element.data,
+                 ee->element.size);
+  emsg->element_type = htons (ee->element.element_type);
+  GNUNET_MQ_send (op->mq,
+                  ev);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handle a request for full set transmission.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_request_full (void *cls,
+                               const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received request for full set transmission\n");
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  if (PHASE_EXPECT_IBF != op->state->phase)
+  {
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+
+  // FIXME: we need to check that our set is larger than the
+  // byzantine_lower_bound by some threshold
+  send_full_set (op);
+  GNUNET_CADET_receive_done (op->channel);
+}
+
+
+/**
+ * Handle a "full done" message.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_full_done (void *cls,
+                            const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+
+  switch (op->state->phase)
+  {
+  case PHASE_EXPECT_IBF:
+    {
+      struct GNUNET_MQ_Envelope *ev;
+
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "got FULL DONE, sending elements that other peer is missing\n");
+
+      /* send all the elements that did not come from the remote peer */
+      GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
+                                               &send_missing_full_elements_iter,
+                                               op);
+
+      ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
+      GNUNET_MQ_notify_sent (ev,
+                             &send_done_and_destroy,
+                             op);
+      GNUNET_MQ_send (op->mq,
+                      ev);
+      op->state->phase = PHASE_DONE;
+      /* we now wait until the other peer shuts the tunnel down*/
+    }
+    break;
+  case PHASE_FULL_SENDING:
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "got FULL DONE, finishing\n");
+      /* We sent the full set, and got the response for that.  We're done. */
+      op->state->phase = PHASE_DONE;
+      GNUNET_CADET_receive_done (op->channel);
+      send_done_and_destroy (op);
+      return;
+    }
+    break;
+  default:
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Handle full done phase is %u\n",
+                (unsigned) op->state->phase);
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  GNUNET_CADET_receive_done (op->channel);
+}
+
+
+/**
+ * Check a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_demand (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
   unsigned int num_hashes;
-  struct GNUNET_MQ_Envelope *ev;
 
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
   num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
     / sizeof (struct GNUNET_HashCode);
   if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
       != num_hashes * sizeof (struct GNUNET_HashCode))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_demand (void *cls,
+                         const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+  struct ElementEntry *ee;
+  struct GNUNET_SET_ElementMessage *emsg;
+  const struct GNUNET_HashCode *hash;
+  unsigned int num_hashes;
+  struct GNUNET_MQ_Envelope *ev;
 
+  num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
+    / sizeof (struct GNUNET_HashCode);
   for (hash = (const struct GNUNET_HashCode *) &mh[1];
        num_hashes > 0;
        hash++, num_hashes--)
   {
-    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements, hash);
+    ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
+                                            hash);
     if (NULL == ee)
     {
       /* Demand for non-existing element. */
@@ -1412,7 +1968,7 @@ handle_p2p_demand (void *cls,
                               1,
                               GNUNET_NO);
 
-    switch (op->spec->result_mode)
+    switch (op->result_mode)
     {
       case GNUNET_SET_RESULT_ADDED:
         /* Nothing to do. */
@@ -1426,42 +1982,65 @@ handle_p2p_demand (void *cls,
         break;
     }
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
 /**
- * Handle offers (of GNUNET_HashCode-s) and
- * respond with demands (of GNUNET_HashCode-s).
+ * Check offer (of `struct GNUNET_HashCode`s).
  *
  * @param cls the union operation
  * @param mh the message
+ * @return #GNUNET_OK if @a mh is well-formed
  */
-static void
-handle_p2p_offer (void *cls,
-                    const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
-  const struct GNUNET_HashCode *hash;
   unsigned int num_hashes;
 
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
   /* look up elements and send them */
   if ( (op->state->phase != PHASE_INVENTORY_PASSIVE) &&
        (op->state->phase != PHASE_INVENTORY_ACTIVE))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
   num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
     / sizeof (struct GNUNET_HashCode);
-  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
-      != num_hashes * sizeof (struct GNUNET_HashCode))
+  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader)) !=
+      num_hashes * sizeof (struct GNUNET_HashCode))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle offers (of `struct GNUNET_HashCode`s) and
+ * respond with demands (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+  const struct GNUNET_HashCode *hash;
+  unsigned int num_hashes;
 
+  num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
+    / sizeof (struct GNUNET_HashCode);
   for (hash = (const struct GNUNET_HashCode *) &mh[1];
        num_hashes > 0;
        hash++, num_hashes--)
@@ -1470,7 +2049,7 @@ handle_p2p_offer (void *cls,
     struct GNUNET_MessageHeader *demands;
     struct GNUNET_MQ_Envelope *ev;
 
-    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements,
+    ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
                                             hash);
     if (NULL != ee)
       if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
@@ -1497,9 +2076,12 @@ handle_p2p_offer (void *cls,
     ev = GNUNET_MQ_msg_header_extra (demands,
                                      sizeof (struct GNUNET_HashCode),
                                      GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND);
-    *(struct GNUNET_HashCode *) &demands[1] = *hash;
+    GNUNET_memcpy (&demands[1],
+                   hash,
+                   sizeof (struct GNUNET_HashCode));
     GNUNET_MQ_send (op->mq, ev);
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1509,16 +2091,22 @@ handle_p2p_offer (void *cls,
  * @param cls the union operation
  * @param mh the message
  */
-static void
-handle_p2p_done (void *cls,
-                 const struct GNUNET_MessageHeader *mh)
+void
+handle_union_p2p_done (void *cls,
+                       const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
 
-  if (op->state->phase == PHASE_INVENTORY_PASSIVE)
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  switch (op->state->phase)
   {
+  case PHASE_INVENTORY_PASSIVE:
     /* We got all requests, but still have to send our elements in response. */
-
     op->state->phase = PHASE_FINISH_WAITING;
 
     LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -1532,11 +2120,10 @@ handle_p2p_done (void *cls,
      * all our demands are satisfied, so that the active
      * peer can quit if we gave him everything.
      */
+    GNUNET_CADET_receive_done (op->channel);
     maybe_finish (op);
     return;
-  }
-  if (op->state->phase == PHASE_INVENTORY_ACTIVE)
-  {
+  case PHASE_INVENTORY_ACTIVE:
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "got DONE (as active partner), waiting to finish\n");
     /* All demands of the other peer are satisfied,
@@ -1547,11 +2134,14 @@ handle_p2p_done (void *cls,
      * to the other peer once our demands are met.
      */
     op->state->phase = PHASE_FINISH_CLOSING;
+    GNUNET_CADET_receive_done (op->channel);
     maybe_finish (op);
     return;
+  default:
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
   }
-  GNUNET_break_op (0);
-  fail_union_operation (op);
 }
 
 
@@ -1562,21 +2152,31 @@ handle_p2p_done (void *cls,
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
  */
-static void
+static struct OperationState *
 union_evaluate (struct Operation *op,
                 const struct GNUNET_MessageHeader *opaque_context)
 {
+  struct OperationState *state;
   struct GNUNET_MQ_Envelope *ev;
   struct OperationRequestMessage *msg;
 
-  GNUNET_assert (NULL == op->state);
-  op->state = GNUNET_new (struct OperationState);
-  op->state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
+  ev = GNUNET_MQ_msg_nested_mh (msg,
+                                GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
+                                opaque_context);
+  if (NULL == ev)
+  {
+    /* the context message is too large */
+    GNUNET_break (0);
+    return NULL;
+  }
+  state = GNUNET_new (struct OperationState);
+  state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
+                                                                 GNUNET_NO);
   /* copy the current generation's strata estimator for this operation */
-  op->state->se = strata_estimator_dup (op->spec->set->state->se);
+  state->se = strata_estimator_dup (op->set->state->se);
   /* we started the operation, thus we have to send the operation request */
-  op->state->phase = PHASE_EXPECT_SE;
-  op->state->salt_receive = op->state->salt_send = 42;
+  state->phase = PHASE_EXPECT_SE;
+  state->salt_receive = state->salt_send = 42; // FIXME?????
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Initiating union operation evaluation\n");
   GNUNET_STATISTICS_update (_GSS_statistics,
@@ -1587,16 +2187,6 @@ union_evaluate (struct Operation *op,
                             "# of initiated union operations",
                             1,
                             GNUNET_NO);
-  ev = GNUNET_MQ_msg_nested_mh (msg,
-                                GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
-                                opaque_context);
-  if (NULL == ev)
-  {
-    /* the context message is too large */
-    GNUNET_break (0);
-    GNUNET_SERVER_client_disconnect (op->spec->set->client);
-    return;
-  }
   msg->operation = htonl (GNUNET_SET_OPERATION_UNION);
   GNUNET_MQ_send (op->mq,
                   ev);
@@ -1607,6 +2197,11 @@ union_evaluate (struct Operation *op,
   else
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "sent op request without context message\n");
+
+  op->state = state;
+  initialize_key_to_element (op);
+  state->initial_size = GNUNET_CONTAINER_multihashmap32_size (state->key_to_element);
+  return state;
 }
 
 
@@ -1616,13 +2211,19 @@ union_evaluate (struct Operation *op,
  *
  * @param op operation that will be accepted as a union operation
  */
-static void
+static struct OperationState *
 union_accept (struct Operation *op)
 {
+  struct OperationState *state;
+  const struct StrataEstimator *se;
+  struct GNUNET_MQ_Envelope *ev;
+  struct StrataEstimatorMessage *strata_msg;
+  char *buf;
+  size_t len;
+  uint16_t type;
+
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "accepting set union operation\n");
-  GNUNET_assert (NULL == op->state);
-
   GNUNET_STATISTICS_update (_GSS_statistics,
                             "# of accepted union operations",
                             1,
@@ -1632,12 +2233,37 @@ union_accept (struct Operation *op)
                             1,
                             GNUNET_NO);
 
-  op->state = GNUNET_new (struct OperationState);
-  op->state->se = strata_estimator_dup (op->spec->set->state->se);
-  op->state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
-  op->state->salt_receive = op->state->salt_send = 42;
+  state = GNUNET_new (struct OperationState);
+  state->se = strata_estimator_dup (op->set->state->se);
+  state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
+                                                                 GNUNET_NO);
+  state->salt_receive = state->salt_send = 42; // FIXME?????
+  op->state = state;
+  initialize_key_to_element (op);
+  state->initial_size = GNUNET_CONTAINER_multihashmap32_size (state->key_to_element);
+
   /* kick off the operation */
-  send_strata_estimator (op);
+  se = state->se;
+  buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
+  len = strata_estimator_write (se,
+                                buf);
+  if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
+  else
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
+  ev = GNUNET_MQ_msg_extra (strata_msg,
+                            len,
+                            type);
+  GNUNET_memcpy (&strata_msg[1],
+                 buf,
+                 len);
+  GNUNET_free (buf);
+  strata_msg->set_size
+    = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (op->set->content->elements));
+  GNUNET_MQ_send (op->mq,
+                  ev);
+  state->phase = PHASE_EXPECT_IBF;
+  return state;
 }
 
 
@@ -1677,7 +2303,8 @@ union_set_create (void)
  * @param ee the element to add to the set
  */
 static void
-union_add (struct SetState *set_state, struct ElementEntry *ee)
+union_add (struct SetState *set_state,
+           struct ElementEntry *ee)
 {
   strata_estimator_insert (set_state->se,
                            get_ibf_key (&ee->element_hash));
@@ -1692,7 +2319,8 @@ union_add (struct SetState *set_state, struct ElementEntry *ee)
  * @param ee set element to remove
  */
 static void
-union_remove (struct SetState *set_state, struct ElementEntry *ee)
+union_remove (struct SetState *set_state,
+              struct ElementEntry *ee)
 {
   strata_estimator_remove (set_state->se,
                            get_ibf_key (&ee->element_hash));
@@ -1717,104 +2345,35 @@ union_set_destroy (struct SetState *set_state)
 
 
 /**
- * Dispatch messages for a union operation.
+ * Copy union-specific set state.
  *
- * @param op the state of the union evaluate operation
- * @param mh the received message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param state source state for copying the union state
+ * @return a copy of the union-specific set state
  */
-int
-union_handle_p2p_message (struct Operation *op,
-                          const struct GNUNET_MessageHeader *mh)
+static struct SetState *
+union_copy_state (struct SetState *state)
 {
-  //LOG (GNUNET_ERROR_TYPE_DEBUG,
-  //            "received p2p message (t: %u, s: %u)\n",
-  //            ntohs (mh->type),
-  //            ntohs (mh->size));
-  switch (ntohs (mh->type))
-  {
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF:
-      return handle_p2p_ibf (op, mh);
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE:
-      return handle_p2p_strata_estimator (op, mh, GNUNET_NO);
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC:
-      return handle_p2p_strata_estimator (op, mh, GNUNET_YES);
-    case GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS:
-      handle_p2p_elements (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY:
-      handle_p2p_inquiry (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE:
-      handle_p2p_done (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER:
-      handle_p2p_offer (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND:
-      handle_p2p_demand (op, mh);
-      break;
-    default:
-      /* Something wrong with cadet's message handlers? */
-      GNUNET_assert (0);
-  }
-  return GNUNET_OK;
-}
+  struct SetState *new_state;
 
+  GNUNET_assert ( (NULL != state) &&
+                  (NULL != state->se) );
+  new_state = GNUNET_new (struct SetState);
+  new_state->se = strata_estimator_dup (state->se);
 
-/**
- * Handler for peer-disconnects, notifies the client
- * about the aborted operation in case the op was not concluded.
- *
- * @param op the destroyed operation
- */
-static void
-union_peer_disconnect (struct Operation *op)
-{
-  if (PHASE_DONE != op->state->phase)
-  {
-    struct GNUNET_MQ_Envelope *ev;
-    struct GNUNET_SET_ResultMessage *msg;
-
-    ev = GNUNET_MQ_msg (msg,
-                        GNUNET_MESSAGE_TYPE_SET_RESULT);
-    msg->request_id = htonl (op->spec->client_request_id);
-    msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-    msg->element_type = htons (0);
-    GNUNET_MQ_send (op->spec->set->client_mq,
-                    ev);
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "other peer disconnected prematurely, phase %u\n",
-         op->state->phase);
-    _GSS_operation_destroy (op,
-                            GNUNET_YES);
-    return;
-  }
-  // else: the session has already been concluded
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "other peer disconnected (finished)\n");
-  if (GNUNET_NO == op->state->client_done_sent)
-    send_done_and_destroy (op);
+  return new_state;
 }
 
 
 /**
- * Copy union-specific set state.
+ * Handle case where channel went down for an operation.
  *
- * @param set source set for copying the union state
- * @return a copy of the union-specific set state
+ * @param op operation that lost the channel
  */
-static struct SetState *
-union_copy_state (struct Set *set)
+static void
+union_channel_death (struct Operation *op)
 {
-  struct SetState *new_state;
-
-  new_state = GNUNET_new (struct SetState);
-  GNUNET_assert ( (NULL != set->state) && (NULL != set->state->se) );
-  new_state->se = strata_estimator_dup (set->state->se);
-
-  return new_state;
+  _GSS_operation_destroy (op,
+                          GNUNET_YES);
 }
 
 
@@ -1829,15 +2388,14 @@ _GSS_union_vt ()
 {
   static const struct SetVT union_vt = {
     .create = &union_set_create,
-    .msg_handler = &union_handle_p2p_message,
     .add = &union_add,
     .remove = &union_remove,
     .destroy_set = &union_set_destroy,
     .evaluate = &union_evaluate,
     .accept = &union_accept,
-    .peer_disconnect = &union_peer_disconnect,
     .cancel = &union_op_cancel,
     .copy_state = &union_copy_state,
+    .channel_death = &union_channel_death
   };
 
   return &union_vt;
diff --git a/src/set/gnunet-service-set_union.h b/src/set/gnunet-service-set_union.h
new file mode 100644
index 000000000..cbf60bcbc
--- /dev/null
+++ b/src/set/gnunet-service-set_union.h
@@ -0,0 +1,239 @@
+
+/*
+      This file is part of GNUnet
+      Copyright (C) 2013-2017 GNUnet e.V.
+
+      GNUnet is free software; you can redistribute it and/or modify
+      it under the terms of the GNU General Public License as published
+      by the Free Software Foundation; either version 3, or (at your
+      option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      General Public License for more details.
+
+      You should have received a copy of the GNU General Public License
+      along with GNUnet; see the file COPYING.  If not, write to the
+      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+      Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file set/gnunet-service-set_union.h
+ * @brief two-peer set operations
+ * @author Florian Dold
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_SET_UNION_H
+#define GNUNET_SERVICE_SET_UNION_H
+
+#include "gnunet-service-set.h"
+#include "gnunet-service-set_protocol.h"
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+int
+check_union_p2p_strata_estimator (void *cls,
+                                  const struct StrataEstimatorMessage *msg);
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_strata_estimator (void *cls,
+                                   const struct StrataEstimatorMessage *msg);
+
+
+/**
+ * Check an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_union_p2p_ibf (void *cls,
+                     const struct IBFMessage *msg);
+
+
+/**
+ * Handle an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ */
+void
+handle_union_p2p_ibf (void *cls,
+                      const struct IBFMessage *msg);
+
+
+/**
+ * Check an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+int
+check_union_p2p_elements (void *cls,
+                          const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Handle an element message from a remote peer.
+ * Sent by the other peer either because we decoded an IBF and placed a demand,
+ * or because the other peer switched to full set transmission.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_elements (void *cls,
+                           const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Check a full element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+int
+check_union_p2p_full_element (void *cls,
+                              const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Handle an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_full_element (void *cls,
+                               const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+int
+check_union_p2p_inquiry (void *cls,
+                         const struct InquiryMessage *msg);
+
+
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_inquiry (void *cls,
+                          const struct InquiryMessage *msg);
+
+
+
+/**
+ * Handle a request for full set transmission.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_request_full (void *cls,
+                               const struct GNUNET_MessageHeader *mh);
+
+
+
+/**
+ * Handle a "full done" message.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_full_done (void *cls,
+                            const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Check a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_demand (void *cls,
+                        const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_demand (void *cls,
+                         const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Check offer (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_offer (void *cls,
+                       const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle offers (of `struct GNUNET_HashCode`s) and
+ * respond with demands (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle a done message from a remote peer
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_done (void *cls,
+                       const struct GNUNET_MessageHeader *mh);
+
+
+#endif
diff --git a/src/set/gnunet-set-ibf-profiler.c b/src/set/gnunet-set-ibf-profiler.c
index 8d832e358..70fafa1e1 100644
--- a/src/set/gnunet-set-ibf-profiler.c
+++ b/src/set/gnunet-set-ibf-profiler.c
@@ -244,24 +244,41 @@ run (void *cls,
 int
 main (int argc, char **argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'A', "asize", NULL,
-     gettext_noop ("number of element in set A-B"), 1,
-     &GNUNET_GETOPT_set_uint, &asize},
-    {'B', "bsize", NULL,
-     gettext_noop ("number of element in set B-A"), 1,
-     &GNUNET_GETOPT_set_uint, &bsize},
-    {'C', "csize", NULL,
-     gettext_noop ("number of common elements in A and B"), 1,
-     &GNUNET_GETOPT_set_uint, &csize},
-    {'k', "hash-num", NULL,
-     gettext_noop ("hash num"), 1,
-     &GNUNET_GETOPT_set_uint, &hash_num},
-    {'s', "ibf-size", NULL,
-     gettext_noop ("ibf size"), 1,
-     &GNUNET_GETOPT_set_uint, &ibf_size},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('A',
+                                   "asize",
+                                   NULL,
+                                   gettext_noop ("number of element in set A-B"),
+                                   &asize),
+
+    GNUNET_GETOPT_option_uint ('B',
+                                   "bsize",
+                                   NULL,
+                                   gettext_noop ("number of element in set B-A"),
+                                   &bsize),
+
+    GNUNET_GETOPT_option_uint ('C',
+                                   "csize",
+                                   NULL,
+                                   gettext_noop ("number of common elements in A and B"),
+                                   &csize),
+    
+    GNUNET_GETOPT_option_uint ('k',
+                                   "hash-num",
+                                   NULL,
+                                   gettext_noop ("hash num"),
+                                   &hash_num),
+
+    GNUNET_GETOPT_option_uint ('s',
+                                   "ibf-size",
+                                   NULL,
+                                   gettext_noop ("ibf size"),
+                                   &ibf_size),
+
     GNUNET_GETOPT_OPTION_END
   };
+
   GNUNET_PROGRAM_run2 (argc, argv, "gnunet-consensus-ibf",
                       "help",
                       options, &run, NULL, GNUNET_YES);
diff --git a/src/set/gnunet-set-profiler.c b/src/set/gnunet-set-profiler.c
index f89817ff5..68f7be690 100644
--- a/src/set/gnunet-set-profiler.c
+++ b/src/set/gnunet-set-profiler.c
@@ -58,6 +58,11 @@ static struct GNUNET_PeerIdentity local_peer;
 
 static struct GNUNET_SET_ListenHandle *set_listener;
 
+static int byzantine;
+static unsigned int force_delta;
+static unsigned int force_full;
+static unsigned int element_size = 32;
+
 /**
  * Handle to the statistics service.
  */
@@ -86,7 +91,7 @@ map_remove_iterator (void *cls,
 
   GNUNET_assert (NULL != key);
 
-  ret = GNUNET_CONTAINER_multihashmap_remove (m, key, NULL);
+  ret = GNUNET_CONTAINER_multihashmap_remove_all (m, key);
   if (GNUNET_OK != ret)
     printf ("spurious element\n");
   return GNUNET_YES;
@@ -158,6 +163,7 @@ check_all_done (void)
 static void
 set_result_cb (void *cls,
                const struct GNUNET_SET_Element *element,
+               uint64_t current_size,
                enum GNUNET_SET_Status status)
 {
   struct SetInfo *info = cls;
@@ -191,7 +197,7 @@ set_result_cb (void *cls,
       GNUNET_assert (0);
   }
 
-  if (element->size != sizeof (struct GNUNET_HashCode))
+  if (element->size != element_size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "wrong element size: %u, expected %u\n",
@@ -203,8 +209,10 @@ set_result_cb (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: got element (%s)\n",
               info->id, GNUNET_h2s (element->data));
   GNUNET_assert (NULL != element->data);
+  struct GNUNET_HashCode data_hash;
+  GNUNET_CRYPTO_hash (element->data, element_size, &data_hash);
   GNUNET_CONTAINER_multihashmap_put (info->received,
-                                     element->data, NULL,
+                                     &data_hash, NULL,
                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
 }
 
@@ -215,6 +223,10 @@ set_listen_cb (void *cls,
                const struct GNUNET_MessageHeader *context_msg,
                struct GNUNET_SET_Request *request)
 {
+  /* max. 2 options plus terminator */
+  struct GNUNET_SET_Option opts[3] = {{0}};
+  unsigned int n_opts = 0;
+
   if (NULL == request)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -224,8 +236,24 @@ set_listen_cb (void *cls,
   GNUNET_assert (NULL == info2.oh);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "set listen cb called\n");
+  if (byzantine)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_BYZANTINE };
+  }
+  GNUNET_assert (!(force_full && force_delta));
+  if (force_full)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_FORCE_FULL };
+  }
+  if (force_delta)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_FORCE_DELTA };
+  }
+
+  opts[n_opts].type = 0;
   info2.oh = GNUNET_SET_accept (request, GNUNET_SET_RESULT_SYMMETRIC,
-                               set_result_cb, &info2);
+                                opts,
+                                set_result_cb, &info2);
   GNUNET_SET_commit (info2.oh, info2.set);
 }
 
@@ -236,16 +264,12 @@ set_insert_iterator (void *cls,
                      void *value)
 {
   struct GNUNET_SET_Handle *set = cls;
-  struct GNUNET_SET_Element *el;
-
-  el = GNUNET_malloc (sizeof (struct GNUNET_SET_Element) +
-                      sizeof (struct GNUNET_HashCode));
-  el->element_type = 0;
-  GNUNET_memcpy (&el[1], key, sizeof *key);
-  el->data = &el[1];
-  el->size = sizeof *key;
-  GNUNET_SET_add_element (set, el, NULL, NULL);
-  GNUNET_free (el);
+  struct GNUNET_SET_Element el;
+
+  el.element_type = 0;
+  el.data = value;
+  el.size = element_size;
+  GNUNET_SET_add_element (set, &el, NULL, NULL);
   return GNUNET_YES;
 }
 
@@ -291,9 +315,14 @@ run (void *cls,
 {
   unsigned int i;
   struct GNUNET_HashCode hash;
+  /* max. 2 options plus terminator */
+  struct GNUNET_SET_Option opts[3] = {{0}};
+  unsigned int n_opts = 0;
 
   config = cfg;
 
+  GNUNET_assert (element_size > 0);
+
   if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &local_peer))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
@@ -317,22 +346,28 @@ run (void *cls,
 
   for (i = 0; i < num_a; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
   for (i = 0; i < num_b; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
   for (i = 0; i < num_c; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
@@ -350,8 +385,26 @@ run (void *cls,
   set_listener = GNUNET_SET_listen (config, GNUNET_SET_OPERATION_UNION,
                                     &app_id, set_listen_cb, NULL);
 
+
+  if (byzantine)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_BYZANTINE };
+  }
+  GNUNET_assert (!(force_full && force_delta));
+  if (force_full)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_FORCE_FULL };
+  }
+  if (force_delta)
+  {
+    opts[n_opts++] = (struct GNUNET_SET_Option) { .type = GNUNET_SET_OPTION_FORCE_DELTA };
+  }
+
+  opts[n_opts].type = 0;
+
   info1.oh = GNUNET_SET_prepare (&local_peer, &app_id, NULL,
                                  GNUNET_SET_RESULT_SYMMETRIC,
+                                 opts,
                                  set_result_cb, &info1);
   GNUNET_SET_commit (info1.oh, info1.set);
   GNUNET_SET_destroy (info1.set);
@@ -373,22 +426,60 @@ pre_run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'A', "num-first", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_a },
-      { 'B', "num-second", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_b },
-      { 'C', "num-common", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_c },
-      { 'x', "operation", NULL,
-        gettext_noop ("operation to execute"),
-        GNUNET_YES, &GNUNET_GETOPT_set_string, &op_str },
-      { 's', "statistics", NULL,
-        gettext_noop ("write statistics to file"),
-        GNUNET_YES, &GNUNET_GETOPT_set_filename, &statistics_filename },
+   struct GNUNET_GETOPT_CommandLineOption options[] = {
+      GNUNET_GETOPT_option_uint ('A',
+                                     "num-first",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_a),
+
+      GNUNET_GETOPT_option_uint ('B',
+                                     "num-second",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_b),
+
+      GNUNET_GETOPT_option_flag ('b',
+                                    "byzantine",
+                                    gettext_noop ("use byzantine mode"),
+                                    &byzantine),
+
+      GNUNET_GETOPT_option_uint ('f',
+                                     "force-full",
+                                     NULL,
+                                     gettext_noop ("force sending full set"),
+                                     &force_full),
+
+      GNUNET_GETOPT_option_uint ('d',
+                                     "force-delta",
+                                     NULL,
+                                     gettext_noop ("number delta operation"),
+                                     &force_delta),
+
+      GNUNET_GETOPT_option_uint ('C',
+                                     "num-common",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_c),
+
+      GNUNET_GETOPT_option_string ('x',
+                                   "operation",
+                                   NULL,
+                                   gettext_noop ("operation to execute"),
+                                   &op_str),
+
+      GNUNET_GETOPT_option_uint ('w',
+                                     "element-size",
+                                     NULL,
+                                     gettext_noop ("element size"),
+                                     &element_size),
+
+      GNUNET_GETOPT_option_filename ('s',
+                                     "statistics",
+                                     "FILENAME",
+                                     gettext_noop ("write statistics to file"),
+                                     &statistics_filename),
+
       GNUNET_GETOPT_OPTION_END
   };
   GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler",
diff --git a/src/set/plugin_block_set_test.c b/src/set/plugin_block_set_test.c
new file mode 100644
index 000000000..01b0c8602
--- /dev/null
+++ b/src/set/plugin_block_set_test.c
@@ -0,0 +1,123 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file set/plugin_block_set_test.c
+ * @brief set test block, recognizes elements with non-zero first byte as invalid
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_block_plugin.h"
+#include "gnunet_block_group_lib.h"
+
+
+/**
+ * Function called to validate a reply or a request.  For
+ * request evaluation, simply pass "NULL" for the reply_block.
+ *
+ * @param cls closure
+ * @param ctx block context
+ * @param type block type
+ * @param group block group to use
+ * @param eo control flags
+ * @param query original query (hash)
+ * @param xquery extrended query data (can be NULL, depending on type)
+ * @param xquery_size number of bytes in xquery
+ * @param reply_block response to validate
+ * @param reply_block_size number of bytes in reply block
+ * @return characterization of result
+ */
+static enum GNUNET_BLOCK_EvaluationResult
+block_plugin_set_test_evaluate (void *cls,
+                                struct GNUNET_BLOCK_Context *ctx,
+                                enum GNUNET_BLOCK_Type type,
+                                struct GNUNET_BLOCK_Group *group,
+                                enum GNUNET_BLOCK_EvaluationOptions eo,
+                                const struct GNUNET_HashCode *query,
+                                const void *xquery,
+                                size_t xquery_size,
+                                const void *reply_block,
+                                size_t reply_block_size)
+{
+  if ( (NULL == reply_block) ||
+       (reply_block_size == 0) ||
+       (0 != ((char *) reply_block)[0]) )
+    return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
+  return GNUNET_BLOCK_EVALUATION_OK_MORE;
+}
+
+
+/**
+ * Function called to obtain the key for a block.
+ *
+ * @param cls closure
+ * @param type block type
+ * @param block block to get the key for
+ * @param block_size number of bytes in block
+ * @param key set to the key (query) for the given block
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
+ *         (or if extracting a key from a block of this type does not work)
+ */
+static int
+block_plugin_set_test_get_key (void *cls,
+                               enum GNUNET_BLOCK_Type type,
+                               const void *block,
+                               size_t block_size,
+                               struct GNUNET_HashCode *key)
+{
+  return GNUNET_SYSERR;
+}
+
+
+/**
+ * Entry point for the plugin.
+ */
+void *
+libgnunet_plugin_block_set_test_init (void *cls)
+{
+  static enum GNUNET_BLOCK_Type types[] =
+  {
+    GNUNET_BLOCK_TYPE_SET_TEST,
+    GNUNET_BLOCK_TYPE_ANY       /* end of list */
+  };
+  struct GNUNET_BLOCK_PluginFunctions *api;
+
+  api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
+  api->evaluate = &block_plugin_set_test_evaluate;
+  api->get_key = &block_plugin_set_test_get_key;
+  api->types = types;
+  return api;
+}
+
+
+/**
+ * Exit point from the plugin.
+ */
+void *
+libgnunet_plugin_block_set_test_done (void *cls)
+{
+  struct GNUNET_BLOCK_PluginFunctions *api = cls;
+
+  GNUNET_free (api);
+  return NULL;
+}
+
+/* end of plugin_block_set_test.c */
diff --git a/src/set/set.conf.in b/src/set/set.conf.in
index 707bc3575..deaa07efb 100644
--- a/src/set/set.conf.in
+++ b/src/set/set.conf.in
@@ -9,4 +9,4 @@ UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-set.sock
 UNIX_MATCH_UID = YES
 UNIX_MATCH_GID = YES
 
-# PREFIX = valgrind
+#PREFIX = valgrind
diff --git a/src/set/set.h b/src/set/set.h
index f31216cb8..258e2bff9 100644
--- a/src/set/set.h
+++ b/src/set/set.h
@@ -102,6 +102,30 @@ struct GNUNET_SET_AcceptMessage
    * See `enum GNUNET_SET_ResultMode`.
    */
   uint32_t result_mode GNUNET_PACKED;
+
+  /**
+   * Always use delta operation instead of sending full sets,
+   * even it it's less efficient.
+   */
+  uint8_t force_delta;
+
+  /**
+   * Always send full sets, even if delta operations would
+   * be more efficient.
+   */
+  uint8_t force_full;
+
+  /**
+   * #GNUNET_YES to fail operations where Byzantine faults
+   * are suspected
+   */
+  uint8_t byzantine;
+
+  /**
+   * Lower bound for the set size, used only when
+   * byzantine mode is enabled.
+   */
+  uint8_t byzantine_lower_bound;
 };
 
 
@@ -184,6 +208,30 @@ struct GNUNET_SET_EvaluateMessage
    */
   uint32_t request_id GNUNET_PACKED;
 
+  /**
+   * Always use delta operation instead of sending full sets,
+   * even it it's less efficient.
+   */
+  uint8_t force_delta;
+
+  /**
+   * Always send full sets, even if delta operations would
+   * be more efficient.
+   */
+  uint8_t force_full;
+
+  /**
+   * #GNUNET_YES to fail operations where Byzantine faults
+   * are suspected
+   */
+  uint8_t byzantine;
+
+  /**
+   * Lower bound for the set size, used only when
+   * byzantine mode is enabled.
+   */
+  uint8_t byzantine_lower_bound;
+
   /* rest: context message, that is, application-specific
      message to convince listener to pick up */
 };
@@ -203,6 +251,11 @@ struct GNUNET_SET_ResultMessage
   struct GNUNET_MessageHeader header;
 
   /**
+   * Current set size.
+   */
+  uint64_t current_size;
+
+  /**
    * id the result belongs to
    */
   uint32_t request_id GNUNET_PACKED;
diff --git a/src/set/set_api.c b/src/set/set_api.c
index 7a7267a33..f5c43a9a7 100644
--- a/src/set/set_api.c
+++ b/src/set/set_api.c
@@ -76,6 +76,8 @@ struct GNUNET_SET_Handle
 
   /**
    * Should the set be destroyed once all operations are gone?
+   * #GNUNET_SYSERR if #GNUNET_SET_destroy() must raise this flag,
+   * #GNUNET_YES if #GNUNET_SET_destroy() did raise this flag.
    */
   int destroy_requested;
 
@@ -297,9 +299,9 @@ check_iter_element (void *cls,
  * @param cls the `struct GNUNET_SET_Handle *`
  * @param mh the message
  */
- static void
- handle_iter_element (void *cls,
-                      const struct GNUNET_SET_IterResponseMessage *msg)
+static void
+handle_iter_element (void *cls,
+                     const struct GNUNET_SET_IterResponseMessage *msg)
 {
   struct GNUNET_SET_Handle *set = cls;
   GNUNET_SET_ElementIterator iter = set->iterator;
@@ -308,6 +310,8 @@ check_iter_element (void *cls,
   struct GNUNET_MQ_Envelope *ev;
   uint16_t msize;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received element in set iteration\n");
   msize = ntohs (msg->header.size);
   if (set->iteration_id != ntohs (msg->iteration_id))
   {
@@ -344,11 +348,24 @@ handle_iter_done (void *cls,
   GNUNET_SET_ElementIterator iter = set->iterator;
 
   if (NULL == iter)
+  {
+    /* FIXME: if this is true, could cancel+start a fresh one
+       cause elements to go to the wrong iteration? */
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Service completed set iteration that was already cancelled\n");
     return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Set iteration completed\n");
+  set->destroy_requested = GNUNET_SYSERR;
   set->iterator = NULL;
   set->iteration_id++;
   iter (set->iterator_cls,
         NULL);
+  if (GNUNET_SYSERR == set->destroy_requested)
+    set->destroy_requested = GNUNET_NO;
+  if (GNUNET_YES == set->destroy_requested)
+    GNUNET_SET_destroy (set);
 }
 
 
@@ -385,7 +402,7 @@ handle_result (void *cls,
   int destroy_set;
 
   GNUNET_assert (NULL != set->mq);
-  result_status = ntohs (msg->result_status);
+  result_status = (enum GNUNET_SET_Status) ntohs (msg->result_status);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Got result message with status %d\n",
        result_status);
@@ -432,6 +449,7 @@ do_final:
   {
     oh->result_cb (oh->result_cls,
                    NULL,
+                   GNUNET_ntohll (msg->current_size),
                    result_status);
   }
   else
@@ -453,6 +471,7 @@ do_element:
   if (NULL != oh->result_cb)
     oh->result_cb (oh->result_cls,
                    &e,
+                   GNUNET_ntohll (msg->current_size),
                    result_status);
 }
 
@@ -498,6 +517,8 @@ GNUNET_SET_operation_cancel (struct GNUNET_SET_OperationHandle *oh)
   struct GNUNET_SET_CancelMessage *m;
   struct GNUNET_MQ_Envelope *mqm;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Cancelling SET operation\n");
   if (NULL != set)
   {
     mqm = GNUNET_MQ_msg (m, GNUNET_MESSAGE_TYPE_SET_CANCEL);
@@ -530,7 +551,7 @@ handle_client_set_error (void *cls,
   struct GNUNET_SET_Handle *set = cls;
   GNUNET_SET_ElementIterator iter = set->iterator;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
+  LOG (GNUNET_ERROR_TYPE_ERROR,
        "Handling client set error %d\n",
        error);
   while (NULL != set->ops_head)
@@ -538,24 +559,22 @@ handle_client_set_error (void *cls,
     if (NULL != set->ops_head->result_cb)
       set->ops_head->result_cb (set->ops_head->result_cls,
                                 NULL,
+                                0,
                                 GNUNET_SET_STATUS_FAILURE);
     set_operation_destroy (set->ops_head);
   }
   set->iterator = NULL;
   set->iteration_id++;
+  set->invalid = GNUNET_YES;
   if (NULL != iter)
     iter (set->iterator_cls,
           NULL);
-  set->invalid = GNUNET_YES;
-  if (GNUNET_YES == set->destroy_requested)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Destroying set after operation failure\n");
-    GNUNET_SET_destroy (set);
-  }
 }
 
 
+/**
+ * FIXME.
+ */
 static struct GNUNET_SET_Handle *
 create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
                  enum GNUNET_SET_OperationType op,
@@ -614,7 +633,8 @@ create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
                          GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT);
     copy_msg->cookie = *cookie;
   }
-  GNUNET_MQ_send (set->mq, mqm);
+  GNUNET_MQ_send (set->mq,
+                  mqm);
   return set;
 }
 
@@ -634,7 +654,16 @@ struct GNUNET_SET_Handle *
 GNUNET_SET_create (const struct GNUNET_CONFIGURATION_Handle *cfg,
                    enum GNUNET_SET_OperationType op)
 {
-  return create_internal (cfg, op, NULL);
+  struct GNUNET_SET_Handle *set;
+
+  set = create_internal (cfg,
+                          op,
+                          NULL);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating set %p for operation %d\n",
+       set,
+       op);
+  return set;
 }
 
 
@@ -660,13 +689,18 @@ GNUNET_SET_add_element (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *mqm;
   struct GNUNET_SET_ElementMessage *msg;
 
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "adding element of type %u to set %p\n",
+       (unsigned int) element->element_type,
+       set);
   if (GNUNET_YES == set->invalid)
   {
     if (NULL != cont)
       cont (cont_cls);
     return GNUNET_SYSERR;
   }
-  mqm = GNUNET_MQ_msg_extra (msg, element->size,
+  mqm = GNUNET_MQ_msg_extra (msg,
+                             element->size,
                              GNUNET_MESSAGE_TYPE_SET_ADD);
   msg->element_type = htons (element->element_type);
   GNUNET_memcpy (&msg[1],
@@ -701,6 +735,9 @@ GNUNET_SET_remove_element (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *mqm;
   struct GNUNET_SET_ElementMessage *msg;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing element from set %p\n",
+       set);
   if (GNUNET_YES == set->invalid)
   {
     if (NULL != cont)
@@ -733,8 +770,9 @@ GNUNET_SET_destroy (struct GNUNET_SET_Handle *set)
   /* destroying set while iterator is active is currently
      not supported; we should expand the API to allow
      clients to explicitly cancel the iteration! */
-  GNUNET_assert (NULL == set->iterator);
-  if (NULL != set->ops_head)
+  if ( (NULL != set->ops_head) ||
+       (NULL != set->iterator) ||
+       (GNUNET_SYSERR == set->destroy_requested) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Set operations are pending, delaying set destruction\n");
@@ -771,13 +809,18 @@ GNUNET_SET_prepare (const struct GNUNET_PeerIdentity *other_peer,
                     const struct GNUNET_HashCode *app_id,
                     const struct GNUNET_MessageHeader *context_msg,
                     enum GNUNET_SET_ResultMode result_mode,
+                    struct GNUNET_SET_Option options[],
                     GNUNET_SET_ResultIterator result_cb,
                     void *result_cls)
 {
   struct GNUNET_MQ_Envelope *mqm;
   struct GNUNET_SET_OperationHandle *oh;
   struct GNUNET_SET_EvaluateMessage *msg;
+  struct GNUNET_SET_Option *opt;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Client prepares set operation (%d)\n",
+       result_mode);
   oh = GNUNET_new (struct GNUNET_SET_OperationHandle);
   oh->result_cb = result_cb;
   oh->result_cls = result_cls;
@@ -787,6 +830,25 @@ GNUNET_SET_prepare (const struct GNUNET_PeerIdentity *other_peer,
   msg->app_id = *app_id;
   msg->result_mode = htonl (result_mode);
   msg->target_peer = *other_peer;
+  for (opt = options; opt->type != 0; opt++)
+  {
+    switch (opt->type)
+    {
+      case GNUNET_SET_OPTION_BYZANTINE:
+        msg->byzantine = GNUNET_YES;
+        msg->byzantine_lower_bound = opt->v.num;
+        break;
+      case GNUNET_SET_OPTION_FORCE_FULL:
+        msg->force_full = GNUNET_YES;
+        break;
+      case GNUNET_SET_OPTION_FORCE_DELTA:
+        msg->force_delta = GNUNET_YES;
+        break;
+      default:
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "Option with type %d not recognized\n", (int) opt->type);
+    }
+  }
   oh->conclude_mqm = mqm;
   oh->request_id_addr = &msg->request_id;
 
@@ -846,7 +908,8 @@ handle_request (void *cls,
   struct GNUNET_SET_RejectMessage *rmsg;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing incoming operation request\n");
+       "Processing incoming operation request with id %u\n",
+       ntohl (msg->accept_id));
   /* we got another valid request => reset the backoff */
   lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
   req.accept_id = ntohl (msg->accept_id);
@@ -860,7 +923,8 @@ handle_request (void *cls,
   if (GNUNET_YES == req.accepted)
     return; /* the accept-case is handled in #GNUNET_SET_accept() */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Rejecting request\n");
+       "Rejected request %u\n",
+       ntohl (msg->accept_id));
   mqm = GNUNET_MQ_msg (rmsg,
                        GNUNET_MESSAGE_TYPE_SET_REJECT);
   rmsg->accept_reject_id = msg->accept_id;
@@ -950,6 +1014,9 @@ GNUNET_SET_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
 {
   struct GNUNET_SET_ListenHandle *lh;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Starting listener for app %s\n",
+       GNUNET_h2s (app_id));
   lh = GNUNET_new (struct GNUNET_SET_ListenHandle);
   lh->listen_cb = listen_cb;
   lh->listen_cls = listen_cls;
@@ -976,7 +1043,8 @@ void
 GNUNET_SET_listen_cancel (struct GNUNET_SET_ListenHandle *lh)
 {
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Canceling listener\n");
+       "Canceling listener %s\n",
+       GNUNET_h2s (&lh->app_id));
   if (NULL != lh->mq)
   {
     GNUNET_MQ_destroy (lh->mq);
@@ -1008,6 +1076,7 @@ GNUNET_SET_listen_cancel (struct GNUNET_SET_ListenHandle *lh)
 struct GNUNET_SET_OperationHandle *
 GNUNET_SET_accept (struct GNUNET_SET_Request *request,
                    enum GNUNET_SET_ResultMode result_mode,
+                   struct GNUNET_SET_Option options[],
                    GNUNET_SET_ResultIterator result_cb,
                    void *result_cls)
 {
@@ -1016,8 +1085,13 @@ GNUNET_SET_accept (struct GNUNET_SET_Request *request,
   struct GNUNET_SET_AcceptMessage *msg;
 
   GNUNET_assert (GNUNET_NO == request->accepted);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Client accepts set operation (%d) with id %u\n",
+       result_mode,
+       request->accept_id);
   request->accepted = GNUNET_YES;
-  mqm = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_ACCEPT);
+  mqm = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_SET_ACCEPT);
   msg->accept_reject_id = htonl (request->accept_id);
   msg->result_mode = htonl (result_mode);
   oh = GNUNET_new (struct GNUNET_SET_OperationHandle);
@@ -1055,14 +1129,18 @@ GNUNET_SET_commit (struct GNUNET_SET_OperationHandle *oh,
   }
   if (GNUNET_YES == set->invalid)
     return GNUNET_SYSERR;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Client commits to SET\n");
   GNUNET_assert (NULL != oh->conclude_mqm);
   oh->set = set;
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                oh);
-  oh->request_id = GNUNET_MQ_assoc_add (set->mq, oh);
+  oh->request_id = GNUNET_MQ_assoc_add (set->mq,
+                                        oh);
   *oh->request_id_addr = htonl (oh->request_id);
-  GNUNET_MQ_send (set->mq, oh->conclude_mqm);
+  GNUNET_MQ_send (set->mq,
+                  oh->conclude_mqm);
   oh->conclude_mqm = NULL;
   oh->request_id_addr = NULL;
   return GNUNET_OK;
@@ -1111,6 +1189,8 @@ GNUNET_SET_copy_lazy (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *ev;
   struct SetCopyRequest *req;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating lazy copy of set\n");
   ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE);
   GNUNET_MQ_send (set->mq, ev);
 
diff --git a/src/set/test_set.conf b/src/set/test_set.conf
index 69e7f5c52..e28dfc6e9 100644
--- a/src/set/test_set.conf
+++ b/src/set/test_set.conf
@@ -5,7 +5,6 @@ GNUNET_TEST_HOME = /tmp/test-gnunet-set/
 
 [set]
 AUTOSTART = YES
-# PREFIX = valgrind
 #PREFIX = valgrind --leak-check=full
 #PREFIX = gdbserver :1234
 OPTIONS = -L INFO
diff --git a/src/set/test_set_api.c b/src/set/test_set_api.c
index 21af45f8a..ca7d8a4e2 100644
--- a/src/set/test_set_api.c
+++ b/src/set/test_set_api.c
@@ -55,6 +55,7 @@ static struct GNUNET_SCHEDULER_Task *tt;
 static void
 result_cb_set1 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t size,
                 enum GNUNET_SET_Status status)
 {
   switch (status)
@@ -101,6 +102,7 @@ result_cb_set1 (void *cls,
 static void
 result_cb_set2 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t size,
                 enum GNUNET_SET_Status status)
 {
   switch (status)
@@ -114,6 +116,7 @@ result_cb_set2 (void *cls,
     oh2 = NULL;
     fprintf (stderr,
              "set 2: received failure status\n");
+    GNUNET_SCHEDULER_shutdown ();
     ret = 1;
     break;
   case GNUNET_SET_STATUS_DONE:
@@ -145,10 +148,9 @@ listen_cb (void *cls,
   GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "listen cb called\n");
-  GNUNET_SET_listen_cancel (listen_handle);
-  listen_handle = NULL;
   oh2 = GNUNET_SET_accept (request,
                            GNUNET_SET_RESULT_ADDED,
+                           (struct GNUNET_SET_Option[]) { 0 },
                            &result_cb_set2,
                            NULL);
   GNUNET_SET_commit (oh2,
@@ -179,6 +181,7 @@ start (void *cls)
                             &app_id,
                             &context_msg,
                             GNUNET_SET_RESULT_ADDED,
+                            (struct GNUNET_SET_Option[]) { 0 },
                             &result_cb_set1,
                             NULL);
   GNUNET_SET_commit (oh1,
@@ -196,19 +199,25 @@ init_set2 (void *cls)
 {
   struct GNUNET_SET_Element element;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initializing set 2\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "initializing set 2\n");
 
   element.element_type = 0;
-
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL, NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL, NULL);
   element.data = "baz";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, &start, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          &start, NULL);
 }
 
 
@@ -221,14 +230,17 @@ init_set1 (void)
   struct GNUNET_SET_Element element;
 
   element.element_type = 0;
-
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, NULL, NULL);
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          NULL, NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, init_set2, NULL);
-
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          &init_set2,
+                          NULL);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "initialized set 1\n");
 }
@@ -238,10 +250,15 @@ static int
 iter_cb (void *cls,
          const struct GNUNET_SET_Element *element)
 {
+  struct GNUNET_SET_Handle *set = cls;
+
   if (NULL == element)
   {
     GNUNET_assert (3 == iter_count);
-    GNUNET_SET_destroy (cls);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Iteration finished, destroying set %p\n",
+                set);
+    GNUNET_SET_destroy (set);
     return GNUNET_YES;
   }
   iter_count++;
@@ -258,21 +275,31 @@ test_iter ()
   struct GNUNET_SET_Element element;
   struct GNUNET_SET_Handle *iter_set;
 
-  iter_set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
-
+  iter_set = GNUNET_SET_create (config,
+                                GNUNET_SET_OPERATION_UNION);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testing iteration over 3 elements on set %p\n",
+              iter_set);
   element.element_type = 0;
 
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
-
-  GNUNET_SET_iterate (iter_set, iter_cb, iter_set);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
+  GNUNET_SET_iterate (iter_set,
+                      &iter_cb,
+                      iter_set);
 }
 
 
@@ -368,19 +395,27 @@ run (void *cls,
               GNUNET_i2s (&local_id));
   test_iter ();
 
-  set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
-  set2 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+  set1 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
+  set2 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Created sets %p and %p for union operation\n",
+              set1,
+              set2);
   GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
                                     &app_id);
 
-  ///* test if canceling an uncommited request works! */
+  /* test if canceling an uncommited request works! */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching and instantly stopping set operation\n");
   my_oh = GNUNET_SET_prepare (&local_id,
                               &app_id,
                               NULL,
                               GNUNET_SET_RESULT_ADDED,
+                              (struct GNUNET_SET_Option[]) { 0 },
                               NULL,
                               NULL);
-
   GNUNET_SET_operation_cancel (my_oh);
 
   /* test the real set reconciliation */
diff --git a/src/set/test_set_intersection_result_full.c b/src/set/test_set_intersection_result_full.c
index f78a03eaf..a36aae4d5 100644
--- a/src/set/test_set_intersection_result_full.c
+++ b/src/set/test_set_intersection_result_full.c
@@ -56,10 +56,14 @@ static struct GNUNET_SET_OperationHandle *oh2;
 static void
 result_cb_set1 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t current_size,
                 enum GNUNET_SET_Status status)
 {
   static int count;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Processing result set 1 (%d)\n",
+              status);
   switch (status)
   {
   case GNUNET_SET_STATUS_OK:
@@ -74,6 +78,8 @@ result_cb_set1 (void *cls,
     GNUNET_assert (1 == count);
     GNUNET_SET_destroy (set1);
     set1 = NULL;
+    if (NULL == set2)
+      GNUNET_SCHEDULER_shutdown ();
     break;
   default:
     GNUNET_assert (0);
@@ -84,10 +90,14 @@ result_cb_set1 (void *cls,
 static void
 result_cb_set2 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t current_size,
                 enum GNUNET_SET_Status status)
 {
   static int count;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Processing result set 2 (%d)\n",
+              status);
   switch (status)
   {
   case GNUNET_SET_STATUS_OK:
@@ -102,6 +112,8 @@ result_cb_set2 (void *cls,
     GNUNET_assert (1 == count);
     GNUNET_SET_destroy (set2);
     set2 = NULL;
+    if (NULL == set1)
+      GNUNET_SCHEDULER_shutdown ();
     break;
   default:
     GNUNET_assert (0);
@@ -115,14 +127,17 @@ listen_cb (void *cls,
            const struct GNUNET_MessageHeader *context_msg,
            struct GNUNET_SET_Request *request)
 {
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "starting intersection by accepting and committing\n");
   GNUNET_assert (NULL != context_msg);
   GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
   GNUNET_SET_listen_cancel (listen_handle);
   listen_handle = NULL;
   oh2 = GNUNET_SET_accept (request,
-                          GNUNET_SET_RESULT_FULL,
-                          &result_cb_set2,
-                          NULL);
+                           GNUNET_SET_RESULT_FULL,
+                           (struct GNUNET_SET_Option[]) { 0 },
+                           &result_cb_set2,
+                           NULL);
   GNUNET_SET_commit (oh2,
                      set2);
 }
@@ -138,17 +153,21 @@ start (void *cls)
 {
   struct GNUNET_MessageHeader context_msg;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "starting listener\n");
   context_msg.size = htons (sizeof context_msg);
   context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
   listen_handle = GNUNET_SET_listen (config,
                                      GNUNET_SET_OPERATION_INTERSECTION,
                                      &app_id,
-                                     &listen_cb, NULL);
+                                     &listen_cb,
+                                     NULL);
   oh1 = GNUNET_SET_prepare (&local_id,
-                           &app_id,
-                           &context_msg,
-                           GNUNET_SET_RESULT_FULL,
-                           &result_cb_set1,
+                            &app_id,
+                            &context_msg,
+                            GNUNET_SET_RESULT_FULL,
+                            (struct GNUNET_SET_Option[]) { 0 },
+                            &result_cb_set1,
                             NULL);
   GNUNET_SET_commit (oh1,
                      set1);
@@ -170,13 +189,22 @@ init_set2 (void *cls)
   element.element_type = 0;
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL,
+                          NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL,
+                          NULL);
   element.data = "baz";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, &start, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          &start,
+                          NULL);
 }
 
 
@@ -188,13 +216,21 @@ init_set1 (void)
 {
   struct GNUNET_SET_Element element;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "initializing set 1\n");
   element.element_type = 0;
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, NULL, NULL);
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          NULL,
+                          NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, &init_set2, NULL);
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          &init_set2,
+                          NULL);
 }
 
 
@@ -219,17 +255,27 @@ test_iter ()
   struct GNUNET_SET_Element element;
   struct GNUNET_SET_Handle *iter_set;
 
-  iter_set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_INTERSECTION);
+  iter_set = GNUNET_SET_create (config,
+                                GNUNET_SET_OPERATION_INTERSECTION);
   element.element_type = 0;
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL,
+                          NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL,
+                          NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL,
+                          NULL);
   GNUNET_SET_iterate (iter_set,
                       &iter_cb,
                       iter_set);
@@ -307,8 +353,10 @@ run (void *cls,
      struct GNUNET_TESTING_Peer *peer)
 {
   config = cfg;
-  GNUNET_TESTING_peer_get_identity (peer, &local_id);
-  if (0) test_iter ();
+  GNUNET_TESTING_peer_get_identity (peer,
+                                    &local_id);
+  if (0)
+    test_iter ();
 
   tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
                                      &timeout_fail,
@@ -316,8 +364,10 @@ run (void *cls,
   GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                  NULL);
 
-  set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_INTERSECTION);
-  set2 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_INTERSECTION);
+  set1 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_INTERSECTION);
+  set2 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_INTERSECTION);
   GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
                                     &app_id);
 
@@ -327,9 +377,10 @@ run (void *cls,
 
 
 int
-main (int argc, char **argv)
+main (int argc,
+      char **argv)
 {
-  if (0 != GNUNET_TESTING_peer_run ("test_set_api",
+  if (0 != GNUNET_TESTING_peer_run ("test_set_intersection_result_full",
                                     "test_set.conf",
                                     &run, NULL))
     return 1;
diff --git a/src/set/test_set_union_copy.c b/src/set/test_set_union_copy.c
index 6fd78d43a..242b9f2f2 100644
--- a/src/set/test_set_union_copy.c
+++ b/src/set/test_set_union_copy.c
@@ -115,15 +115,23 @@ check_count_iter (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Expected count (what: %s) to be %u, but it's actually %u\n",
                   ci_cls->what,
-                  ci_cls->expected_count, ci_cls->ongoing_count);
+                  ci_cls->expected_count,
+                  ci_cls->ongoing_count);
       ret = 1;
+      GNUNET_SCHEDULER_shutdown ();
       return GNUNET_NO;
     }
     ci_cls->cont (ci_cls->cont_cls);
+    GNUNET_free (ci_cls);
     return GNUNET_NO;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Set `%s' has element %.*s\n",
+              ci_cls->what,
+              (int) element->size,
+              (const char *) element->data);
 
-  ci_cls->ongoing_count += 1;
+  ci_cls->ongoing_count++;
   return GNUNET_YES;
 }
 
@@ -137,6 +145,10 @@ check_count (struct GNUNET_SET_Handle *set,
 {
   struct CountIterClosure *ci_cls = GNUNET_new (struct CountIterClosure);
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Checking count of %s\n",
+              what);
+
   ci_cls->expected_count = expected_count;
   ci_cls->ongoing_count = 0;
   ci_cls->cont = cont;
@@ -162,7 +174,7 @@ check_new_set_count (void *cls)
 {
   check_count (set2,
                "new set",
-               4,
+               3,
                &test_done,
                NULL);
 }
@@ -172,15 +184,23 @@ static void
 copy_done (void *cls,
            struct GNUNET_SET_Handle *new_set)
 {
-  printf ("copy done\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "copy done\n");
   set2 = new_set;
-  remove_element_str (set2, "spam");
-  add_element_str (set2, "new1");
-  add_element_str (set2, "new2");
-  remove_element_str (set2, "new2");
-  remove_element_str (set2, "new3");
+  remove_element_str (set2,
+                      "k5555");
+  add_element_str (set2,
+                   "n66666");
+  add_element_str (set2,
+                   "new2butremoved");
+  remove_element_str (set2,
+                      "new2butremoved");
+  remove_element_str (set2,
+                      "new3justremoved");
   // Check that set1 didn't change.
-  check_count (set1, "old set", 3,
+  check_count (set1,
+               "old set",
+               3,
                &check_new_set_count,
                NULL);
 }
@@ -189,7 +209,8 @@ copy_done (void *cls,
 static void
 test_copy (void *cls)
 {
-  printf ("about to copy\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "about to copy\n");
   GNUNET_SET_copy_lazy (set1,
                         &copy_done,
                         NULL);
@@ -244,18 +265,27 @@ run (void *cls,
   GNUNET_TESTING_peer_get_identity (peer,
                                     &local_id);
 
-  set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
-  add_element_str (set1, "foo");
-  add_element_str (set1, "bar");
+  set1 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
+  add_element_str (set1,
+                   "333");
+  add_element_str (set1,
+                   "k444");
   /* duplicate -- ignored */
-  add_element_str (set1, "bar");
-  remove_element_str (set1, "foo");
+  add_element_str (set1,
+                   "k444");
+  remove_element_str (set1,
+                      "333");
   /* non-existent -- ignored */
-  remove_element_str (set1, "nonexist1");
-  add_element_str (set1, "spam");
+  remove_element_str (set1,
+                      "999999999");
+  add_element_str (set1,
+                   "k5555");
   /* duplicate -- ignored */
-  remove_element_str (set1, "foo");
-  add_element_str (set1, "eggs");
+  remove_element_str (set1,
+                      "333");
+  add_element_str (set1,
+                   "k2");
 
   check_count (set1,
                "initial test",
diff --git a/src/set/test_set_union_result_symmetric.c b/src/set/test_set_union_result_symmetric.c
index ab191a34a..f81c7b8f7 100644
--- a/src/set/test_set_union_result_symmetric.c
+++ b/src/set/test_set_union_result_symmetric.c
@@ -77,6 +77,7 @@ static struct GNUNET_SCHEDULER_Task *timeout_task;
 static void
 result_cb_set1 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t current_size,
                 enum GNUNET_SET_Status status)
 {
   switch (status)
@@ -125,6 +126,7 @@ result_cb_set1 (void *cls,
 static void
 result_cb_set2 (void *cls,
                 const struct GNUNET_SET_Element *element,
+                uint64_t current_size,
                 enum GNUNET_SET_Status status)
 {
   switch (status)
@@ -184,6 +186,7 @@ listen_cb (void *cls,
   listen_handle = NULL;
   oh2 = GNUNET_SET_accept (request,
                            GNUNET_SET_RESULT_SYMMETRIC,
+                           (struct GNUNET_SET_Option[]) { 0 },
                            &result_cb_set2,
                            NULL);
   GNUNET_SET_commit (oh2,
@@ -212,6 +215,7 @@ start (void *cls)
                             &app_id,
                             &context_msg,
                             GNUNET_SET_RESULT_SYMMETRIC,
+                            (struct GNUNET_SET_Option[]) { 0 },
                             &result_cb_set1, NULL);
   GNUNET_SET_commit (oh1, set1);
 }
diff --git a/src/social/.gitignore b/src/social/.gitignore
index 41954615f..875aa1105 100644
--- a/src/social/.gitignore
+++ b/src/social/.gitignore
@@ -1,2 +1,3 @@
 gnunet-social
 gnunet-service-social
+test_social
diff --git a/src/social/Makefile.am b/src/social/Makefile.am
index 693a61411..94a9ba108 100644
--- a/src/social/Makefile.am
+++ b/src/social/Makefile.am
@@ -62,7 +62,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/social/gnunet-service-social.c b/src/social/gnunet-service-social.c
index e6ee24a4f..dee68fdb8 100644
--- a/src/social/gnunet-service-social.c
+++ b/src/social/gnunet-service-social.c
@@ -1955,9 +1955,11 @@ handle_client_guest_enter_by_name (void *cls,
     GNUNET_memcpy (gcls->password, password, password_size);
   }
 
-  GNUNET_GNS_lookup (gns, gns_name, &greq->ego_pub_key,
-                     GNUNET_GNSRECORD_TYPE_PLACE, GNUNET_GNS_LO_DEFAULT,
-                     NULL, gns_result_guest_enter, gcls);
+  GNUNET_GNS_lookup (gns, gns_name,
+                     &greq->ego_pub_key,
+                     GNUNET_GNSRECORD_TYPE_PLACE,
+                     GNUNET_GNS_LO_DEFAULT,
+                     &gns_result_guest_enter, gcls);
 }
 
 
diff --git a/src/social/gnunet-social.c b/src/social/gnunet-social.c
index 6d72ca552..228d69d58 100644
--- a/src/social/gnunet-social.c
+++ b/src/social/gnunet-social.c
@@ -67,10 +67,10 @@ static int op_guest_leave;
 static int op_guest_talk;
 
 /** --replay */
-static char *op_replay;
+static int op_replay;
 
 /** --replay-latest */
-static char *op_replay_latest;
+static int op_replay_latest;
 
 /** --look-at */
 static int op_look_at;
@@ -116,13 +116,13 @@ static char *opt_data;
 static char *opt_name;
 
 /** --start */
-static uint64_t opt_start;
+static unsigned long long opt_start;
 
 /** --until */
-static uint64_t opt_until;
+static unsigned long long opt_until;
 
 /** --limit */
-static int opt_limit;
+static unsigned long long opt_limit;
 
 
 /* global vars */
@@ -154,6 +154,8 @@ struct GNUNET_SOCIAL_Host *hst;
 struct GNUNET_SOCIAL_Guest *gst;
 struct GNUNET_SOCIAL_Place *plc;
 
+const char *method_received;
+
 
 /* DISCONNECT */
 
@@ -308,7 +310,7 @@ static void
 guest_left (void *cls)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "The guest has left the place.\n");
+              "Guest has left the place.\n");
 }
 
 
@@ -555,10 +557,15 @@ slicer_recv_method (void *cls,
                     uint64_t message_id,
                     const char *method_name)
 {
+  method_received = method_name;
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
               "Received method for message ID %" PRIu64 ":\n"
               "%s (flags: %x)\n",
               message_id, method_name, ntohl (meth->flags));
+  /* routing header is missing, so we just print double newline */
+  printf("\n");
+  /* we output . instead of | to indicate that this is not proper PSYC syntax */
+  /* FIXME: use libpsyc here */
 }
 
 
@@ -576,10 +583,16 @@ slicer_recv_modifier (void *cls,
                       uint16_t value_size,
                       uint16_t full_value_size)
 {
+#if 0
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
               "Received modifier for message ID %" PRIu64 ":\n"
               "%c%s: %.*s (size: %u)\n",
               message_id, oper, name, value_size, (const char *) value, value_size);
+#else
+  /* obviously not binary safe */
+  printf("%c%s\t%.*s\n",
+              oper, name, value_size, (const char *) value);
+#endif
 }
 
 
@@ -594,10 +607,16 @@ slicer_recv_data (void *cls,
                   const void *data,
                   uint16_t data_size)
 {
+#if 0
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
               "Received data for message ID %" PRIu64 ":\n"
               "%.*s\n",
               message_id, data_size, (const char *) data);
+#else
+  /* obviously not binary safe */
+  printf("%s\n%.*s\n",
+              method_received, data_size, (const char *) data);
+#endif
 }
 
 
@@ -611,6 +630,7 @@ slicer_recv_eom (void *cls,
                 uint64_t message_id,
                 uint8_t is_cancelled)
 {
+  printf(".\n");
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
               "Received end of message ID %" PRIu64
               ", cancelled: %u\n",
@@ -685,7 +705,7 @@ guest_recv_local_enter (void *cls, int result,
 {
   char *pub_str = GNUNET_CRYPTO_eddsa_public_key_to_string (pub_key);
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "Guest entered to local place: %s, max_message_id: %" PRIu64 "\n",
+              "Guest entered local place: %s, max_message_id: %" PRIu64 "\n",
               pub_str, max_message_id);
   GNUNET_free (pub_str);
   GNUNET_assert (0 <= result);
@@ -998,6 +1018,7 @@ app_connected (void *cls)
       guest_enter (&place_pub_key, &peer);
     }
   }
+  printf(".\n");
 }
 
 
@@ -1012,8 +1033,7 @@ app_recv_host (void *cls,
                enum GNUNET_SOCIAL_AppPlaceState place_state)
 {
   char *host_pub_str = GNUNET_CRYPTO_eddsa_public_key_to_string (host_pub_key);
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "Host:  %s\n", host_pub_str);
+  printf ("Host\t%s\n", host_pub_str);
   GNUNET_free (host_pub_str);
 
   if ((op_host_reconnect || op_host_leave || op_host_announce || op_host_assign
@@ -1039,8 +1059,7 @@ app_recv_guest (void *cls,
                 enum GNUNET_SOCIAL_AppPlaceState place_state)
 {
   char *guest_pub_str = GNUNET_CRYPTO_eddsa_public_key_to_string (guest_pub_key);
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "Guest: %s\n", guest_pub_str);
+  printf ("Guest\t%s\n", guest_pub_str);
   GNUNET_free (guest_pub_str);
 
   if ((op_guest_reconnect || op_guest_leave || op_guest_talk
@@ -1065,7 +1084,7 @@ app_recv_ego (void *cls,
               const char *name)
 {
   char *s = GNUNET_CRYPTO_ecdsa_public_key_to_string (pub_key);
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ego:   %s\t%s\n", s, name);
+  printf ("Ego\t%s\t%s\n", s, name);
   GNUNET_free (s);
 
   if (0 == memcmp (&ego_pub_key, pub_key, sizeof (*pub_key))
@@ -1124,6 +1143,7 @@ run (void *cls, char *const *args, const char *cfgfile,
          || op_look_at || op_look_for))
   {
     op_status = 1;
+    fputs("Caution: This tool does not produce correct binary safe PSYC syntax.\n\n", stderr);
   }
 
   GNUNET_SCHEDULER_add_shutdown (scheduler_shutdown, NULL);
@@ -1144,6 +1164,7 @@ run (void *cls, char *const *args, const char *cfgfile,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 _("--place missing or invalid.\n"));
+    /* FIXME: why does it segfault here? */
     exit_fail ();
     return;
   }
@@ -1178,7 +1199,7 @@ int
 main (int argc, char *const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
     /*
      * gnunet program options in addition to the ones below:
      *
@@ -1191,120 +1212,165 @@ main (int argc, char *const *argv)
 
     /* operations */
 
-    { 'A', "host-assign", NULL,
-      gettext_noop ("assign --name in state to --data"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_assign },
-
-    { 'B', "guest-leave", NULL,
-      gettext_noop ("say good-bye and leave somebody else's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_leave },
-
-    { 'C', "host-enter", NULL,
-      gettext_noop ("create a place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_enter },
-
-    { 'D', "host-leave", NULL,
-      gettext_noop ("destroy a place we were hosting"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_leave },
-
-    { 'E', "guest-enter", NULL,
-      gettext_noop ("enter somebody else's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_enter },
-
-    { 'F', "look-for", NULL,
-      gettext_noop ("find state matching name prefix"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_look_for },
-
-    { 'H', "replay-latest", NULL,
-      gettext_noop ("replay history of messages up to the given --limit"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_replay_latest },
-
-    { 'N', "host-reconnect", NULL,
-      gettext_noop ("reconnect to a previously created place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_reconnect },
-
-    { 'P', "host-announce", NULL,
-      gettext_noop ("publish something to a place we are hosting"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_announce },
-
-    { 'R', "guest-reconnect", NULL,
-      gettext_noop ("reconnect to a previously entered place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_reconnect },
-
-    { 'S', "look-at", NULL,
-      gettext_noop ("search for state matching exact name"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_look_at },
-
-    { 'T', "guest-talk", NULL,
-      gettext_noop ("submit something to somebody's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_talk },
-
-    { 'U', "status", NULL,
-      gettext_noop ("list of egos and subscribed places"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_status },
-
-    { 'X', "replay", NULL,
-      gettext_noop ("extract and replay history between message IDs --start and --until"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_replay },
+    GNUNET_GETOPT_option_flag ('A',
+                                  "host-assign",
+                                  gettext_noop ("assign --name in state to --data"),
+                                  &op_host_assign),
+
+    GNUNET_GETOPT_option_flag ('B',
+                                  "guest-leave",
+                                  gettext_noop ("say good-bye and leave somebody else's place"),
+                                  &op_guest_leave),
+
+    GNUNET_GETOPT_option_flag ('C',
+                                  "host-enter",
+                                  gettext_noop ("create a place"),
+                                  &op_host_enter),
+
+    GNUNET_GETOPT_option_flag ('C',
+                                  "host-enter",
+                                  gettext_noop ("create a place"),
+                                  &op_host_enter),
+
+    GNUNET_GETOPT_option_flag ('D',
+                                  "host-leave",
+                                  gettext_noop ("destroy a place we were hosting"),
+                                  &op_host_leave),
+
+    GNUNET_GETOPT_option_flag ('E',
+                                  "guest-enter",
+                                  gettext_noop ("enter somebody else's place"),
+                                  &op_guest_enter),
+
+
+    GNUNET_GETOPT_option_flag ('F',
+                                  "look-for",
+                                  gettext_noop ("find state matching name prefix"),
+                                  &op_look_for),
+
+    GNUNET_GETOPT_option_flag ('H',
+                                  "replay-latest",
+                                  gettext_noop ("replay history of messages up to the given --limit"),
+                                  &op_replay_latest),
+
+    GNUNET_GETOPT_option_flag ('N',
+                                  "host-reconnect",
+                                  gettext_noop ("reconnect to a previously created place"),
+                                  &op_host_reconnect),
+
+    GNUNET_GETOPT_option_flag ('P',
+                                  "host-announce",
+                                  gettext_noop ("publish something to a place we are hosting"),
+                                  &op_host_announce),
+
+    GNUNET_GETOPT_option_flag ('R',
+                                  "guest-reconnect",
+                                  gettext_noop ("reconnect to a previously entered place"),
+                                  &op_guest_reconnect),
+
+    GNUNET_GETOPT_option_flag ('S',
+                                  "look-at",
+                                  gettext_noop ("search for state matching exact name"),
+                                  &op_look_at),
+
+    GNUNET_GETOPT_option_flag ('T',
+                                  "guest-talk",
+                                  gettext_noop ("submit something to somebody's place"),
+                                  &op_guest_talk),
+
+    GNUNET_GETOPT_option_flag ('U',
+                                  "status",
+                                  gettext_noop ("list of egos and subscribed places"),
+                                  &op_status),
+
+    GNUNET_GETOPT_option_flag ('X',
+                                  "replay",
+                                  gettext_noop ("extract and replay history between message IDs --start and --until"),
+                                  &op_replay),
 
 
     /* options */
 
-    { 'a', "app", "APPLICATION_ID",
-      gettext_noop ("application ID to use when connecting"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_app },
-
-    { 'd', "data", "DATA",
-      gettext_noop ("message body or state value"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_data },
-
-    { 'e', "ego", "NAME|PUBKEY",
-      gettext_noop ("name or public key of ego"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_ego },
-
-    { 'f', "follow", NULL,
-      gettext_noop ("wait for incoming messages"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_follow },
-
-    { 'g', "gns", "GNS_NAME",
-      gettext_noop ("GNS name"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_gns },
-
-    { 'i', "peer", "PEER_ID",
-      gettext_noop ("peer ID for --guest-enter"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_peer },
-
-    { 'k', "name", "VAR_NAME",
-      gettext_noop ("name (key) to query from state"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_name },
-
-    { 'm', "method", "METHOD_NAME",
-      gettext_noop ("method name"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_method },
-
-    { 'n', "limit", NULL,
-      gettext_noop ("number of messages to replay from history"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_limit },
-
-    { 'p', "place", "PUBKEY",
-      gettext_noop ("key address of place"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_place },
-
-    { 's', "start", NULL,
-      gettext_noop ("start message ID for history replay"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_start },
-
-    { 'w', "welcome", NULL,
-      gettext_noop ("respond to entry requests by admitting all guests"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_welcome },
-
-    { 'u', "until", NULL,
-      gettext_noop ("end message ID for history replay"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_until },
-
-    { 'y', "deny", NULL,
-      gettext_noop ("respond to entry requests by refusing all guests"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_deny },
+    GNUNET_GETOPT_option_string ('a',
+                                 "app",
+                                 "APPLICATION_ID",
+                                 gettext_noop ("application ID to use when connecting"),
+                                 &opt_app),
+
+    GNUNET_GETOPT_option_string ('d',
+                                 "data",
+                                 "DATA",
+                                 gettext_noop ("message body or state value"),
+                                 &opt_data),
+
+    GNUNET_GETOPT_option_string ('e',
+                                 "ego",
+                                 "NAME|PUBKEY",
+                                 gettext_noop ("name or public key of ego"),
+                                 &opt_ego),
+
+    GNUNET_GETOPT_option_flag ('f',
+                                  "follow",
+                                  gettext_noop ("wait for incoming messages"),
+                                  &opt_follow),
+
+    GNUNET_GETOPT_option_string ('g',
+                                 "gns",
+                                 "GNS_NAME",
+                                 gettext_noop ("GNS name"),
+                                 &opt_gns),
+
+    GNUNET_GETOPT_option_string ('i',
+                                 "peer",
+                                 "PEER_ID",
+                                 gettext_noop ("peer ID for --guest-enter"),
+                                 &opt_peer),
+
+    GNUNET_GETOPT_option_string ('k',
+                                 "name",
+                                 "VAR_NAME",
+                                 gettext_noop ("name (key) to query from state"),
+                                 &opt_name),
+
+    GNUNET_GETOPT_option_string ('m',
+                                 "method",
+                                 "METHOD_NAME",
+                                 gettext_noop ("method name"),
+                                 &opt_method),
+
+    GNUNET_GETOPT_option_ulong ('n',
+                                    "limit",
+                                    NULL,
+                                    gettext_noop ("number of messages to replay from history"),
+                                    &opt_limit),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "place",
+                                 "PUBKEY",
+                                 gettext_noop ("key address of place"),
+                                 &opt_place),
+
+    GNUNET_GETOPT_option_ulong ('s',
+                                    "start",
+                                    NULL,
+                                    gettext_noop ("start message ID for history replay"),
+                                    &opt_start),
+
+    GNUNET_GETOPT_option_flag ('w',
+                                  "welcome",
+                                  gettext_noop ("respond to entry requests by admitting all guests"),
+                                  &opt_welcome),
+
+    GNUNET_GETOPT_option_ulong ('u',
+                                    "until",
+                                    NULL,
+                                    gettext_noop ("end message ID for history replay"),
+                                    &opt_until),
+
+    GNUNET_GETOPT_option_flag ('y',
+                                  "deny",
+                                  gettext_noop ("respond to entry requests by refusing all guests"),
+                                  &opt_deny),
 
     GNUNET_GETOPT_OPTION_END
   };
@@ -1333,8 +1399,8 @@ main (int argc, char *const *argv)
     "gnunet-social --guest-leave --place <PUBKEY>\n"
     "gnunet-social --guest-talk --place <PUBKEY> --method <METHOD_NAME> --data <MESSAGE_BODY>\n"
     "\n"
-    "gnunet-social --history-replay --place <PUBKEY> --start <MSGID> --until <MSGID>  [--method <METHOD_PREFIX>]\n"
-    "gnunet-social --history-replay-latest --place <PUBKEY> --limit <MSG_LIMIT> [--method <METHOD_PREFIX>]\n"
+    "gnunet-social --replay --place <PUBKEY> --start <MSGID> --until <MSGID>  [--method <METHOD_PREFIX>]\n"
+    "gnunet-social --replay-latest --place <PUBKEY> --limit <MSG_LIMIT> [--method <METHOD_PREFIX>]\n"
     "\n"
     "gnunet-social --look-at --place <PUBKEY> --name <FULL_NAME>\n"
     "gnunet-social --look-for --place <PUBKEY> --name <NAME_PREFIX>\n";
diff --git a/src/social/social_api.c b/src/social/social_api.c
index a7fe0916f..af1d6e57e 100644
--- a/src/social/social_api.c
+++ b/src/social/social_api.c
@@ -2110,7 +2110,7 @@ GNUNET_SOCIAL_place_msg_proc_set (struct GNUNET_SOCIAL_Place *plc,
   GNUNET_assert (NULL != method_prefix);
   struct MsgProcRequest *mpreq;
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*mpreq)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -2159,7 +2159,7 @@ place_history_replay (struct GNUNET_SOCIAL_Place *plc,
 
   GNUNET_assert (NULL != method_prefix);
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -2285,7 +2285,7 @@ place_state_get (struct GNUNET_SOCIAL_Place *plc,
   look->op_id = GNUNET_OP_add (plc->op, &op_recv_state_result, look, NULL);
 
   GNUNET_assert (NULL != name);
-  size_t name_size = strnlen (name, GNUNET_SERVER_MAX_MESSAGE_SIZE
+  size_t name_size = strnlen (name, GNUNET_MAX_MESSAGE_SIZE
                               - sizeof (*req)) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (req, name_size, type);
@@ -2426,7 +2426,7 @@ GNUNET_SOCIAL_zone_add_place (const struct GNUNET_SOCIAL_App *app,
   size_t relay_size = relay_count * sizeof (*relays);
   size_t payload_size = name_size + password_size + relay_size;
 
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < sizeof (*preq) + payload_size)
+  if (GNUNET_MAX_MESSAGE_SIZE < sizeof (*preq) + payload_size)
     return GNUNET_SYSERR;
 
   struct GNUNET_MQ_Envelope *
@@ -2506,7 +2506,7 @@ GNUNET_SOCIAL_zone_add_nym (const struct GNUNET_SOCIAL_App *app,
   struct ZoneAddNymRequest *nreq;
 
   size_t name_size = strlen (name) + 1;
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < sizeof (*nreq) + name_size)
+  if (GNUNET_MAX_MESSAGE_SIZE < sizeof (*nreq) + name_size)
     return GNUNET_SYSERR;
 
   struct GNUNET_MQ_Envelope *
diff --git a/src/sq/.gitignore b/src/sq/.gitignore
new file mode 100644
index 000000000..951587047
--- /dev/null
+++ b/src/sq/.gitignore
@@ -0,0 +1 @@
+test_sq
diff --git a/src/sq/Makefile.am b/src/sq/Makefile.am
new file mode 100644
index 000000000..cfccd89f6
--- /dev/null
+++ b/src/sq/Makefile.am
@@ -0,0 +1,40 @@
+# This Makefile.am is in the public domain
+AM_CPPFLAGS = -I$(top_srcdir)/src/include $(POSTGRESQL_CPPFLAGS)
+
+if MINGW
+  WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols
+endif
+
+if USE_COVERAGE
+  AM_CFLAGS = --coverage
+endif
+
+if HAVE_SQLITE
+lib_LTLIBRARIES = libgnunetsq.la
+endif
+
+libgnunetsq_la_SOURCES = \
+  sq.c \
+  sq_query_helper.c \
+  sq_result_helper.c
+libgnunetsq_la_LIBADD = -lsqlite3 \
+ $(top_builddir)/src/util/libgnunetutil.la
+libgnunetsq_la_LDFLAGS = \
+ $(POSTGRESQL_LDFLAGS) \
+ $(GN_LIB_LDFLAGS) \
+  -version-info 0:0:0
+
+if ENABLE_TEST_RUN
+TESTS = \
+ test_sq
+endif
+
+check_PROGRAMS= \
+ test_sq
+
+test_sq_SOURCES = \
+  test_sq.c
+test_sq_LDADD = \
+  libgnunetsq.la \
+  $(top_builddir)/src/util/libgnunetutil.la  \
+  -lsqlite3 $(XLIB)
diff --git a/src/sq/sq.c b/src/sq/sq.c
new file mode 100644
index 000000000..089ebf0ff
--- /dev/null
+++ b/src/sq/sq.c
@@ -0,0 +1,140 @@
+/*
+  This file is part of GNUnet
+  Copyright (C) 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file sq/sq.c
+ * @brief helper functions for Sqlite3 DB interactions
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_sq_lib.h"
+
+
+/**
+ * Execute a prepared statement.
+ *
+ * @param db_conn database connection
+ * @param params parameters to the statement
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_SQ_bind (sqlite3_stmt *stmt,
+                const struct GNUNET_SQ_QueryParam *params)
+{
+  unsigned int j;
+
+  j = 1;
+  for (unsigned int i=0;NULL != params[i].conv; i++)
+  {
+    if (GNUNET_OK !=
+        params[i].conv (params[i].conv_cls,
+                        params[i].data,
+                        params[i].size,
+                        stmt,
+                        j))
+    {
+      GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                       "sq",
+                       _("Failure to bind %u-th SQL parameter\n"),
+                       i);
+      if (SQLITE_OK !=
+          sqlite3_reset (stmt))
+      {
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sq",
+                         _("Failure in sqlite3_reset (!)\n"));
+        return GNUNET_SYSERR;
+      }
+    }
+    GNUNET_assert (0 != params[i].num_params);
+    j += params[i].num_params;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Extract results from a query result according to the given specification.
+ *
+ * @param result result to process
+ * @param[in,out] rs result specification to extract for
+ * @return
+ *   #GNUNET_OK if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field)
+ */
+int
+GNUNET_SQ_extract_result (sqlite3_stmt *result,
+                          struct GNUNET_SQ_ResultSpec *rs)
+{
+  unsigned int j = 0;
+
+  for (unsigned int i=0;NULL != rs[i].conv; i++)
+  {
+    if (NULL == rs[i].result_size)
+      rs[i].result_size = &rs[i].dst_size;
+    if (GNUNET_OK !=
+        rs[i].conv (rs[i].cls,
+                    result,
+                    j,
+                    rs[i].result_size,
+                    rs[i].dst))
+    {
+      for (unsigned int k=0;k<i;k++)
+        if (NULL != rs[k].cleaner)
+          rs[k].cleaner (rs[k].cls);
+      return GNUNET_SYSERR;
+    }
+    GNUNET_assert (0 != rs[i].num_params);
+    j += rs[i].num_params;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Free all memory that was allocated in @a rs during
+ * #GNUNET_SQ_extract_result().
+ *
+ * @param rs reult specification to clean up
+ */
+void
+GNUNET_SQ_cleanup_result (struct GNUNET_SQ_ResultSpec *rs)
+{
+  for (unsigned int i=0;NULL != rs[i].conv; i++)
+    if (NULL != rs[i].cleaner)
+      rs[i].cleaner (rs[i].cls);
+}
+
+
+/**
+ * Reset @a stmt and log error.
+ *
+ * @param dbh database handle
+ * @param stmt statement to reset
+ */
+void
+GNUNET_SQ_reset (sqlite3 *dbh,
+                 sqlite3_stmt *stmt)
+{
+  if (SQLITE_OK !=
+      sqlite3_reset (stmt))
+    GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                     "sqlite",
+                     _("Failed to reset sqlite statement with error: %s\n"),
+                     sqlite3_errmsg (dbh));
+}
+
+
+/* end of sq.c */
diff --git a/src/sq/sq_query_helper.c b/src/sq/sq_query_helper.c
new file mode 100644
index 000000000..94a3a3f1c
--- /dev/null
+++ b/src/sq/sq_query_helper.c
@@ -0,0 +1,493 @@
+/*
+  This file is part of GNUnet
+  Copyright (C) 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file sq/sq_query_helper.c
+ * @brief helper functions for queries
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_sq_lib.h"
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_fixed_blob (void *cls,
+                 const void *data,
+                 size_t data_len,
+                 sqlite3_stmt *stmt,
+                 unsigned int off)
+{
+  if (SQLITE_OK !=
+      sqlite3_bind_blob64 (stmt,
+                           (int) off,
+                           data,
+                           (sqlite3_uint64) data_len,
+                           SQLITE_TRANSIENT))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for a buffer @a ptr of
+ * @a ptr_size bytes.
+ *
+ * @param ptr pointer to the query parameter to pass
+ * @oaran ptr_size number of bytes in @a ptr
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_fixed_size (const void *ptr,
+                                  size_t ptr_size)
+{
+  struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_fixed_blob,
+    .data = ptr,
+    .size = ptr_size,
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_string (void *cls,
+             const void *data,
+             size_t data_len,
+             sqlite3_stmt *stmt,
+             unsigned int off)
+{
+  if (NULL == data)
+  {
+    if (SQLITE_OK !=
+        sqlite3_bind_null (stmt,
+                           (int) off))
+      return GNUNET_SYSERR;
+    return GNUNET_OK;
+  }
+  if (SQLITE_OK !=
+      sqlite3_bind_text (stmt,
+                         (int) off,
+                         (const char *) data,
+                         -1,
+                         SQLITE_TRANSIENT))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for a string.
+ *
+ * @param ptr pointer to the string query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_string (const char *ptr)
+{
+  struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_string,
+    .data = ptr,
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_rsa_pub (void *cls,
+              const void *data,
+              size_t data_len,
+              sqlite3_stmt *stmt,
+              unsigned int off)
+{
+  const struct GNUNET_CRYPTO_RsaPublicKey *rsa = data;
+  char *buf;
+  size_t buf_size;
+
+  GNUNET_break (NULL == cls);
+  buf_size = GNUNET_CRYPTO_rsa_public_key_encode (rsa,
+                                                  &buf);
+  if (SQLITE_OK !=
+      sqlite3_bind_blob64 (stmt,
+                           (int) off,
+                           buf,
+                           (sqlite3_uint64) buf_size,
+                           SQLITE_TRANSIENT))
+  {
+    GNUNET_free (buf);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (buf);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an RSA public key.  The
+ * database must contain a BLOB type in the respective position.
+ *
+ * @param x the query parameter to pass.
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_rsa_public_key (const struct GNUNET_CRYPTO_RsaPublicKey *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_rsa_pub,
+    .data = x,
+    .num_params = 1
+  };
+ return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_rsa_sig (void *cls,
+              const void *data,
+              size_t data_len,
+              sqlite3_stmt *stmt,
+              unsigned int off)
+{
+  const struct GNUNET_CRYPTO_RsaSignature *sig = data;
+  char *buf;
+  size_t buf_size;
+
+  GNUNET_break (NULL == cls);
+  buf_size = GNUNET_CRYPTO_rsa_signature_encode (sig,
+                                                 &buf);
+  if (SQLITE_OK !=
+      sqlite3_bind_blob64 (stmt,
+                           (int) off,
+                           buf,
+                           (sqlite3_uint64) buf_size,
+                           SQLITE_TRANSIENT))
+  {
+    GNUNET_free (buf);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (buf);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an RSA signature.  The
+ * database must contain a BLOB type in the respective position.
+ *
+ * @param x the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_rsa_sig,
+    .data = x,
+    .num_params = 1
+  };
+ return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_abstime (void *cls,
+              const void *data,
+              size_t data_len,
+              sqlite3_stmt *stmt,
+              unsigned int off)
+{
+  const struct GNUNET_TIME_Absolute *u = data;
+  struct GNUNET_TIME_Absolute abs;
+
+  abs = *u;
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
+  GNUNET_assert (sizeof (uint64_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int64 (stmt,
+                          (int) off,
+                          (sqlite3_int64) abs.abs_value_us))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an absolute time value.
+ * The database must store a 64-bit integer.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_absolute_time (const struct GNUNET_TIME_Absolute *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_abstime,
+    .data = x,
+    .size = sizeof (struct GNUNET_TIME_Absolute),
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_nbotime (void *cls,
+              const void *data,
+              size_t data_len,
+              sqlite3_stmt *stmt,
+              unsigned int off)
+{
+  const struct GNUNET_TIME_AbsoluteNBO *u = data;
+  struct GNUNET_TIME_Absolute abs;
+
+  abs = GNUNET_TIME_absolute_ntoh (*u);
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
+  GNUNET_assert (sizeof (uint64_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int64 (stmt,
+                          (int) off,
+                          (sqlite3_int64) abs.abs_value_us))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an absolute time value.
+ * The database must store a 64-bit integer.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_absolute_time_nbo (const struct GNUNET_TIME_AbsoluteNBO *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_nbotime,
+    .data = x,
+    .size = sizeof (struct GNUNET_TIME_AbsoluteNBO),
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_u16 (void *cls,
+          const void *data,
+          size_t data_len,
+          sqlite3_stmt *stmt,
+          unsigned int off)
+{
+  const uint16_t *u = data;
+
+  GNUNET_assert (sizeof (uint16_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int (stmt,
+                        (int) off,
+                        (int) *u))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an uint16_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint16 (const uint16_t *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_u16,
+    .data = x,
+    .size = sizeof (uint16_t),
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_u32 (void *cls,
+          const void *data,
+          size_t data_len,
+          sqlite3_stmt *stmt,
+          unsigned int off)
+{
+  const uint32_t *u = data;
+
+  GNUNET_assert (sizeof (uint32_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int64 (stmt,
+                          (int) off,
+                          (sqlite3_int64) *u))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+/**
+ * Generate query parameter for an uint32_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint32 (const uint32_t *x)
+{
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_u32,
+    .data = x,
+    .size = sizeof (uint32_t),
+    .num_params = 1
+  };
+  return qp;
+}
+
+
+/**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_u64 (void *cls,
+          const void *data,
+          size_t data_len,
+          sqlite3_stmt *stmt,
+          unsigned int off)
+{
+  const uint64_t *u = data;
+
+  GNUNET_assert (sizeof (uint64_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int64 (stmt,
+                          (int) off,
+                          (sqlite3_int64) *u))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generate query parameter for an uint16_t in host byte order.
+ *
+ * @param x pointer to the query parameter to pass
+ */
+struct GNUNET_SQ_QueryParam
+GNUNET_SQ_query_param_uint64 (const uint64_t *x)
+{
+  struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_u64,
+    .data = x,
+    .size = sizeof (uint64_t),
+    .num_params = 1
+  };
+  return qp;
+}
+
+/* end of sq_query_helper.c */
diff --git a/src/sq/sq_result_helper.c b/src/sq/sq_result_helper.c
new file mode 100644
index 000000000..9579863b2
--- /dev/null
+++ b/src/sq/sq_result_helper.c
@@ -0,0 +1,782 @@
+
+/*
+  This file is part of GNUnet
+  Copyright (C) 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file sq/sq_result_helper.c
+ * @brief helper functions for queries
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_sq_lib.h"
+
+
+/**
+ * Extract variable-sized binary data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result (actually a `void **`)
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_var_blob (void *cls,
+                  sqlite3_stmt *result,
+                  unsigned int column,
+                  size_t *dst_size,
+                  void *dst)
+{
+  int have;
+  const void *ret;
+  void **rdst = (void **) dst;
+
+  if (SQLITE_NULL ==
+      sqlite3_column_type (result,
+                           column))
+  {
+    *rdst = NULL;
+    *dst_size = 0;
+    return GNUNET_YES;
+  }
+
+  if (SQLITE_BLOB !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* sqlite manual says to invoke 'sqlite3_column_blob()'
+     before calling sqlite3_column_bytes() */
+  ret = sqlite3_column_blob (result,
+                             column);
+  have = sqlite3_column_bytes (result,
+                               column);
+  if (have < 0)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *dst_size = have;
+  if (0 == have)
+  {
+    *rdst = NULL;
+    return GNUNET_OK;
+  }
+  *rdst = GNUNET_malloc (have);
+  GNUNET_memcpy (*rdst,
+                 ret,
+                 have);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Cleanup memory allocated by #extract_var_blob().
+ *
+ * @param cls pointer to pointer of allocation
+ */
+static void
+clean_var_blob (void *cls)
+{
+  void **dptr = (void **) cls;
+
+  if (NULL != *dptr)
+  {
+    GNUNET_free (*dptr);
+    *dptr = NULL;
+  }
+}
+
+
+/**
+ * Variable-size result expected.
+ *
+ * @param[out] dst where to store the result, allocated
+ * @param[out] sptr where to store the size of @a dst
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_variable_size (void **dst,
+                                     size_t *sptr)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_var_blob,
+    .cleaner = &clean_var_blob,
+    .dst = dst,
+    .cls = dst,
+    .result_size = sptr,
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract fixed-sized binary data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_fixed_blob (void *cls,
+                    sqlite3_stmt *result,
+                    unsigned int column,
+                    size_t *dst_size,
+                    void *dst)
+{
+  int have;
+  const void *ret;
+
+  if ( (0 == *dst_size) &&
+       (SQLITE_NULL ==
+        sqlite3_column_type (result,
+                             column)) )
+  {
+    return GNUNET_YES;
+  }
+
+  if (SQLITE_BLOB !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* sqlite manual says to invoke 'sqlite3_column_blob()'
+     before calling sqlite3_column_bytes() */
+  ret = sqlite3_column_blob (result,
+                             column);
+  have = sqlite3_column_bytes (result,
+                               column);
+  if (*dst_size != have)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_memcpy (dst,
+                 ret,
+                 have);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Fixed-size result expected.
+ *
+ * @param[out] dst where to store the result
+ * @param dst_size number of bytes in @a dst
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_fixed_size (void *dst,
+                                  size_t dst_size)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_fixed_blob,
+    .dst = dst,
+    .dst_size = dst_size,
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract fixed-sized binary data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_utf8_string (void *cls,
+                     sqlite3_stmt *result,
+                     unsigned int column,
+                     size_t *dst_size,
+                     void *dst)
+{
+  const char *text;
+  char **rdst = dst;
+
+  if (SQLITE_NULL ==
+      sqlite3_column_type (result,
+                           column))
+  {
+    *rdst = NULL;
+    return GNUNET_OK;
+  }
+  if (SQLITE_TEXT !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* sqlite manual guarantees that 'sqlite3_column_text()'
+     is 0-terminated */
+  text = (const char *) sqlite3_column_text (result,
+                                             column);
+  if (NULL == text)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *dst_size = strlen (text) + 1;
+  *rdst = GNUNET_strdup (text);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Cleanup memory allocated by #extract_var_blob().
+ *
+ * @param cls pointer to pointer of allocation
+ */
+static void
+clean_utf8_string (void *cls)
+{
+  char **dptr = (char **) cls;
+
+  if (NULL != *dptr)
+  {
+    GNUNET_free (*dptr);
+    *dptr = NULL;
+  }
+}
+
+
+/**
+ * 0-terminated string expected.
+ *
+ * @param[out] dst where to store the result, allocated
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_string (char **dst)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_utf8_string,
+    .cleaner = &clean_utf8_string,
+    .cls = dst,
+    .dst = dst,
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_rsa_pub (void *cls,
+                 sqlite3_stmt *result,
+                 unsigned int column,
+                 size_t *dst_size,
+                 void *dst)
+{
+  struct GNUNET_CRYPTO_RsaPublicKey **pk = dst;
+  int have;
+  const void *ret;
+
+  if (SQLITE_BLOB !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* sqlite manual says to invoke 'sqlite3_column_blob()'
+     before calling sqlite3_column_bytes() */
+  ret = sqlite3_column_blob (result,
+                             column);
+  have = sqlite3_column_bytes (result,
+                               column);
+  if (have < 0)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+
+  *pk = GNUNET_CRYPTO_rsa_public_key_decode (ret,
+                                             have);
+  if (NULL == *pk)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called to clean up memory allocated
+ * by a #GNUNET_PQ_ResultConverter.
+ *
+ * @param cls closure
+ */
+static void
+clean_rsa_pub (void *cls)
+{
+  struct GNUNET_CRYPTO_RsaPublicKey **pk = cls;
+
+  if (NULL != *pk)
+  {
+    GNUNET_CRYPTO_rsa_public_key_free (*pk);
+    *pk = NULL;
+  }
+}
+
+
+/**
+ * RSA public key expected.
+ *
+ * @param[out] rsa where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_rsa_public_key (struct GNUNET_CRYPTO_RsaPublicKey **rsa)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_rsa_pub,
+    .cleaner = &clean_rsa_pub,
+    .dst = rsa,
+    .cls = rsa,
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract data from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_rsa_sig (void *cls,
+                 sqlite3_stmt *result,
+                 unsigned int column,
+                 size_t *dst_size,
+                 void *dst)
+{
+  struct GNUNET_CRYPTO_RsaSignature **sig = dst;
+  int have;
+  const void *ret;
+
+  if (SQLITE_BLOB !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* sqlite manual says to invoke 'sqlite3_column_blob()'
+     before calling sqlite3_column_bytes() */
+  ret = sqlite3_column_blob (result,
+                             column);
+  have = sqlite3_column_bytes (result,
+                               column);
+  if (have < 0)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+
+  *sig = GNUNET_CRYPTO_rsa_signature_decode (ret,
+                                             have);
+  if (NULL == *sig)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called to clean up memory allocated
+ * by a #GNUNET_PQ_ResultConverter.
+ *
+ * @param cls result data to clean up
+ */
+static void
+clean_rsa_sig (void *cls)
+{
+  struct GNUNET_CRYPTO_RsaSignature **sig = cls;
+
+  if (NULL != *sig)
+  {
+    GNUNET_CRYPTO_rsa_signature_free (*sig);
+    *sig = NULL;
+  }
+}
+
+
+/**
+ * RSA signature expected.
+ *
+ * @param[out] sig where to store the result;
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_rsa_signature (struct GNUNET_CRYPTO_RsaSignature **sig)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_rsa_sig,
+    .cleaner = &clean_rsa_sig,
+    .dst = sig,
+    .cls = sig,
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract absolute time value from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_abs_time (void *cls,
+                  sqlite3_stmt *result,
+                  unsigned int column,
+                  size_t *dst_size,
+                  void *dst)
+{
+  struct GNUNET_TIME_Absolute *u = dst;
+  struct GNUNET_TIME_Absolute t;
+
+  GNUNET_assert (sizeof (uint64_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  t.abs_value_us = (uint64_t) sqlite3_column_int64 (result,
+                                                    column);
+  if (INT64_MAX == t.abs_value_us)
+    t = GNUNET_TIME_UNIT_FOREVER_ABS;
+  *u = t;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Absolute time expected.
+ *
+ * @param[out] at where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_absolute_time (struct GNUNET_TIME_Absolute *at)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_abs_time,
+    .dst = at,
+    .dst_size = sizeof (struct GNUNET_TIME_Absolute),
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract absolute time value in NBO from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_abs_time_nbo (void *cls,
+                      sqlite3_stmt *result,
+                      unsigned int column,
+                      size_t *dst_size,
+                      void *dst)
+{
+  struct GNUNET_TIME_AbsoluteNBO *u = dst;
+  struct GNUNET_TIME_Absolute t;
+
+  GNUNET_assert (sizeof (uint64_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  t.abs_value_us = (uint64_t) sqlite3_column_int64 (result,
+                                                    column);
+  if (INT64_MAX == t.abs_value_us)
+    t = GNUNET_TIME_UNIT_FOREVER_ABS;
+  *u = GNUNET_TIME_absolute_hton (t);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Absolute time expected.
+ *
+ * @param[out] at where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_absolute_time_nbo (struct GNUNET_TIME_AbsoluteNBO *at)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_abs_time_nbo,
+    .dst = at,
+    .dst_size = sizeof (struct GNUNET_TIME_AbsoluteNBO),
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract 16-bit integer from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_uint16 (void *cls,
+                sqlite3_stmt *result,
+                unsigned int column,
+                size_t *dst_size,
+                void *dst)
+{
+  uint64_t v;
+  uint32_t *u = dst;
+
+  GNUNET_assert (sizeof (uint16_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  v = (uint64_t) sqlite3_column_int64 (result,
+                                       column);
+  if (v > UINT16_MAX)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *u = (uint16_t) v;
+  return GNUNET_OK;
+}
+
+
+/**
+ * uint16_t expected.
+ *
+ * @param[out] u16 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint16 (uint16_t *u16)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_uint16,
+    .dst = u16,
+    .dst_size = sizeof (uint16_t),
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract 32-bit integer from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_uint32 (void *cls,
+                sqlite3_stmt *result,
+                unsigned int column,
+                size_t *dst_size,
+                void *dst)
+{
+  uint64_t v;
+  uint32_t *u = dst;
+
+  GNUNET_assert (sizeof (uint32_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  v = (uint64_t) sqlite3_column_int64 (result,
+                                       column);
+  if (v > UINT32_MAX)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *u = (uint32_t) v;
+  return GNUNET_OK;
+}
+
+
+/**
+ * uint32_t expected.
+ *
+ * @param[out] u32 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint32 (uint32_t *u32)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_uint32,
+    .dst = u32,
+    .dst_size = sizeof (uint32_t),
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/**
+ * Extract 64-bit integer from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_uint64 (void *cls,
+                sqlite3_stmt *result,
+                unsigned int column,
+                size_t *dst_size,
+                void *dst)
+{
+  uint64_t *u = dst;
+
+  GNUNET_assert (sizeof (uint64_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *u = (uint64_t) sqlite3_column_int64 (result,
+                                        column);
+  return GNUNET_OK;
+}
+
+
+/**
+ * uint64_t expected.
+ *
+ * @param[out] u64 where to store the result
+ * @return array entry for the result specification to use
+ */
+struct GNUNET_SQ_ResultSpec
+GNUNET_SQ_result_spec_uint64 (uint64_t *u64)
+{
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_uint64,
+    .dst = u64,
+    .dst_size = sizeof (uint64_t),
+    .num_params = 1
+  };
+
+  return rs;
+}
+
+
+/* end of sq_result_helper.c */
diff --git a/src/sq/test_sq.c b/src/sq/test_sq.c
new file mode 100644
index 000000000..713809030
--- /dev/null
+++ b/src/sq/test_sq.c
@@ -0,0 +1,285 @@
+/*
+  This file is part of GNUnet
+  (C) 2015, 2016, 2017 GNUnet e.V.
+
+  GNUnet is free software; you can redistribute it and/or modify it under the
+  terms of the GNU General Public License as published by the Free Software
+  Foundation; either version 3, or (at your option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY
+  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along with
+  GNUnet; see the file COPYING.  If not, If not, see <http://www.gnu.org/licenses/>
+*/
+/**
+ * @file sq/test_sq.c
+ * @brief Tests for sqlite3 convenience API
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_sq_lib.h"
+
+
+/**
+ * @brief Prepare a SQL statement
+ *
+ * @param dbh handle to the database
+ * @param zSql SQL statement, UTF-8 encoded
+ * @param[out] ppStmt set to the prepared statement
+ * @return 0 on success
+ */
+static int
+sq_prepare (sqlite3 *dbh,
+            const char *zSql,
+            sqlite3_stmt **ppStmt)
+{
+  char *dummy;
+  int result;
+
+  result = sqlite3_prepare_v2 (dbh,
+                               zSql,
+                               strlen (zSql),
+                               ppStmt,
+                               (const char **) &dummy);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Prepared `%s' / %p: %d\n",
+              zSql,
+              *ppStmt,
+              result);
+  return result;
+}
+
+
+/**
+ * Run actual test queries.
+ *
+ * @return 0 on success
+ */
+static int
+run_queries (sqlite3 *dbh)
+{
+  struct GNUNET_CRYPTO_RsaPublicKey *pub;
+  struct GNUNET_CRYPTO_RsaPublicKey *pub2 = NULL;
+  struct GNUNET_CRYPTO_RsaSignature *sig;
+  struct GNUNET_CRYPTO_RsaSignature *sig2 = NULL;
+  struct GNUNET_TIME_Absolute abs_time = GNUNET_TIME_absolute_get ();
+  struct GNUNET_TIME_Absolute abs_time2;
+  struct GNUNET_TIME_Absolute forever = GNUNET_TIME_UNIT_FOREVER_ABS;
+  struct GNUNET_TIME_Absolute forever2;
+  struct GNUNET_HashCode hc;
+  struct GNUNET_HashCode hc2;
+  sqlite3_stmt *stmt;
+  struct GNUNET_CRYPTO_RsaPrivateKey *priv;
+  const char msg[] = "hello";
+  void *msg2;
+  struct GNUNET_HashCode hmsg;
+  size_t msg2_len;
+  uint16_t u16;
+  uint16_t u162;
+  uint32_t u32;
+  uint32_t u322;
+  uint64_t u64;
+  uint64_t u642;
+
+  priv = GNUNET_CRYPTO_rsa_private_key_create (1024);
+  pub = GNUNET_CRYPTO_rsa_private_key_get_public (priv);
+  memset (&hmsg, 42, sizeof (hmsg));
+  sig = GNUNET_CRYPTO_rsa_sign_fdh (priv,
+                                    &hmsg);
+  u16 = 16;
+  u32 = 32;
+  u64 = 64;
+  /* FIXME: test GNUNET_SQ_result_spec_variable_size */
+
+  sq_prepare (dbh,
+              "INSERT INTO test_sq ("
+              " pub"
+              ",sig"
+              ",abs_time"
+              ",forever"
+              ",hash"
+              ",vsize"
+              ",u16"
+              ",u32"
+              ",u64"
+              ") VALUES "
+              "($1, $2, $3, $4, $5, $6,"
+              "$7, $8, $9);",
+              &stmt);
+  {
+    struct GNUNET_SQ_QueryParam params_insert[] = {
+      GNUNET_SQ_query_param_rsa_public_key (pub),
+      GNUNET_SQ_query_param_rsa_signature (sig),
+      GNUNET_SQ_query_param_absolute_time (&abs_time),
+      GNUNET_SQ_query_param_absolute_time (&forever),
+      GNUNET_SQ_query_param_auto_from_type (&hc),
+      GNUNET_SQ_query_param_fixed_size (msg, strlen (msg)),
+      GNUNET_SQ_query_param_uint16 (&u16),
+      GNUNET_SQ_query_param_uint32 (&u32),
+      GNUNET_SQ_query_param_uint64 (&u64),
+      GNUNET_SQ_query_param_end
+    };
+
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_SQ_bind (stmt,
+                                   params_insert));
+    if (SQLITE_DONE !=
+        sqlite3_step (stmt))
+    {
+      GNUNET_CRYPTO_rsa_signature_free (sig);
+      GNUNET_CRYPTO_rsa_private_key_free (priv);
+      GNUNET_CRYPTO_rsa_public_key_free (pub);
+      return 1;
+    }
+  }
+  sqlite3_finalize (stmt);
+
+  sq_prepare (dbh,
+              "SELECT"
+              " pub"
+              ",sig"
+              ",abs_time"
+              ",forever"
+              ",hash"
+              ",vsize"
+              ",u16"
+              ",u32"
+              ",u64"
+              " FROM test_sq"
+              " ORDER BY abs_time DESC "
+              " LIMIT 1;",
+              &stmt);
+  {
+    struct GNUNET_SQ_QueryParam params_select[] = {
+      GNUNET_SQ_query_param_end
+    };
+    struct GNUNET_SQ_ResultSpec results_select[] = {
+      GNUNET_SQ_result_spec_rsa_public_key (&pub2),
+      GNUNET_SQ_result_spec_rsa_signature (&sig2),
+      GNUNET_SQ_result_spec_absolute_time (&abs_time2),
+      GNUNET_SQ_result_spec_absolute_time (&forever2),
+      GNUNET_SQ_result_spec_auto_from_type (&hc2),
+      GNUNET_SQ_result_spec_variable_size (&msg2, &msg2_len),
+      GNUNET_SQ_result_spec_uint16 (&u162),
+      GNUNET_SQ_result_spec_uint32 (&u322),
+      GNUNET_SQ_result_spec_uint64 (&u642),
+      GNUNET_SQ_result_spec_end
+    };
+
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_SQ_bind (stmt,
+                                   params_select));
+    if (SQLITE_ROW !=
+        sqlite3_step (stmt))
+    {
+      GNUNET_break (0);
+      sqlite3_finalize (stmt);
+      GNUNET_CRYPTO_rsa_signature_free (sig);
+      GNUNET_CRYPTO_rsa_private_key_free (priv);
+      GNUNET_CRYPTO_rsa_public_key_free (pub);
+      return 1;
+    }
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_SQ_extract_result (stmt,
+                                             results_select));
+    GNUNET_break (abs_time.abs_value_us == abs_time2.abs_value_us);
+    GNUNET_break (forever.abs_value_us == forever2.abs_value_us);
+    GNUNET_break (0 ==
+                  memcmp (&hc,
+                          &hc2,
+                          sizeof (struct GNUNET_HashCode)));
+    GNUNET_break (0 ==
+                  GNUNET_CRYPTO_rsa_signature_cmp (sig,
+                                                   sig2));
+    GNUNET_break (0 ==
+                  GNUNET_CRYPTO_rsa_public_key_cmp (pub,
+                                                    pub2));
+    GNUNET_break (strlen (msg) == msg2_len);
+    GNUNET_break (0 ==
+                  strncmp (msg,
+                           msg2,
+                           msg2_len));
+    GNUNET_break (16 == u162);
+    GNUNET_break (32 == u322);
+    GNUNET_break (64 == u642);
+    GNUNET_SQ_cleanup_result (results_select);
+  }
+  sqlite3_finalize (stmt);
+
+  GNUNET_CRYPTO_rsa_signature_free (sig);
+  GNUNET_CRYPTO_rsa_private_key_free (priv);
+  GNUNET_CRYPTO_rsa_public_key_free (pub);
+  return 0;
+}
+
+
+int
+main(int argc,
+     const char *const argv[])
+{
+  sqlite3 *dbh;
+  int ret;
+
+  GNUNET_log_setup ("test-sq",
+                    "WARNING",
+                    NULL);
+  if (SQLITE_OK !=
+      sqlite3_open ("test.db",
+                    &dbh))
+  {
+    fprintf (stderr,
+             "Cannot run test, sqlite3 initialization failed\n");
+    GNUNET_break (0);
+    return 77; /* Signal test was skipped... */
+  }
+
+  if (SQLITE_OK !=
+      sqlite3_exec (dbh,
+                    "CREATE TEMPORARY TABLE IF NOT EXISTS test_sq ("
+                    " pub BYTEA NOT NULL"
+                    ",sig BYTEA NOT NULL"
+                    ",abs_time INT8 NOT NULL"
+                    ",forever INT8 NOT NULL"
+                    ",hash BYTEA NOT NULL"
+                    ",vsize VARCHAR NOT NULL"
+                    ",u16 INT2 NOT NULL"
+                    ",u32 INT4 NOT NULL"
+                    ",u64 INT8 NOT NULL"
+                    ")",
+                    NULL, NULL, NULL))
+  {
+    fprintf (stderr,
+             "Failed to create table\n");
+    GNUNET_break (SQLITE_OK ==
+                  sqlite3_close (dbh));
+    if (0 != unlink ("test.db"))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "unlink",
+                                "test.db");
+    return 1;
+  }
+
+  ret = run_queries (dbh);
+  if (SQLITE_OK !=
+      sqlite3_exec (dbh,
+                    "DROP TABLE test_sq",
+                    NULL, NULL, NULL))
+  {
+    fprintf (stderr,
+             "Failed to drop table\n");
+    ret = 1;
+  }
+  GNUNET_break (SQLITE_OK ==
+                sqlite3_close (dbh));
+  if (0 != unlink ("test.db"))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                              "unlink",
+                              "test.db");
+  return ret;
+}
+
+
+/* end of test_sq.c */
diff --git a/src/statistics/Makefile.am b/src/statistics/Makefile.am
index 0907c4a37..b2e256960 100644
--- a/src/statistics/Makefile.am
+++ b/src/statistics/Makefile.am
@@ -55,7 +55,7 @@ check_PROGRAMS = \
  test_statistics_api_watch_zero_value
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
 endif
 
diff --git a/src/statistics/gnunet-service-statistics.c b/src/statistics/gnunet-service-statistics.c
index 161327421..0cb136b99 100644
--- a/src/statistics/gnunet-service-statistics.c
+++ b/src/statistics/gnunet-service-statistics.c
@@ -359,7 +359,7 @@ transmit (struct ClientEntry *ce,
 
   size = strlen (e->subsystem->service) + 1 +
     strlen (e->name) + 1;
-  GNUNET_assert (size < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (size < GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (m,
                              size,
                              GNUNET_MESSAGE_TYPE_STATISTICS_VALUE);
@@ -776,7 +776,7 @@ check_watch (void *cls,
   size_t size;
   const char *service;
   const char *name;
-  
+
   size = ntohs (message->size) - sizeof (struct GNUNET_MessageHeader);
   if (size !=
       GNUNET_STRINGS_buffer_tokenize ((const char *) &message[1],
@@ -870,7 +870,7 @@ handle_watch (void *cls,
 
 /**
  * Handle DISCONNECT-message.  Sync to disk and send
- * back a #GNUNET_MESSAGE_TYPE_STATISTICS_DISCONNECT_CONFIRM 
+ * back a #GNUNET_MESSAGE_TYPE_STATISTICS_DISCONNECT_CONFIRM
  * message.
  *
  * @param cls the `struct ClientEntry *`
@@ -984,6 +984,7 @@ client_disconnect_cb (void *cls,
       }
     }
   }
+  GNUNET_free (ce);
   if ( (0 == client_count) &&
        (GNUNET_YES == in_shutdown) )
     do_shutdown ();
@@ -992,7 +993,7 @@ client_disconnect_cb (void *cls,
 
 /**
  * We've read a `struct GNUNET_STATISTICS_SetMessage *` from
- * disk. Check that it is well-formed, and if so pass it to 
+ * disk. Check that it is well-formed, and if so pass it to
  * the handler for set messages.
  *
  * @param cls NULL
diff --git a/src/statistics/gnunet-statistics.c b/src/statistics/gnunet-statistics.c
index ed0c3f27d..159afda53 100644
--- a/src/statistics/gnunet-statistics.c
+++ b/src/statistics/gnunet-statistics.c
@@ -378,28 +378,44 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "name", "NAME",
-     gettext_noop ("limit output to statistics for the given NAME"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'p', "persistent", NULL,
-     gettext_noop ("make the value being set persistent"), 0,
-     &GNUNET_GETOPT_set_one, &persistent},
-    {'s', "subsystem", "SUBSYSTEM",
-     gettext_noop ("limit output to the given SUBSYSTEM"), 1,
-     &GNUNET_GETOPT_set_string, &subsystem},
-    {'q', "quiet", NULL,
-     gettext_noop ("just print the statistics value"), 0,
-     &GNUNET_GETOPT_set_one, &quiet},
-    {'w', "watch", NULL,
-     gettext_noop ("watch value continuously"), 0,
-     &GNUNET_GETOPT_set_one, &watch},
-     {'r', "remote", NULL,
-      gettext_noop ("connect to remote host"), 1,
-      &GNUNET_GETOPT_set_string, &remote_host},
-    {'o', "port", NULL,
-     gettext_noop ("port for remote host"), 1,
-     &GNUNET_GETOPT_set_uint, &remote_port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_string ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("limit output to statistics for the given NAME"),
+                                 &name),
+
+    GNUNET_GETOPT_option_flag ('p',
+                                  "persistent",
+                                  gettext_noop ("make the value being set persistent"),
+                                  &persistent),
+
+    GNUNET_GETOPT_option_string ('s',
+                                 "subsystem",
+                                 "SUBSYSTEM",
+                                 gettext_noop ("limit output to the given SUBSYSTEM"),
+                                 &subsystem),
+
+    GNUNET_GETOPT_option_flag ('q',
+                                  "quiet",
+                                  gettext_noop ("just print the statistics value"),
+                                  &quiet),
+
+    GNUNET_GETOPT_option_flag ('w',
+                                  "watch",
+                                  gettext_noop ("watch value continuously"),
+                                  &watch),
+
+    GNUNET_GETOPT_option_string ('r',
+                                 "remote",
+                                 "REMOTE",
+                                 gettext_noop ("connect to remote host"),
+                                 &remote_host),
+    GNUNET_GETOPT_option_ulong ('o',
+                                    "port",
+                                    "PORT",
+                                    gettext_noop ("port for remote host"),
+                                    &remote_port),
     GNUNET_GETOPT_OPTION_END
   };
   remote_port = 0;
diff --git a/src/statistics/statistics_api.c b/src/statistics/statistics_api.c
index ad4453b2a..9d04e854f 100644
--- a/src/statistics/statistics_api.c
+++ b/src/statistics/statistics_api.c
@@ -349,7 +349,7 @@ schedule_watch_request (struct GNUNET_STATISTICS_Handle *h,
   slen = strlen (watch->subsystem) + 1;
   nlen = strlen (watch->name) + 1;
   nsize = sizeof (struct GNUNET_MessageHeader) + slen + nlen;
-  if (nsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (nsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1098,7 +1098,7 @@ GNUNET_STATISTICS_get (struct GNUNET_STATISTICS_Handle *handle,
   slen1 = strlen (subsystem) + 1;
   slen2 = strlen (name) + 1;
   GNUNET_assert (slen1 + slen2 + sizeof (struct GNUNET_MessageHeader) <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   ai = GNUNET_new (struct GNUNET_STATISTICS_GetHandle);
   ai->sh = handle;
   ai->subsystem = GNUNET_strdup (subsystem);
@@ -1246,7 +1246,7 @@ add_setter_action (struct GNUNET_STATISTICS_Handle *h,
   slen = strlen (h->subsystem) + 1;
   nlen = strlen (name) + 1;
   nsize = sizeof (struct GNUNET_STATISTICS_SetMessage) + slen + nlen;
-  if (nsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (nsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/template/Makefile.am b/src/template/Makefile.am
index bdd5d6ebf..5a27a6d60 100644
--- a/src/template/Makefile.am
+++ b/src/template/Makefile.am
@@ -42,7 +42,7 @@ check_PROGRAMS = \
  test_template_api
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/testbed-logger/Makefile.am b/src/testbed-logger/Makefile.am
index 7f372fd02..a8c4c7b05 100644
--- a/src/testbed-logger/Makefile.am
+++ b/src/testbed-logger/Makefile.am
@@ -41,7 +41,7 @@ check_PROGRAMS = \
   test_testbed_logger_api
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = \
   test_testbed_logger_api
 endif
diff --git a/src/testbed-logger/gnunet-service-testbed-logger.c b/src/testbed-logger/gnunet-service-testbed-logger.c
index 1c250b306..f915e70af 100644
--- a/src/testbed-logger/gnunet-service-testbed-logger.c
+++ b/src/testbed-logger/gnunet-service-testbed-logger.c
@@ -40,42 +40,6 @@
   LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
 
 /**
- * The message queue for sending messages to clients
- */
-struct MessageQueue
-{
-  /**
-   * The message to be sent
-   */
-  struct GNUNET_MessageHeader *msg;
-
-  /**
-   * The client to send the message to
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * next pointer for DLL
-   */
-  struct MessageQueue *next;
-
-  /**
-   * prev pointer for DLL
-   */
-  struct MessageQueue *prev;
-};
-
-/**
- * The message queue head
- */
-static struct MessageQueue *mq_head;
-
-/**
- * The message queue tail
- */
-static struct MessageQueue *mq_tail;
-
-/**
  * Handle for buffered writing.
  */
 struct GNUNET_BIO_WriteHandle *bio;
@@ -92,23 +56,38 @@ static int in_shutdown;
 
 
 /**
- * Message handler for #GNUNET_MESSAGE_TYPE_TESTBED_ADDHOST messages
+ * Check #GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG messages
  *
- * @param cls NULL
- * @param client identification of the client
+ * @param cls client identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK (they are all always OK)
+ */
+static int
+check_log_msg (void *cls,
+               const struct GNUNET_MessageHeader *msg)
+{
+  return GNUNET_OK;
+}
+
+
+/**
+ * Message handler for #GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG messages
+ *
+ * @param cls client identification of the client
  * @param msg the actual message
  */
 static void
 handle_log_msg (void *cls,
-                struct GNUNET_SERVER_Client *client,
                 const struct GNUNET_MessageHeader *msg)
 {
+  struct GNUNET_SERVICE_Client *client = cls;
   uint16_t ms;
 
-  ms = ntohs (msg->size);
-  ms -= sizeof (struct GNUNET_MessageHeader);
-  GNUNET_BIO_write (bio, &msg[1], ms);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  ms = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
+  GNUNET_BIO_write (bio,
+                    &msg[1],
+                    ms);
+  GNUNET_SERVICE_client_continue (client);
 }
 
 
@@ -120,69 +99,55 @@ handle_log_msg (void *cls,
 static void
 shutdown_task (void *cls)
 {
-  struct MessageQueue *mq_entry;
-
   in_shutdown = GNUNET_YES;
   if (0 != nconn)
   {
     /* Delay shutdown if there are active connections */
-    GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+    GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                   NULL);
     return;
   }
-  while (NULL != (mq_entry = mq_head))
-  {
-    GNUNET_free (mq_entry->msg);
-    GNUNET_SERVER_client_drop (mq_entry->client);
-    GNUNET_CONTAINER_DLL_remove (mq_head,
-                                 mq_tail,
-                                 mq_entry);
-    GNUNET_free (mq_entry);
-  }
-  GNUNET_break (GNUNET_OK == GNUNET_BIO_write_close (bio));
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_BIO_write_close (bio));
 }
 
 
 /**
-x * Functions with this signature are called whenever a client
- * is disconnected on the network level.
+ * Callback called when a client connects to the service.
  *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
  */
-static void
-client_disconnected (void *cls,
-                     struct GNUNET_SERVER_Client *client)
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  if (NULL == client)
-  {
-    GNUNET_break (0 == nconn);
-    return;
-  }
-  nconn--;
-  if (GNUNET_YES == in_shutdown)
-    GNUNET_SCHEDULER_shutdown ();
+  /* FIXME: is this really what we want here? */
+  GNUNET_SERVICE_client_persist (c);
+  nconn++;
+  return c;
 }
 
 
 /**
- * Functions with this signature are called whenever a client
- * is connected on the network level.
+ * Callback called when a client disconnected from the service
  *
- * @param cls closure
- * @param client identification of the client
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
 static void
-client_connected (void *cls,
-                  struct GNUNET_SERVER_Client *client)
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
 {
-  if (NULL == client)
-  {
-    GNUNET_break (0 == nconn);
-    return;
-  }
-  GNUNET_SERVER_client_persist_ (client);
-  nconn++;
+  nconn--;
+  if (GNUNET_YES == in_shutdown)
+    GNUNET_SCHEDULER_shutdown ();
+  GNUNET_assert (c == internal_cls);
 }
 
 
@@ -190,18 +155,14 @@ client_connected (void *cls,
  * Testbed setup
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
 logger_run (void *cls,
-            struct GNUNET_SERVER_Handle *server,
-            const struct GNUNET_CONFIGURATION_Handle *cfg)
+            const struct GNUNET_CONFIGURATION_Handle *cfg,
+            struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler message_handlers[] = {
-    {&handle_log_msg, NULL, GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG, 0},
-    {NULL, NULL, 0, 0}
-  };
   char *dir;
   char *fn;
   char *hname;
@@ -223,7 +184,8 @@ logger_run (void *cls,
   pid = getpid ();
   hname_len = GNUNET_OS_get_hostname_max_length ();
   hname = GNUNET_malloc (hname_len);
-  if (0 != gethostname (hname, hname_len))
+  if (0 != gethostname (hname,
+                        hname_len))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
          "Cannot get hostname.  Exiting\n");
@@ -247,24 +209,27 @@ logger_run (void *cls,
     return;
   }
   GNUNET_free (fn);
-  GNUNET_SERVER_add_handlers (server, message_handlers);
-  GNUNET_SERVER_connect_notify (server, &client_connected, NULL);
-  GNUNET_SERVER_disconnect_notify (server, &client_disconnected, NULL);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
   LOG_DEBUG ("TESTBED-LOGGER startup complete\n");
 }
 
 
 /**
- * The starting point of execution
+ * Define "main" method using service macro.
  */
-int
-main (int argc, char *const *argv)
-{
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "testbed-logger",
-                              GNUNET_SERVICE_OPTION_NONE,
-                              &logger_run, NULL)) ? 0 : 1;
-}
+GNUNET_SERVICE_MAIN
+("testbed-logger",
+ GNUNET_SERVICE_OPTION_NONE,
+ &logger_run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (log_msg,
+                        GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG,
+                        struct GNUNET_MessageHeader,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
 
 /* end of gnunet-service-testbed-logger.c */
diff --git a/src/testbed-logger/test_testbed_logger_api.c b/src/testbed-logger/test_testbed_logger_api.c
index 0ebe0c3f4..e627feeb4 100644
--- a/src/testbed-logger/test_testbed_logger_api.c
+++ b/src/testbed-logger/test_testbed_logger_api.c
@@ -258,11 +258,15 @@ main (int argc, char **argv)
   GNUNET_log_setup ("test-testbed-logger-api",
                     "WARNING",
                     NULL);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_directory_remove ("/tmp/test-testbed"));
   ret = GNUNET_TESTING_service_run ("test-testbed-logger",
                                     "testbed-logger",
                                     "test_testbed_logger_api.conf",
                                     &test_main,
                                     NULL);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_directory_remove ("/tmp/test-testbed"));
   if (0 != ret)
     return 1;
   if (GNUNET_OK != result)
diff --git a/src/testbed-logger/testbed_logger_api.c b/src/testbed-logger/testbed_logger_api.c
index 25494aed0..7aa4ade35 100644
--- a/src/testbed-logger/testbed_logger_api.c
+++ b/src/testbed-logger/testbed_logger_api.c
@@ -39,7 +39,7 @@
 /**
  * The size of the buffer we fill before sending out the message
  */
-#define BUFFER_SIZE (GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct GNUNET_MessageHeader))
+#define BUFFER_SIZE (GNUNET_MAX_MESSAGE_SIZE - sizeof (struct GNUNET_MessageHeader))
 
 /**
  * Connection handle for the logger service
diff --git a/src/testbed/Makefile.am b/src/testbed/Makefile.am
index e9b667b6c..61cfba2ff 100644
--- a/src/testbed/Makefile.am
+++ b/src/testbed/Makefile.am
@@ -104,13 +104,13 @@ libgnunettestbed_la_SOURCES = \
   testbed_api_test.c \
   testbed_api_topology.c testbed_api_topology.h \
   testbed_api_sd.c testbed_api_sd.h \
-  testbed_api_barriers.c 
+  testbed_api_barriers.c
 libgnunettestbed_la_LIBADD = $(XLIB) \
  $(top_builddir)/src/core/libgnunetcore.la \
  $(top_builddir)/src/statistics/libgnunetstatistics.la \
  $(top_builddir)/src/transport/libgnunettransport.la \
  $(top_builddir)/src/hello/libgnunethello.la \
- -lm \
+ -lm $(Z_LIBS) \
  $(top_builddir)/src/util/libgnunetutil.la \
  $(top_builddir)/src/testing/libgnunettesting.la \
  $(LTLIBINTL)
@@ -157,7 +157,7 @@ check_PROGRAMS = \
   $(underlay_testcases)
 
 if ENABLE_TEST_RUN
- AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
  TESTS = \
   test_testbed_api \
   test_testbed_api_sd \
diff --git a/src/testbed/generate-underlay-topology.c b/src/testbed/generate-underlay-topology.c
index 36580a2a3..2e78c60be 100644
--- a/src/testbed/generate-underlay-topology.c
+++ b/src/testbed/generate-underlay-topology.c
@@ -70,7 +70,7 @@ enum GNUNET_TESTBED_TopologyOption topology;
 /**
  * The number of peers to include in the topology
  */
-static int num_peers;
+static unsigned int num_peers;
 
 /**
  * program result
@@ -335,11 +335,15 @@ int
 main (int argc, char *const argv[])
 {
   struct GNUNET_GETOPT_CommandLineOption option[] = {
-    {'p', "num-peers", "COUNT",
-     gettext_noop ("create COUNT number of peers"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers},
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT number of peers"),
+                                   &num_peers),
     GNUNET_GETOPT_OPTION_END
   };
+
   int ret;
 
   exit_result = GNUNET_SYSERR;
diff --git a/src/testbed/gnunet-helper-testbed.c b/src/testbed/gnunet-helper-testbed.c
index 6368fb74b..d2a3a98b7 100644
--- a/src/testbed/gnunet-helper-testbed.c
+++ b/src/testbed/gnunet-helper-testbed.c
@@ -495,7 +495,7 @@ error:
 static void
 read_task (void *cls)
 {
-  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
+  char buf[GNUNET_MAX_MESSAGE_SIZE];
   ssize_t sread;
 
   read_task_id = NULL;
diff --git a/src/testbed/gnunet-service-testbed_barriers.c b/src/testbed/gnunet-service-testbed_barriers.c
index a201d22bb..681ed6df2 100644
--- a/src/testbed/gnunet-service-testbed_barriers.c
+++ b/src/testbed/gnunet-service-testbed_barriers.c
@@ -478,7 +478,7 @@ GST_barriers_init (struct GNUNET_CONFIGURATION_Handle *cfg)
   LOG_DEBUG ("Launching testbed-barrier service\n");
   barrier_map = GNUNET_CONTAINER_multihashmap_create (3,
                                                       GNUNET_YES);
-  ctx = GNUNET_SERVICE_starT ("testbed-barrier",
+  ctx = GNUNET_SERVICE_start ("testbed-barrier",
                               cfg,
                               &connect_cb,
                               &disconnect_cb,
@@ -524,7 +524,7 @@ GST_barriers_destroy ()
                                                         NULL));
   GNUNET_CONTAINER_multihashmap_destroy (barrier_map);
   GNUNET_assert (NULL != ctx);
-  GNUNET_SERVICE_stoP (ctx);
+  GNUNET_SERVICE_stop (ctx);
 }
 
 
diff --git a/src/testbed/gnunet-service-testbed_oc.c b/src/testbed/gnunet-service-testbed_oc.c
index b775f31bd..09849797c 100644
--- a/src/testbed/gnunet-service-testbed_oc.c
+++ b/src/testbed/gnunet-service-testbed_oc.c
@@ -371,7 +371,7 @@ GST_cleanup_focc (struct ForwardedOverlayConnectContext *focc)
 /**
  * Timeout task for cancelling a forwarded overlay connect connect
  *
- * @param cls the ForwardedOverlayConnectContext
+ * @param cls the `struct ForwardedOperationContext`
  */
 static void
 forwarded_overlay_connect_timeout (void *cls)
diff --git a/src/testbed/gnunet-service-testbed_peers.c b/src/testbed/gnunet-service-testbed_peers.c
index a977b2b9b..aacd62583 100644
--- a/src/testbed/gnunet-service-testbed_peers.c
+++ b/src/testbed/gnunet-service-testbed_peers.c
@@ -1137,14 +1137,8 @@ arm_req_string (enum GNUNET_ARM_RequestStatus rs)
   {
   case GNUNET_ARM_REQUEST_SENT_OK:
     return _("Message was sent successfully");
-  case GNUNET_ARM_REQUEST_CONFIGURATION_ERROR:
-    return _("Misconfiguration (can't connect to the ARM service)");
   case GNUNET_ARM_REQUEST_DISCONNECTED:
     return _("We disconnected from ARM before we could send a request");
-  case GNUNET_ARM_REQUEST_BUSY:
-    return _("ARM API is busy");
-  case GNUNET_ARM_REQUEST_TIMEOUT:
-    return _("Request timed out");
   }
   return _("Unknown request status");
 }
diff --git a/src/testbed/gnunet-testbed-profiler.c b/src/testbed/gnunet-testbed-profiler.c
index 9468b3c91..9829bbf0d 100644
--- a/src/testbed/gnunet-testbed-profiler.c
+++ b/src/testbed/gnunet-testbed-profiler.c
@@ -276,23 +276,34 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "num-peers", "COUNT",
-     gettext_noop ("create COUNT number of peers"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'e', "num-errors", "COUNT",
-     gettext_noop ("tolerate COUNT number of continious timeout failures"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_cont_fails},
-    {'n', "non-interactive", NULL,
-     gettext_noop ("run profiler in non-interactive mode where upon "
-                   "testbed setup the profiler does not wait for a "
-                   "keystroke but continues to run until a termination "
-                   "signal is received"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &noninteractive},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_uint ('p',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT number of peers"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_option_uint ('e',
+                                   "num-errors",
+                                   "COUNT",
+                                   gettext_noop ("tolerate COUNT number of continious timeout failures"),
+                                   &num_cont_fails),
+
+    GNUNET_GETOPT_option_flag ('n',
+                                  "non-interactive",
+                                  gettext_noop ("run profiler in non-interactive mode where upon "
+                                                "testbed setup the profiler does not wait for a "
+                                                "keystroke but continues to run until a termination "
+                                                "signal is received"),
+                                  &noninteractive),
+
 #if !ENABLE_SUPERMUC
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &hosts_file},
+    GNUNET_GETOPT_option_string ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
 #endif
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/testbed/test_gnunet_helper_testbed.c b/src/testbed/test_gnunet_helper_testbed.c
index 7a2dce8a5..7ba9e7c31 100644
--- a/src/testbed/test_gnunet_helper_testbed.c
+++ b/src/testbed/test_gnunet_helper_testbed.c
@@ -121,10 +121,12 @@ do_abort (void *cls)
  *               #GNUNET_SYSERR during GNUNET_HELPER_stop()
  */
 static void
-cont_cb (void *cls, int result)
+cont_cb (void *cls,
+         int result)
 {
   shandle = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Message sent\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message sent\n");
   GNUNET_assert (GNUNET_OK == result);
 }
 
@@ -138,11 +140,11 @@ cont_cb (void *cls, int result)
  * @param cls closure
  * @param client identification of the client
  * @param message the actual message
- *
  * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
  */
 static int
-mst_cb (void *cls, void *client, const struct GNUNET_MessageHeader *message)
+mst_cb (void *cls,
+        const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TESTBED_HelperReply *msg;
   char *config;
@@ -207,8 +209,12 @@ run (void *cls, char *const *args, const char *cfgfile,
   const char *trusted_ip = "127.0.0.1";
 
   helper =
-      GNUNET_HELPER_start (GNUNET_YES, "gnunet-helper-testbed", binary_argv,
-                           &mst_cb, &exp_cb, NULL);
+      GNUNET_HELPER_start (GNUNET_YES,
+                           "gnunet-helper-testbed",
+                           binary_argv,
+                           &mst_cb,
+                           &exp_cb,
+                           NULL);
   GNUNET_assert (NULL != helper);
   cfg = GNUNET_CONFIGURATION_dup (cfg2);
   msg = GNUNET_TESTBED_create_helper_init_msg_ (trusted_ip, NULL, cfg);
diff --git a/src/testbed/testbed_api_hosts.c b/src/testbed/testbed_api_hosts.c
index 731944bc4..5d2c1cc37 100644
--- a/src/testbed/testbed_api_hosts.c
+++ b/src/testbed/testbed_api_hosts.c
@@ -952,10 +952,11 @@ gen_rsh_suffix_args (const char * const *append_args)
  * @param client identification of the client
  * @param message the actual message
  *
- * @return GNUNET_OK on success, GNUNET_SYSERR to stop further processing
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
  */
 static int
-helper_mst (void *cls, void *client, const struct GNUNET_MessageHeader *message)
+helper_mst (void *cls,
+            const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_TESTBED_ControllerProc *cp = cls;
   const struct GNUNET_TESTBED_HelperReply *msg;
diff --git a/src/testbed/testbed_api_peers.c b/src/testbed/testbed_api_peers.c
index 2af62b44a..871e554a9 100644
--- a/src/testbed/testbed_api_peers.c
+++ b/src/testbed/testbed_api_peers.c
@@ -969,7 +969,7 @@ GNUNET_TESTBED_peer_manage_service (void *op_cls,
   GNUNET_assert (TESTBED_PS_STARTED == peer->state); /* peer is not running? */
   msize = strlen (service_name) + 1;
   msize += sizeof (struct GNUNET_TESTBED_ManagePeerServiceMessage);
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < msize)
+  if (GNUNET_MAX_MESSAGE_SIZE < msize)
     return NULL;
   data = GNUNET_new (struct ManageServiceData);
   data->cb = cb;
diff --git a/src/testbed/testbed_api_test.c b/src/testbed/testbed_api_test.c
index e64e9fe2d..559c4abd9 100644
--- a/src/testbed/testbed_api_test.c
+++ b/src/testbed/testbed_api_test.c
@@ -114,13 +114,16 @@ run (void *cls, char *const *args, const char *cfgfile,
  *        handle of each peer
  * @param cc_cls closure for cc
  * @param test_master task to run once the test is ready
- * @param test_master_cls closure for 'task'.
- * @return GNUNET_SYSERR on error, GNUNET_OK on success
+ * @param test_master_cls closure for @a test_master
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
  */
 int
-GNUNET_TESTBED_test_run (const char *testname, const char *cfg_filename,
-                         unsigned int num_peers, uint64_t event_mask,
-                         GNUNET_TESTBED_ControllerCallback cc, void *cc_cls,
+GNUNET_TESTBED_test_run (const char *testname,
+                         const char *cfg_filename,
+                         unsigned int num_peers,
+                         uint64_t event_mask,
+                         GNUNET_TESTBED_ControllerCallback cc,
+                         void *cc_cls,
                          GNUNET_TESTBED_TestMaster test_master,
                          void *test_master_cls)
 {
@@ -148,9 +151,8 @@ GNUNET_TESTBED_test_run (const char *testname, const char *cfg_filename,
   rc->event_mask = event_mask;
   rc->cc = cc;
   rc->cc_cls = cc_cls;
-  ret =
-      GNUNET_PROGRAM_run ((sizeof (argv2) / sizeof (char *)) - 1, argv2,
-                          testname, "nohelp", options, &run, rc);
+  ret = GNUNET_PROGRAM_run ((sizeof (argv2) / sizeof (char *)) - 1, argv2,
+                            testname, "nohelp", options, &run, rc);
   GNUNET_free (rc);
   GNUNET_free (argv2[0]);
   GNUNET_free (argv2[2]);
diff --git a/src/testbed/testbed_api_topology.c b/src/testbed/testbed_api_topology.c
index a21a7cf53..7bc36d1b4 100644
--- a/src/testbed/testbed_api_topology.c
+++ b/src/testbed/testbed_api_topology.c
@@ -1147,9 +1147,11 @@ gen_topo_from_file (struct TopologyContext *tc,
              other_peer_id);
       while (('\n' != data[offset]) && ('|' != data[offset]) && (offset < fs))
         offset++;
-      if ('\n' == data[offset])
+      if ( (offset < fs) &&
+           ('\n' == data[offset]) )
         state = PEER_INDEX;
-      else if ('|' == data[offset])
+      else if ( (offset < fs) &&
+                ('|' == data[offset]) )
       {
         state = OTHER_PEER_INDEX;
         offset++;
@@ -1491,7 +1493,7 @@ GNUNET_TESTBED_topology_get_ (enum GNUNET_TESTBED_TopologyOption *topology,
     {
       if (NULL != topology)
         *topology = (enum GNUNET_TESTBED_TopologyOption) cnt;
-      GNUNET_assert (GNUNET_TESTBED_TOPOLOGY_OPTION_END != *topology);
+      GNUNET_assert (GNUNET_TESTBED_TOPOLOGY_OPTION_END != (enum GNUNET_TESTBED_TopologyOption) cnt);
       return GNUNET_YES;
     }
   }
diff --git a/src/testing/Makefile.am b/src/testing/Makefile.am
index c290dc909..727bfb134 100644
--- a/src/testing/Makefile.am
+++ b/src/testing/Makefile.am
@@ -56,7 +56,7 @@ check_PROGRAMS = \
  test_testing_sharedservices
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
  test_testing_portreservation \
  test_testing_peerstartup \
diff --git a/src/testing/gnunet-testing.c b/src/testing/gnunet-testing.c
index 07f1560cb..3dbe65883 100644
--- a/src/testing/gnunet-testing.c
+++ b/src/testing/gnunet-testing.c
@@ -348,17 +348,35 @@ run_no_scheduler (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "cfg", NULL, gettext_noop ("create unique configuration files"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &create_cfg},
-    {'k', "key", "FILENAME", gettext_noop ("extract hostkey file from pre-computed hostkey list"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &create_hostkey},
-    {'n', "number", "NUMBER", gettext_noop ("number of unique configuration files to create, or number of the hostkey to extract"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &create_no},
-    {'t', "template", "FILENAME", gettext_noop ("configuration template"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &create_cfg_template},
-    {'r', "run", "SERVICE", gettext_noop ("run the given service, wait on stdin for 'r' (restart) or 'q' (quit)"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &run_service_name},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('C',
+                                  "cfg",
+                                  gettext_noop ("create unique configuration files"),
+                                  &create_cfg),
+    GNUNET_GETOPT_option_string ('k',
+                                 "key",
+                                 "FILENAME",
+                                 gettext_noop ("extract hostkey file from pre-computed hostkey list"),
+                                 &create_hostkey),
+
+    GNUNET_GETOPT_option_uint ('n',
+                                   "number",
+                                   "NUMBER",
+                                   gettext_noop ("number of unique configuration files to create, or number of the hostkey to extract"),
+                                   &create_no),
+
+
+    GNUNET_GETOPT_option_string ('t',
+                                 "template",
+                                 "FILENAME",
+                                 gettext_noop ("configuration template"),
+                                 &create_cfg_template),
+
+    GNUNET_GETOPT_option_string ('r',
+                                 "run",
+                                 "SERVICE",
+                                 gettext_noop ("run the given service, wait on stdin for 'r' (restart) or 'q' (quit)"),
+                                 &run_service_name),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/testing/list-keys.c b/src/testing/list-keys.c
index 1fd46ebf2..9210486d3 100644
--- a/src/testing/list-keys.c
+++ b/src/testing/list-keys.c
@@ -6,10 +6,6 @@ static unsigned int nkeys;
 static unsigned int nskip;
 static int result;
 
-
-
-
-
 /**
  * Main run function.
  *
@@ -87,12 +83,11 @@ run (void *cls, char *const *args, const char *cfgfile,
 int main (int argc, char *argv[])
 {
   struct GNUNET_GETOPT_CommandLineOption option[] = {
-    {'n', "num-keys", "COUNT",
-     gettext_noop ("list COUNT number of keys"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &nkeys},
-    {'s', "skip", "COUNT",
-     gettext_noop ("skip COUNT number of keys in the beginning"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &nskip},
+    GNUNET_GETOPT_option_uint ('n',
+                                   "num-keys",
+                                   "COUNT",
+                                   gettext_noop ("list COUNT number of keys"),
+                                   &nkeys),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/testing/testing.c b/src/testing/testing.c
index 043bdd7d2..ba7bf827e 100644
--- a/src/testing/testing.c
+++ b/src/testing/testing.c
@@ -907,6 +907,7 @@ update_config_sections (void *cls,
       {
         ikeys[key] = ptr;
         ptr = strstr (ptr, ";");
+        GNUNET_assert (NULL != ptr); /* worked just before... */
         *ptr = '\0';
         ptr++;
       }
@@ -1633,7 +1634,9 @@ GNUNET_TESTING_service_run (const char *testdir,
   char *binary;
   char *libexec_binary;
 
-  GNUNET_log_setup (testdir, "WARNING", NULL);
+  GNUNET_log_setup (testdir,
+                    "WARNING",
+                    NULL);
   system = GNUNET_TESTING_system_create (testdir, "127.0.0.1", NULL, NULL);
   if (NULL == system)
     return 1;
diff --git a/src/topology/Makefile.am b/src/topology/Makefile.am
index 6c6cbf8ff..97e4652d0 100644
--- a/src/topology/Makefile.am
+++ b/src/topology/Makefile.am
@@ -48,7 +48,7 @@ check_PROGRAMS = \
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/topology/friends.c b/src/topology/friends.c
index 59c70e4b0..a960fad17 100644
--- a/src/topology/friends.c
+++ b/src/topology/friends.c
@@ -58,14 +58,20 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg,
     return GNUNET_SYSERR;
   }
   if ( (GNUNET_OK != GNUNET_DISK_file_test (fn)) &&
-       (GNUNET_OK != GNUNET_DISK_fn_write (fn, NULL, 0,
+       (GNUNET_OK != GNUNET_DISK_fn_write (fn,
+                                           NULL,
+                                           0,
                                            GNUNET_DISK_PERM_USER_READ |
-                                           GNUNET_DISK_PERM_USER_WRITE)) )
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "write", fn);
+                                           GNUNET_DISK_PERM_USER_WRITE |
+                                           GNUNET_DISK_OPEN_CREATE)) )
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                              "write",
+                              fn);
   if ( (GNUNET_OK !=
         GNUNET_DISK_file_size (fn,
                                &fsize,
-                               GNUNET_NO, GNUNET_YES)) ||
+                               GNUNET_NO,
+                               GNUNET_YES)) ||
        (0 == fsize) )
   {
     GNUNET_free (fn);
@@ -93,8 +99,8 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg,
       pos++;
     if (GNUNET_OK !=
         GNUNET_CRYPTO_eddsa_public_key_from_string (&data[start],
-                                                       pos - start,
-                                                       &pid.public_key))
+                                                    pos - start,
+                                                    &pid.public_key))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   _("Syntax error in FRIENDS file at offset %llu, skipping bytes `%.*s'.\n"),
diff --git a/src/topology/gnunet-daemon-topology.c b/src/topology/gnunet-daemon-topology.c
index 067ebce23..537ffe059 100644
--- a/src/topology/gnunet-daemon-topology.c
+++ b/src/topology/gnunet-daemon-topology.c
@@ -499,7 +499,7 @@ schedule_next_hello (void *cls)
   /* find applicable HELLOs */
   fah.peer = pl;
   fah.result = NULL;
-  fah.max_size = GNUNET_SERVER_MAX_MESSAGE_SIZE - 1;
+  fah.max_size = GNUNET_MAX_MESSAGE_SIZE - 1;
   fah.next_adv = GNUNET_TIME_UNIT_FOREVER_REL;
   GNUNET_CONTAINER_multipeermap_iterate (peers,
                                          &find_advertisable_hello,
diff --git a/src/transport/Makefile.am b/src/transport/Makefile.am
index 0b523eecc..7687f2348 100644
--- a/src/transport/Makefile.am
+++ b/src/transport/Makefile.am
@@ -191,6 +191,8 @@ libexec_PROGRAMS = \
  $(BT_BIN) \
  gnunet-service-transport
 
+
+
 bin_PROGRAMS = \
  gnunet-transport \
  gnunet-transport-certificate-creation
@@ -500,7 +502,7 @@ endif
 endif
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
  test_transport_address_switch_tcp \
  test_transport_address_switch_udp \
@@ -561,7 +563,7 @@ TESTS = \
  $(HTTP_API_TIMEOUT_TEST) \
  $(HTTPS_API_TIMEOUT_TEST) \
  $(WLAN_TIMEOUT_TEST) \
- $(BT_TIMEOUT_TEST) 
+ $(BT_TIMEOUT_TEST)
 if HAVE_GETOPT_BINARY
 TESTS += \
 test_transport_api_slow_ats
diff --git a/src/transport/gnunet-helper-transport-wlan-dummy.c b/src/transport/gnunet-helper-transport-wlan-dummy.c
index 684546314..63ed9c4b7 100644
--- a/src/transport/gnunet-helper-transport-wlan-dummy.c
+++ b/src/transport/gnunet-helper-transport-wlan-dummy.c
@@ -120,11 +120,11 @@ send_mac_to_plugin (char *buffer, struct GNUNET_TRANSPORT_WLAN_MacAddress *mac)
  * type to the output forward and copy it to the buffer for stdout.
  *
  * @param cls the 'struct SendBuffer' to copy the converted message to
- * @param client unused
  * @param hdr inbound message from the FIFO
  */
 static int
-stdin_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
+stdin_send (void *cls,
+            const struct GNUNET_MessageHeader *hdr)
 {
   struct SendBuffer *write_pout = cls;
   const struct GNUNET_TRANSPORT_WLAN_RadiotapSendMessage *in;
@@ -166,11 +166,11 @@ stdin_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
  * We read a full message from stdin.  Copy it to our send buffer.
  *
  * @param cls the 'struct SendBuffer' to copy to
- * @param client unused
  * @param hdr the message we received to copy to the buffer
  */
 static int
-file_in_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
+file_in_send (void *cls,
+              const struct GNUNET_MessageHeader *hdr)
 {
   struct SendBuffer *write_std = cls;
   uint16_t sendsize;
@@ -213,8 +213,8 @@ main (int argc, char *argv[])
   fd_set wfds;
   struct timeval tv;
   int retval;
-  struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst = NULL;
-  struct GNUNET_SERVER_MessageStreamTokenizer *file_in_mst = NULL;
+  struct GNUNET_MessageStreamTokenizer *stdin_mst = NULL;
+  struct GNUNET_MessageStreamTokenizer *file_in_mst = NULL;
   struct GNUNET_TRANSPORT_WLAN_MacAddress macaddr;
   int first;
 
@@ -340,8 +340,8 @@ main (int argc, char *argv[])
   write_std.pos = 0;
   write_pout.size = 0;
   write_pout.pos = 0;
-  stdin_mst = GNUNET_SERVER_mst_create (&stdin_send, &write_pout);
-  file_in_mst = GNUNET_SERVER_mst_create (&file_in_send, &write_std);
+  stdin_mst = GNUNET_MST_create (&stdin_send, &write_pout);
+  file_in_mst = GNUNET_MST_create (&file_in_send, &write_std);
 
   /* Send 'random' mac address */
   macaddr.mac[0] = 0x13;
@@ -453,8 +453,9 @@ main (int argc, char *argv[])
       }
       else if (0 < readsize)
       {
-        GNUNET_SERVER_mst_receive (stdin_mst, NULL, readbuf, readsize,
-                                   GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (stdin_mst,
+                                readbuf, readsize,
+                                GNUNET_NO, GNUNET_NO);
 
       }
       else
@@ -475,8 +476,9 @@ main (int argc, char *argv[])
       }
       else if (0 < readsize)
       {
-        GNUNET_SERVER_mst_receive (file_in_mst, NULL, readbuf, readsize,
-                                   GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (file_in_mst,
+                                readbuf, readsize,
+                                GNUNET_NO, GNUNET_NO);
       }
       else
       {
@@ -489,9 +491,9 @@ main (int argc, char *argv[])
 end:
   /* clean up */
   if (NULL != stdin_mst)
-    GNUNET_SERVER_mst_destroy (stdin_mst);
+    GNUNET_MST_destroy (stdin_mst);
   if (NULL != file_in_mst)
-    GNUNET_SERVER_mst_destroy (file_in_mst);
+    GNUNET_MST_destroy (file_in_mst);
 
   if (NULL != fpout)
     fclose (fpout);
diff --git a/src/transport/gnunet-service-transport.c b/src/transport/gnunet-service-transport.c
index e7b221344..ec4d82164 100644
--- a/src/transport/gnunet-service-transport.c
+++ b/src/transport/gnunet-service-transport.c
@@ -150,11 +150,11 @@ struct TransportClient
 
   /**
    * What type of client is this?
-   */ 
+   */
   enum ClientType type;
 
   union {
-  
+
     /**
      * Peer identity to monitor the addresses of.
      * Zero to monitor all neighbours.  Valid if
@@ -172,21 +172,21 @@ struct TransportClient
        * if we're performing one that has been cancelled).
        */
       struct GST_BlacklistCheck *bc;
-      
+
       /**
        * Set to #GNUNET_YES if we're currently waiting for a reply.
        */
       int waiting_for_reply;
-      
+
       /**
        * #GNUNET_YES if we have to call receive_done for this client
        */
       int call_receive_done;
-            
+
     } blacklist;
-    
+
   } details;
-  
+
 };
 
 
@@ -272,7 +272,7 @@ struct AddressToStringContext
  */
 struct SendTransmitContinuationContext
 {
-  
+
   /**
    * Client that made the request.
    */
@@ -563,7 +563,7 @@ client_disconnect_cb (void *cls,
                                              bc);
     }
     break;
-  }                
+  }
   GNUNET_free (tc);
 }
 
@@ -793,7 +793,7 @@ handle_client_send (void *cls,
                     const struct OutboundMessage *obm)
 {
   static unsigned long long uuid_gen;
-  struct TransportClient *tc = cls;  
+  struct TransportClient *tc = cls;
   const struct GNUNET_MessageHeader *obmm;
   struct SendTransmitContinuationContext *stcc;
 
@@ -937,7 +937,7 @@ check_client_address_to_string (void *cls,
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
-}    
+}
 
 
 /**
@@ -950,7 +950,7 @@ static void
 handle_client_address_to_string (void *cls,
                                  const struct AddressLookupMessage *alum)
 {
-  struct TransportClient *tc = cls;  
+  struct TransportClient *tc = cls;
   struct GNUNET_TRANSPORT_PluginFunctions *papi;
   const char *plugin_name;
   const char *address;
@@ -1308,7 +1308,7 @@ handle_client_monitor_plugins (void *cls,
                                const struct GNUNET_MessageHeader *message)
 {
   struct TransportClient *tc = cls;
-  
+
   GNUNET_SERVICE_client_mark_monitor (tc->client);
   GNUNET_SERVICE_client_disable_continue_warning (tc->client);
   GNUNET_notification_context_add (plugin_nc,
@@ -1690,9 +1690,10 @@ GST_receive_callback (void *cls,
     goto end;
   type = ntohs (message->type);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received message with type %u from peer `%s'\n",
+              "Received message with type %u from peer `%s' at %s\n",
               type,
-              GNUNET_i2s (&address->peer));
+              GNUNET_i2s (&address->peer),
+              GST_plugins_a2s (address));
 
   GNUNET_STATISTICS_update (GST_stats,
                             gettext_noop ("# bytes total received"),
@@ -2226,7 +2227,7 @@ handle_client_set_metric (void *cls,
                           const struct TrafficMetricMessage *tm)
 {
   struct TransportClient *tc = cls;
-  
+
   GST_manipulation_set_metric (tm);
   GNUNET_SERVICE_client_continue (tc->client);
 }
@@ -2375,7 +2376,7 @@ handle_client_blacklist_reply (void *cls,
   bc = tc->details.blacklist.bc;
   tc->details.blacklist.bc = NULL;
   tc->details.blacklist.waiting_for_reply = GNUNET_NO;
-  tc->details.blacklist.call_receive_done = GNUNET_YES; 
+  tc->details.blacklist.call_receive_done = GNUNET_YES;
   if (NULL != bc)
   {
     /* only run this if the blacklist check has not been
@@ -2808,7 +2809,7 @@ run (void *cls,
 #if HAVE_GETRLIMIT
   {
     struct rlimit r_file;
-  
+
     if (0 == getrlimit (RLIMIT_NOFILE,
                         &r_file))
     {
@@ -2893,7 +2894,7 @@ GNUNET_SERVICE_MAIN
                         GNUNET_MESSAGE_TYPE_TRANSPORT_SEND,
                         struct OutboundMessage,
                         NULL),
- GNUNET_MQ_hd_var_size (client_address_to_string,                       
+ GNUNET_MQ_hd_var_size (client_address_to_string,
                         GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_TO_STRING,
                         struct AddressLookupMessage,
                         NULL),
@@ -2913,7 +2914,7 @@ GNUNET_SERVICE_MAIN
                           GNUNET_MESSAGE_TYPE_TRANSPORT_TRAFFIC_METRIC,
                           struct TrafficMetricMessage,
                           NULL),
- GNUNET_MQ_hd_fixed_size (client_monitor_plugins, 
+ GNUNET_MQ_hd_fixed_size (client_monitor_plugins,
                           GNUNET_MESSAGE_TYPE_TRANSPORT_MONITOR_PLUGIN_START,
                           struct GNUNET_MessageHeader,
                           NULL),
diff --git a/src/transport/gnunet-service-transport_neighbours.c b/src/transport/gnunet-service-transport_neighbours.c
index 3952a728e..19f5fd081 100644
--- a/src/transport/gnunet-service-transport_neighbours.c
+++ b/src/transport/gnunet-service-transport_neighbours.c
@@ -1150,17 +1150,18 @@ set_incoming_quota (struct NeighbourMapEntry *n,
     sqm.header.size = htons (sizeof (struct GNUNET_ATS_SessionQuotaMessage));
     sqm.header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_SESSION_QUOTA);
     sqm.quota = quota.value__;
-    (void) send_with_session (n,
-                              &sqm,
-                              sizeof (sqm),
-                              UINT32_MAX - 1,
-                              GNUNET_TIME_UNIT_FOREVER_REL,
-                              GNUNET_NO,
-                              NULL, NULL);
+    if (NULL != n->primary_address.session)
+      (void) send_with_session (n,
+                                &sqm,
+                                sizeof (sqm),
+                                UINT32_MAX - 1,
+                                GNUNET_TIME_UNIT_FOREVER_REL,
+                                GNUNET_NO,
+                                NULL, NULL);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Disconnecting peer `%4s' due to SET_QUOTA\n",
+              "Disconnecting peer `%s' due to SET_QUOTA\n",
               GNUNET_i2s (&n->id));
   if (GNUNET_YES == test_connected (n))
     GNUNET_STATISTICS_update (GST_stats,
@@ -2135,13 +2136,18 @@ setup_neighbour (const struct GNUNET_PeerIdentity *peer)
 {
   struct NeighbourMapEntry *n;
 
+  if (0 ==
+      memcmp (&GST_my_identity,
+              peer,
+              sizeof (struct GNUNET_PeerIdentity)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Cowardly refusing to consider myself my neighbour!\n");
+    return NULL;
+  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Creating new neighbour entry for `%s'\n",
               GNUNET_i2s (peer));
-  GNUNET_assert (0 !=
-                 memcmp (&GST_my_identity,
-                         peer,
-                         sizeof (struct GNUNET_PeerIdentity)));
   n = GNUNET_new (struct NeighbourMapEntry);
   n->id = *peer;
   n->ack_state = ACK_UNDEFINED;
@@ -2249,6 +2255,7 @@ GST_neighbours_handle_session_syn (const struct GNUNET_MessageHeader *message,
   {
     /* This is a new neighbour and set to not connected */
     n = setup_neighbour (peer);
+    GNUNET_assert (NULL != n);
   }
 
   /* Remember this SYN message in neighbour */
@@ -2318,6 +2325,7 @@ GST_neighbours_handle_session_syn (const struct GNUNET_MessageHeader *message,
     /* Get rid of remains and re-try */
     free_neighbour (n);
     n = setup_neighbour (peer);
+    GNUNET_assert (NULL != n);
     /* Remember the SYN time stamp for ACK message */
     n->ack_state = ACK_SEND_SYN_ACK;
     n->connect_ack_timestamp = ts;
@@ -2495,6 +2503,12 @@ switch_address_bl_check_cont (void *cls,
   if (NULL == (n = lookup_neighbour (peer)))
   {
     n = setup_neighbour (peer);
+    if (NULL == n)
+    {
+      /* not sure how this can happen... */
+      GNUNET_break (0);
+      goto cleanup;
+    }
     n->state = GNUNET_TRANSPORT_PS_INIT_ATS;
   }
 
diff --git a/src/transport/gnunet-service-transport_validation.c b/src/transport/gnunet-service-transport_validation.c
index 505626b59..4a6d427be 100644
--- a/src/transport/gnunet-service-transport_validation.c
+++ b/src/transport/gnunet-service-transport_validation.c
@@ -599,7 +599,7 @@ transmit_ping_if_allowed (void *cls,
   ping.challenge = htonl (ve->challenge);
   ping.target = *pid;
 
-  if (tsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (tsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     hsize = 0;
diff --git a/src/transport/gnunet-transport-certificate-creation.c b/src/transport/gnunet-transport-certificate-creation.c
index 129df7b35..edaf4773a 100644
--- a/src/transport/gnunet-transport-certificate-creation.c
+++ b/src/transport/gnunet-transport-certificate-creation.c
@@ -44,7 +44,7 @@ make_dev_zero (int fd,
   GNUNET_assert (-1 != z);
   if (z == fd)
     return;
-  dup2 (z, fd);
+  GNUNET_break (fd == dup2 (z, fd));
   GNUNET_assert (0 == close (z));
 }
 #endif
diff --git a/src/transport/gnunet-transport-profiler.c b/src/transport/gnunet-transport-profiler.c
index dceff7e3b..d580fce08 100644
--- a/src/transport/gnunet-transport-profiler.c
+++ b/src/transport/gnunet-transport-profiler.c
@@ -151,7 +151,7 @@ static struct GNUNET_PeerIdentity pid;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -520,7 +520,7 @@ run (void *cls,
   cfg = (struct GNUNET_CONFIGURATION_Handle *) mycfg;
 
   ret = 1;
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE <= benchmark_size)
+  if (GNUNET_MAX_MESSAGE_SIZE <= benchmark_size)
   {
     FPRINTF (stderr,
              "Message size too big!\n");
@@ -610,27 +610,37 @@ main (int argc, char * const *argv)
   benchmark_iterations = DEFAULT_ITERATION_COUNT;
   benchmark_running = GNUNET_NO;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-
-    { 's', "send", NULL,
-      gettext_noop ("send data to peer"),
-      0, &GNUNET_GETOPT_set_one, &benchmark_send},
-    { 'r', "receive", NULL, gettext_noop
-      ("receive data from peer"), 0,
-      &GNUNET_GETOPT_set_one, &benchmark_receive},
-    { 'i', "iterations", NULL, gettext_noop
-      ("iterations"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_iterations},
-    { 'n', "number", NULL, gettext_noop
-      ("number of messages to send"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_count},
-    { 'm', "messagesize", NULL, gettext_noop
-      ("message size to use"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_size},
-    { 'p', "peer", "PEER",
-      gettext_noop ("peer identity"), 1, &GNUNET_GETOPT_set_string,
-      &cpid },
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_option_flag ('s',
+                                  "send",
+                                  gettext_noop ("send data to peer"),
+                                  &benchmark_send),
+    GNUNET_GETOPT_option_flag ('r',
+                                  "receive",
+                                  gettext_noop ("receive data from peer"),
+                                  &benchmark_receive),
+    GNUNET_GETOPT_option_uint ('i',
+                                   "iterations",
+                                   NULL,
+                                   gettext_noop ("iterations"),
+                                   &benchmark_iterations),
+    GNUNET_GETOPT_option_uint ('n',
+                                   "number",
+                                   NULL,
+                                   gettext_noop ("number of messages to send"),
+                                   &benchmark_count),
+    GNUNET_GETOPT_option_uint ('m',
+                                   "messagesize",
+                                   NULL,
+                                   gettext_noop ("message size to use"),
+                                   &benchmark_size),
+    GNUNET_GETOPT_option_string ('p',
+                                 "peer",
+                                 "PEER",
+                                 gettext_noop ("peer identity"),
+                                 &cpid),
+    GNUNET_GETOPT_option_verbose (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/transport/gnunet-transport.c b/src/transport/gnunet-transport.c
index fcfc94ac8..96d0a6a3a 100644
--- a/src/transport/gnunet-transport.c
+++ b/src/transport/gnunet-transport.c
@@ -167,11 +167,6 @@ struct PeerResolutionContext
 #define BLOCKSIZE 4
 
 /**
- * Which peer should we connect to?
- */
-static char *cpid;
-
-/**
  * Handle to transport service.
  */
 static struct GNUNET_TRANSPORT_CoreHandle *handle;
@@ -283,7 +278,7 @@ static struct GNUNET_TRANSPORT_PluginMonitor *pm;
 
 /**
  * Identity of the peer we transmit to / connect to.
- * (equivalent to 'cpid' string).
+ * ('-p' command-line option).
  */
 static struct GNUNET_PeerIdentity pid;
 
@@ -295,7 +290,7 @@ static struct GNUNET_SCHEDULER_Task *op_timeout;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 /**
  * Resolver process handle.
@@ -485,7 +480,7 @@ operation_timeout (void *cls)
                _("Failed to resolve address for peer `%s'\n"),
                GNUNET_i2s (&cur->addrcp->peer));
 
-      GNUNET_CONTAINER_DLL_remove(rc_head, 
+      GNUNET_CONTAINER_DLL_remove(rc_head,
                                   rc_tail,
                                   cur);
       GNUNET_TRANSPORT_address_to_string_cancel (cur->asc);
@@ -729,7 +724,7 @@ print_info (const struct GNUNET_PeerIdentity *id,
              GNUNET_STRINGS_absolute_time_to_string (state_timeout));
   }
   else if ( (GNUNET_YES == iterate_connections) &&
-            (GNUNET_TRANSPORT_is_connected(state)) ) 
+            (GNUNET_TRANSPORT_is_connected(state)) )
   {
     /* Only connected peers, skip state */
     FPRINTF (stdout,
@@ -1089,10 +1084,9 @@ plugin_monitoring_cb (void *cls,
     }
     return; /* shutdown */
   }
-  if ( (NULL != cpid) &&
-       (0 != memcmp (&info->address->peer,
-                     cpid,
-                     sizeof (struct GNUNET_PeerIdentity))) )
+  if (0 != memcmp (&info->address->peer,
+                   &pid,
+                   sizeof (struct GNUNET_PeerIdentity)))
     return; /* filtered */
   if (NULL == addr)
   {
@@ -1241,21 +1235,11 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *mycfg)
 {
+  static struct GNUNET_PeerIdentity zero_pid;
   int counter = 0;
   ret = 1;
 
   cfg = (struct GNUNET_CONFIGURATION_Handle *) mycfg;
-  if ( (NULL != cpid) &&
-       (GNUNET_OK !=
-        GNUNET_CRYPTO_eddsa_public_key_from_string (cpid,
-                                                    strlen (cpid),
-                                                    &pid.public_key)))
-  {
-    FPRINTF (stderr,
-             _("Failed to parse peer identity `%s'\n"),
-             cpid);
-    return;
-  }
 
   counter = benchmark_send + benchmark_receive + iterate_connections
       + monitor_connections + monitor_connects + do_disconnect +
@@ -1290,7 +1274,9 @@ run (void *cls,
 
   if (do_disconnect) /* -D: Disconnect from peer */
   {
-    if (NULL == cpid)
+    if (0 == memcmp (&zero_pid,
+                     &pid,
+                     sizeof (pid)))
     {
       FPRINTF (stderr,
                _("Option `%s' makes no sense without option `%s'.\n"),
@@ -1315,7 +1301,9 @@ run (void *cls,
   }
   else if (benchmark_send) /* -s: Benchmark sending */
   {
-    if (NULL == cpid)
+    if (0 == memcmp (&zero_pid,
+                     &pid,
+                     sizeof (pid)))
     {
       FPRINTF (stderr,
                _("Option `%s' makes no sense without option `%s'.\n"),
@@ -1352,7 +1340,7 @@ run (void *cls,
                              NULL),
       GNUNET_MQ_handler_end ()
     };
-    
+
     handle = GNUNET_TRANSPORT_core_connect (cfg,
                                             NULL,
                                             handlers,
@@ -1378,7 +1366,7 @@ run (void *cls,
   else if (iterate_connections) /* -i: List information about peers once */
   {
     pic = GNUNET_TRANSPORT_monitor_peers (cfg,
-                                          (NULL == cpid) ? NULL : &pid,
+                                          &pid,
                                           GNUNET_YES,
                                           &process_peer_iteration_cb,
                                           (void *) cfg);
@@ -1391,7 +1379,7 @@ run (void *cls,
     monitored_peers = GNUNET_CONTAINER_multipeermap_create (10,
                                                             GNUNET_NO);
     pic = GNUNET_TRANSPORT_monitor_peers (cfg,
-                                          (NULL == cpid) ? NULL : &pid,
+                                          &pid,
                                           GNUNET_NO,
                                           &process_peer_monitoring_cb,
                                           NULL);
@@ -1439,38 +1427,50 @@ main (int argc,
       char * const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'a', "all", NULL,
-      gettext_noop ("print information for all peers (instead of only connected peers)"),
-      0, &GNUNET_GETOPT_set_one, &iterate_all },
-    { 'b', "benchmark", NULL,
-      gettext_noop ("measure how fast we are receiving data from all peers (until CTRL-C)"),
-      0, &GNUNET_GETOPT_set_one, &benchmark_receive },
-    { 'D', "disconnect",
-      NULL, gettext_noop ("disconnect from a peer"), 0,
-      &GNUNET_GETOPT_set_one, &do_disconnect },
-    { 'i', "information", NULL,
-      gettext_noop ("provide information about all current connections (once)"),
-      0, &GNUNET_GETOPT_set_one, &iterate_connections },
-    { 'm', "monitor", NULL,
-      gettext_noop ("provide information about all current connections (continuously)"),
-      0, &GNUNET_GETOPT_set_one, &monitor_connections },
-    { 'e', "events", NULL,
-      gettext_noop ("provide information about all connects and disconnect events (continuously)"),
-      0, &GNUNET_GETOPT_set_one, &monitor_connects },
-    { 'n', "numeric",
-      NULL, gettext_noop ("do not resolve hostnames"), 0,
-      &GNUNET_GETOPT_set_one, &numeric },
-    { 'p', "peer", "PEER",
-      gettext_noop ("peer identity"), 1, &GNUNET_GETOPT_set_string,
-      &cpid },
-    { 'P', "plugins", NULL,
-      gettext_noop ("monitor plugin sessions"), 0, &GNUNET_GETOPT_set_one,
-      &monitor_plugins },
-    { 's', "send", NULL, gettext_noop
-      ("send data for benchmarking to the other peer (until CTRL-C)"), 0,
-      &GNUNET_GETOPT_set_one, &benchmark_send },
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('a',
+                                  "all",
+                                  gettext_noop ("print information for all peers (instead of only connected peers)"),
+                                  &iterate_all),
+    GNUNET_GETOPT_option_flag ('b',
+                                  "benchmark",
+                                  gettext_noop ("measure how fast we are receiving data from all peers (until CTRL-C)"),
+                                  &benchmark_receive),
+    GNUNET_GETOPT_option_flag ('D',
+                                  "disconnect",
+                                  gettext_noop ("disconnect from a peer"),
+                                  &do_disconnect),
+    GNUNET_GETOPT_option_flag ('i',
+                                  "information",
+                                  gettext_noop ("provide information about all current connections (once)"),
+                                  &iterate_connections),
+    GNUNET_GETOPT_option_flag ('m',
+                                  "monitor",
+                                  gettext_noop ("provide information about all current connections (continuously)"),
+                                  &monitor_connections),
+    GNUNET_GETOPT_option_flag ('e',
+                                  "events",
+                                  gettext_noop ("provide information about all connects and disconnect events (continuously)"),
+                                  &monitor_connects),
+    GNUNET_GETOPT_option_flag ('n',
+                                  "numeric",
+                                  gettext_noop ("do not resolve hostnames"),
+                                  &numeric),
+    GNUNET_GETOPT_option_base32_auto ('p',
+                                          "peer",
+                                          "PEER",
+                                          gettext_noop ("peer identity"),
+                                          &pid),
+    GNUNET_GETOPT_option_flag ('P',
+                                  "plugins",
+                                  gettext_noop ("monitor plugin sessions"),
+                                  &monitor_plugins),
+    GNUNET_GETOPT_option_flag ('s',
+                                  "send",
+                                  gettext_noop
+      ("send data for benchmarking to the other peer (until CTRL-C)"),
+                                  &benchmark_send),
+    GNUNET_GETOPT_option_verbose (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/transport/plugin_transport_http_client.c b/src/transport/plugin_transport_http_client.c
index ceed94af8..e20948c5a 100644
--- a/src/transport/plugin_transport_http_client.c
+++ b/src/transport/plugin_transport_http_client.c
@@ -221,7 +221,7 @@ struct GNUNET_ATS_Session
   /**
    * Message stream tokenizer for incoming data
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *msg_tk;
+  struct GNUNET_MessageStreamTokenizer *msg_tk;
 
   /**
    * Session timeout task
@@ -528,7 +528,7 @@ client_delete_session (struct GNUNET_ATS_Session *s)
                           GNUNET_TRANSPORT_SS_DONE);
   if (NULL != s->msg_tk)
   {
-    GNUNET_SERVER_mst_destroy (s->msg_tk);
+    GNUNET_MST_destroy (s->msg_tk);
     s->msg_tk = NULL;
   }
   GNUNET_HELLO_address_free (s->address);
@@ -1158,13 +1158,11 @@ client_wake_up (void *cls)
  * Callback for message stream tokenizer
  *
  * @param cls the session
- * @param client not used
  * @param message the message received
  * @return always #GNUNET_OK
  */
 static int
 client_receive_mst_cb (void *cls,
-                       void *client,
                        const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_ATS_Session *s = cls;
@@ -1274,14 +1272,13 @@ client_receive (void *stream,
     return CURL_WRITEFUNC_PAUSE;
   }
   if (NULL == s->msg_tk)
-    s->msg_tk = GNUNET_SERVER_mst_create (&client_receive_mst_cb,
-                                          s);
-  GNUNET_SERVER_mst_receive (s->msg_tk,
-                             s,
-                             stream,
-                             len,
-                             GNUNET_NO,
-                             GNUNET_NO);
+    s->msg_tk = GNUNET_MST_create (&client_receive_mst_cb,
+                                   s);
+  GNUNET_MST_from_buffer (s->msg_tk,
+                          stream,
+                          len,
+                          GNUNET_NO,
+                          GNUNET_NO);
   return len;
 }
 
@@ -1641,7 +1638,7 @@ client_connect_get (struct GNUNET_ATS_Session *s)
                     CURLOPT_CONNECTTIMEOUT_MS,
                     (long) (HTTP_CLIENT_NOT_VALIDATED_TIMEOUT.rel_value_us / 1000LL));
   curl_easy_setopt (s->get.easyhandle, CURLOPT_BUFFERSIZE,
-                    2 * GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                    2 * GNUNET_MAX_MESSAGE_SIZE);
 #if CURL_TCP_NODELAY
   curl_easy_setopt (ps->recv_endpoint,
                     CURLOPT_TCP_NODELAY,
@@ -1818,7 +1815,7 @@ client_connect_put (struct GNUNET_ATS_Session *s)
                     CURLOPT_CONNECTTIMEOUT_MS,
                     (long) (HTTP_CLIENT_NOT_VALIDATED_TIMEOUT.rel_value_us / 1000LL));
   curl_easy_setopt (s->put.easyhandle, CURLOPT_BUFFERSIZE,
-                    2 * GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                    2 * GNUNET_MAX_MESSAGE_SIZE);
 #if CURL_TCP_NODELAY
   curl_easy_setopt (s->put.easyhandle, CURLOPT_TCP_NODELAY, 1);
 #endif
diff --git a/src/transport/plugin_transport_http_server.c b/src/transport/plugin_transport_http_server.c
index 63c67b81c..ff2d68602 100644
--- a/src/transport/plugin_transport_http_server.c
+++ b/src/transport/plugin_transport_http_server.c
@@ -201,7 +201,7 @@ struct GNUNET_ATS_Session
   /**
    * Message stream tokenizer for incoming data
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *msg_tk;
+  struct GNUNET_MessageStreamTokenizer *msg_tk;
 
   /**
    * Client recv handle
@@ -608,7 +608,7 @@ server_delete_session (struct GNUNET_ATS_Session *s)
   }
   if (NULL != s->msg_tk)
   {
-    GNUNET_SERVER_mst_destroy (s->msg_tk);
+    GNUNET_MST_destroy (s->msg_tk);
     s->msg_tk = NULL;
   }
   GNUNET_HELLO_address_free (s->address);
@@ -1621,13 +1621,11 @@ server_send_callback (void *cls,
  * Callback called by MessageStreamTokenizer when a message has arrived
  *
  * @param cls current session as closure
- * @param client client
  * @param message the message to be forwarded to transport service
  * @return #GNUNET_OK
  */
 static int
 server_receive_mst_cb (void *cls,
-                       void *client,
                        const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_ATS_Session *s = cls;
@@ -1847,13 +1845,16 @@ server_access_cb (void *cls,
              *upload_data_size);
         if (s->msg_tk == NULL)
         {
-          s->msg_tk = GNUNET_SERVER_mst_create (&server_receive_mst_cb, s);
+          s->msg_tk = GNUNET_MST_create (&server_receive_mst_cb,
+                                         s);
         }
-        GNUNET_SERVER_mst_receive (s->msg_tk, s, upload_data, *upload_data_size,
-            GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (s->msg_tk,
+                                upload_data,
+                                *upload_data_size,
+                                GNUNET_NO, GNUNET_NO);
         server_mhd_connection_timeout (plugin, s,
-            GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT.rel_value_us / 1000LL
-                / 1000LL);
+                                       GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT.rel_value_us / 1000LL
+                                       / 1000LL);
         (*upload_data_size) = 0;
       }
       else
@@ -1935,7 +1936,7 @@ server_disconnect_cb (void *cls,
       sc->session->server_recv = NULL;
       if (NULL != sc->session->msg_tk)
       {
-        GNUNET_SERVER_mst_destroy (sc->session->msg_tk);
+        GNUNET_MST_destroy (sc->session->msg_tk);
         sc->session->msg_tk = NULL;
       }
     }
@@ -2223,7 +2224,7 @@ run_mhd_start_daemon (struct HTTP_Server_Plugin *plugin,
                              timeout,
                              MHD_OPTION_CONNECTION_MEMORY_LIMIT,
                              (size_t) (2 *
-                                       GNUNET_SERVER_MAX_MESSAGE_SIZE),
+                                       GNUNET_MAX_MESSAGE_SIZE),
                              MHD_OPTION_NOTIFY_COMPLETED,
                              &server_disconnect_cb, plugin,
                              MHD_OPTION_EXTERNAL_LOGGER,
@@ -2757,7 +2758,7 @@ server_start_report_addresses (struct HTTP_Server_Plugin *plugin)
     return;
   }
 
-  plugin->nat 
+  plugin->nat
     = GNUNET_NAT_register (plugin->env->cfg,
                            "transport-http_server",
                            IPPROTO_TCP,
diff --git a/src/transport/plugin_transport_tcp.c b/src/transport/plugin_transport_tcp.c
index eca62a8ca..a63013caa 100644
--- a/src/transport/plugin_transport_tcp.c
+++ b/src/transport/plugin_transport_tcp.c
@@ -45,10 +45,538 @@
  */
 #define NAT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10)
 
-GNUNET_NETWORK_STRUCT_BEGIN
+/**
+ * Opaque handle that can be used to cancel
+ * a transmit-ready notification.
+ */
+struct GNUNET_CONNECTION_TransmitHandle;
+
+/**
+ * @brief handle for a server
+ */
+struct GNUNET_SERVER_Handle;
+
+/**
+ * @brief opaque handle for a client of the server
+ */
+struct GNUNET_SERVER_Client;
+
+/**
+ * @brief opaque handle server returns for aborting transmission to a client.
+ */
+struct GNUNET_SERVER_TransmitHandle;
+
+/**
+ * @brief handle for a network connection
+ */
+struct GNUNET_CONNECTION_Handle;
+
+/**
+ * @brief handle for a network service
+ */
+struct LEGACY_SERVICE_Context;
+
+
+/**
+ * Stops a service that was started with #GNUNET_SERVICE_start().
+ *
+ * @param srv service to stop
+ */
+void
+LEGACY_SERVICE_stop (struct LEGACY_SERVICE_Context *srv);
+
+
+
+/**
+ * Function called to notify a client about the connection begin ready
+ * to queue more data.  @a buf will be NULL and @a size zero if the
+ * connection was closed for writing in the meantime.
+ *
+ * @param cls closure
+ * @param size number of bytes available in @a buf
+ * @param buf where the callee should write the message
+ * @return number of bytes written to @a buf
+ */
+typedef size_t
+(*GNUNET_CONNECTION_TransmitReadyNotify) (void *cls,
+                                          size_t size,
+                                          void *buf);
+
+/**
+ * Credentials for UNIX domain sockets.
+ */
+struct GNUNET_CONNECTION_Credentials
+{
+  /**
+   * UID of the other end of the connection.
+   */
+  uid_t uid;
+
+  /**
+   * GID of the other end of the connection.
+   */
+  gid_t gid;
+};
+
+
+/**
+ * Functions with this signature are called whenever a client
+ * is disconnected on the network level.
+ *
+ * @param cls closure
+ * @param client identification of the client; NULL
+ *        for the last call when the server is destroyed
+ */
+typedef void
+(*GNUNET_SERVER_DisconnectCallback) (void *cls,
+                                     struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Functions with this signature are called whenever a client
+ * is connected on the network level.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ */
+typedef void
+(*GNUNET_SERVER_ConnectCallback) (void *cls,
+                                  struct GNUNET_SERVER_Client *client);
+
+
+
+
+/**
+ * Function to call for access control checks.
+ *
+ * @param cls closure
+ * @param ucred credentials, if available, otherwise NULL
+ * @param addr address
+ * @param addrlen length of address
+ * @return GNUNET_YES to allow, GNUNET_NO to deny, GNUNET_SYSERR
+ *   for unknown address family (will be denied).
+ */
+typedef int
+(*GNUNET_CONNECTION_AccessCheck) (void *cls,
+                                  const struct
+                                  GNUNET_CONNECTION_Credentials *
+                                  ucred,
+                                  const struct sockaddr * addr,
+                                  socklen_t addrlen);
+
+/**
+ * Callback function for data received from the network.  Note that
+ * both "available" and "err" would be 0 if the read simply timed out.
+ *
+ * @param cls closure
+ * @param buf pointer to received data
+ * @param available number of bytes availabe in "buf",
+ *        possibly 0 (on errors)
+ * @param addr address of the sender
+ * @param addrlen size of addr
+ * @param errCode value of errno (on errors receiving)
+ */
+typedef void
+(*GNUNET_CONNECTION_Receiver) (void *cls, const void *buf,
+                               size_t available,
+                               const struct sockaddr * addr,
+                               socklen_t addrlen, int errCode);
+
+
+
+/**
+ * Close the connection and free associated resources.  There must
+ * not be any pending requests for reading or writing to the
+ * connection at this time.
+ *
+ * @param connection connection to destroy
+ */
+void
+GNUNET_CONNECTION_destroy (struct GNUNET_CONNECTION_Handle *connection);
+
+
+/**
+ * Signature of a function to create a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param client handle to client the tokenzier will be used for
+ * @return handle to custom tokenizer ('mst')
+ */
+typedef void*
+(*GNUNET_SERVER_MstCreateCallback) (void *cls,
+                                    struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Signature of a function to destroy a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param mst custom tokenizer handle
+ */
+typedef void
+(*GNUNET_SERVER_MstDestroyCallback) (void *cls,
+                                     void *mst);
+
+/**
+ * Signature of a function to receive data for a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param mst custom tokenizer handle
+ * @param client_identity ID of client for which this is a buffer,
+ *        can be NULL (will be passed back to 'cb')
+ * @param buf input data to add
+ * @param size number of bytes in @a buf
+ * @param purge should any excess bytes in the buffer be discarded
+ *       (i.e. for packet-based services like UDP)
+ * @param one_shot only call callback once, keep rest of message in buffer
+ * @return #GNUNET_OK if we are done processing (need more data)
+ *         #GNUNET_NO if one_shot was set and we have another message ready
+ *         #GNUNET_SYSERR if the data stream is corrupt
+ */
+typedef int
+(*GNUNET_SERVER_MstReceiveCallback) (void *cls, void *mst,
+                                     struct GNUNET_SERVER_Client *client,
+                                     const char *buf,
+                                     size_t size,
+                                     int purge,
+                                     int one_shot);
+/**
+ * Functions with this signature are called whenever a message is
+ * received.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ */
+typedef void
+(*GNUNET_SERVER_MessageCallback) (void *cls,
+                                  struct GNUNET_SERVER_Client *client,
+                                  const struct GNUNET_MessageHeader *message);
+
+/**
+ * Message handler.  Each struct specifies how to handle on particular
+ * type of message received.
+ */
+struct GNUNET_SERVER_MessageHandler
+{
+  /**
+   * Function to call for messages of "type".
+   */
+  GNUNET_SERVER_MessageCallback callback;
+
+  /**
+   * Closure argument for @e callback.
+   */
+  void *callback_cls;
+
+  /**
+   * Type of the message this handler covers.
+   */
+  uint16_t type;
+
+  /**
+   * Expected size of messages of this type.  Use 0 for
+   * variable-size.  If non-zero, messages of the given
+   * type will be discarded (and the connection closed)
+   * if they do not have the right size.
+   */
+  uint16_t expected_size;
+
+};
+
+
+/**
+ * Options for the service (bitmask).
+ */
+enum LEGACY_SERVICE_Options
+{
+  /**
+   * Use defaults.  Terminates all client connections and the listen
+   * sockets immediately upon receiving the shutdown signal.
+   */
+  LEGACY_SERVICE_OPTION_NONE = 0,
+
+  /**
+   * Do not trigger server shutdown on signal at all; instead, allow
+   * for the user to terminate the server explicitly when needed
+   * by calling #LEGACY_SERVICE_shutdown().
+   */
+  LEGACY_SERVICE_OPTION_MANUAL_SHUTDOWN = 1,
+
+  /**
+   * Trigger a SOFT server shutdown on signals, allowing active
+   * non-monitor clients to complete their transactions.
+   */
+  LEGACY_SERVICE_OPTION_SOFT_SHUTDOWN = 2
+};
+
+
+
+/**
+ * Ask the server to disconnect from the given client.  This is the
+ * same as passing #GNUNET_SYSERR to #GNUNET_SERVER_receive_done,
+ * except that it allows dropping of a client even when not handling a
+ * message from that client.
+ *
+ * @param client the client to disconnect from
+ */
+void
+GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client);
+
+/**
+ * Return user context associated with the given client.
+ * Note: you should probably use the macro (call without the underscore).
+ *
+ * @param client client to query
+ * @param size number of bytes in user context struct (for verification only)
+ * @return pointer to user context
+ */
+void *
+GNUNET_SERVER_client_get_user_context_ (struct GNUNET_SERVER_Client *client,
+                                        size_t size);
 
 
 /**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer.
+ *
+ * Do not call #GNUNET_SERVER_mst_destroy from within
+ * the scope of this callback.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ */
+typedef int
+(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
+                                           void *client,
+                                           const struct GNUNET_MessageHeader *message);
+
+
+/**
+ * Create a message stream tokenizer.
+ *
+ * @param cb function to call on completed messages
+ * @param cb_cls closure for @a cb
+ * @return handle to tokenizer
+ */
+struct GNUNET_SERVER_MessageStreamTokenizer *
+GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
+                          void *cb_cls);
+
+/**
+ * Add incoming data to the receive buffer and call the
+ * callback for all complete messages.
+ *
+ * @param mst tokenizer to use
+ * @param client_identity ID of client for which this is a buffer,
+ *        can be NULL (will be passed back to 'cb')
+ * @param buf input data to add
+ * @param size number of bytes in @a buf
+ * @param purge should any excess bytes in the buffer be discarded
+ *       (i.e. for packet-based services like UDP)
+ * @param one_shot only call callback once, keep rest of message in buffer
+ * @return #GNUNET_OK if we are done processing (need more data)
+ *         #GNUNET_NO if one_shot was set and we have another message ready
+ *         #GNUNET_SYSERR if the data stream is corrupt
+ */
+int
+GNUNET_SERVER_mst_receive (struct GNUNET_SERVER_MessageStreamTokenizer *mst,
+                           void *client_identity,
+                           const char *buf, size_t size,
+                           int purge, int one_shot);
+
+
+
+/**
+ * Destroys a tokenizer.
+ *
+ * @param mst tokenizer to destroy
+ */
+void
+GNUNET_SERVER_mst_destroy (struct GNUNET_SERVER_MessageStreamTokenizer *mst);
+
+
+/**
+ * Set user context to be associated with the given client.
+ * Note: you should probably use the macro (call without the underscore).
+ *
+ * @param client client to query
+ * @param ptr pointer to user context
+ * @param size number of bytes in user context struct (for verification only)
+ */
+void
+GNUNET_SERVER_client_set_user_context_ (struct GNUNET_SERVER_Client *client,
+                                        void *ptr,
+                                        size_t size);
+/**
+ * Return user context associated with the given client.
+ *
+ * @param client client to query
+ * @param type expected return type (i.e. 'struct Foo')
+ * @return pointer to user context of type 'type *'.
+ */
+#define GNUNET_SERVER_client_get_user_context(client,type)              \
+  (type *) GNUNET_SERVER_client_get_user_context_ (client, sizeof (type))
+
+/**
+ * Set user context to be associated with the given client.
+ *
+ * @param client client to query
+ * @param value pointer to user context
+ */
+#define GNUNET_SERVER_client_set_user_context(client,value)             \
+  GNUNET_SERVER_client_set_user_context_ (client, value, sizeof (*value))
+
+
+
+/**
+ * Notify us when the server has enough space to transmit
+ * a message of the given size to the given client.
+ *
+ * @param client client to transmit message to
+ * @param size requested amount of buffer space
+ * @param timeout after how long should we give up (and call
+ *        notify with buf NULL and size 0)?
+ * @param callback function to call when space is available
+ * @param callback_cls closure for @a callback
+ * @return non-NULL if the notify callback was queued; can be used
+ *           to cancel the request using
+ *           #GNUNET_SERVER_notify_transmit_ready_cancel.
+ *         NULL if we are already going to notify someone else (busy)
+ */
+struct GNUNET_SERVER_TransmitHandle *
+GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
+                                     size_t size,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
+                                     void *callback_cls);
+
+/**
+ * Abort transmission request.
+ *
+ * @param th request to abort
+ */
+void
+GNUNET_SERVER_notify_transmit_ready_cancel (struct GNUNET_SERVER_TransmitHandle *th);
+
+
+
+
+/**
+ * Notify the server that the given client handle should
+ * be kept (keeps the connection up if possible, increments
+ * the internal reference counter).
+ *
+ * @param client the client to keep
+ */
+void
+GNUNET_SERVER_client_keep (struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Notify the server that the given client handle is no
+ * longer required.  Decrements the reference counter.  If
+ * that counter reaches zero an inactive connection maybe
+ * closed.
+ *
+ * @param client the client to drop
+ */
+void
+GNUNET_SERVER_client_drop (struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Function called by the service's run
+ * method to run service-specific setup code.
+ *
+ * @param cls closure
+ * @param server the initialized server
+ * @param cfg configuration to use
+ */
+typedef void
+(*LEGACY_SERVICE_Main) (void *cls,
+                        struct GNUNET_SERVER_Handle *server,
+                        const struct GNUNET_CONFIGURATION_Handle *cfg);
+
+
+
+/**
+ * Suspend accepting connections from the listen socket temporarily.
+ * Resume activity using #GNUNET_SERVER_resume.
+ *
+ * @param server server to stop accepting connections.
+ */
+void
+GNUNET_SERVER_suspend (struct GNUNET_SERVER_Handle *server);
+
+/**
+ * Notify us when the server has enough space to transmit
+ * a message of the given size to the given client.
+ *
+ * @param client client to transmit message to
+ * @param size requested amount of buffer space
+ * @param timeout after how long should we give up (and call
+ *        notify with buf NULL and size 0)?
+ * @param callback function to call when space is available
+ * @param callback_cls closure for @a callback
+ * @return non-NULL if the notify callback was queued; can be used
+ *           to cancel the request using
+ *           #GNUNET_SERVER_notify_transmit_ready_cancel.
+ *         NULL if we are already going to notify someone else (busy)
+ */
+struct GNUNET_SERVER_TransmitHandle *
+GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
+                                     size_t size,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
+                                     void *callback_cls);
+
+
+/**
+ * Add a TCP socket-based connection to the set of handles managed by
+ * this server.  Use this function for outgoing (P2P) connections that
+ * we initiated (and where this server should process incoming
+ * messages).
+ *
+ * @param server the server to use
+ * @param connection the connection to manage (client must
+ *        stop using this connection from now on)
+ * @return the client handle
+ */
+struct GNUNET_SERVER_Client *
+GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
+                              struct GNUNET_CONNECTION_Handle *connection);
+
+
+/**
+ * Resume accepting connections from the listen socket.
+ *
+ * @param server server to resume accepting connections.
+ */
+void
+GNUNET_SERVER_resume (struct GNUNET_SERVER_Handle *server);
+
+/**
+ * Free resources held by this server.
+ *
+ * @param server server to destroy
+ */
+void
+GNUNET_SERVER_destroy (struct GNUNET_SERVER_Handle *server);
+
+
+
+
+#include "tcp_connection_legacy.c"
+#include "tcp_server_mst_legacy.c"
+#include "tcp_server_legacy.c"
+#include "tcp_service_legacy.c"
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
  * Initial handshake message for a session.
  */
 struct WelcomeMessage
@@ -434,7 +962,7 @@ struct Plugin
   /**
    * Handle to the network service.
    */
-  struct GNUNET_SERVICE_Context *service;
+  struct LEGACY_SERVICE_Context *service;
 
   /**
    * Handle to the server for this service.
@@ -521,47 +1049,6 @@ struct Plugin
 };
 
 
-/* begin of ancient copy-and-pasted code that should be
-   specialized for TCP ...*/
-/**
- * Add the given UNIX domain path as an address to the
- * list (as the first entry).
- *
- * @param saddrs array to update
- * @param saddrlens where to store the address length
- * @param unixpath path to add
- * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
- *          parameter is ignore on systems other than LINUX
- */
-static void
-add_unixpath (struct sockaddr **saddrs,
-              socklen_t *saddrlens,
-              const char *unixpath,
-              int abstract)
-{
-#ifdef AF_UNIX
-  struct sockaddr_un *un;
-
-  un = GNUNET_new (struct sockaddr_un);
-  un->sun_family = AF_UNIX;
-  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
-#ifdef LINUX
-  if (GNUNET_YES == abstract)
-    un->sun_path[0] = '\0';
-#endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  un->sun_len = (u_char) sizeof (struct sockaddr_un);
-#endif
-  *saddrs = (struct sockaddr *) un;
-  *saddrlens = sizeof (struct sockaddr_un);
-#else
-  /* this function should never be called
-   * unless AF_UNIX is defined! */
-  GNUNET_assert (0);
-#endif
-}
-
-
 /**
  * Get the list of addresses that a server for the given service
  * should bind to.
@@ -1418,7 +1905,7 @@ create_session (struct Plugin *plugin,
     GNUNET_assert (NULL == client);
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating new session for peer `%4s' at address %s\n",
+       "Creating new session for peer `%s' at address %s\n",
        GNUNET_i2s (&address->peer),
        tcp_plugin_address_to_string (plugin,
                                      address->address,
@@ -1522,7 +2009,7 @@ do_transmit (void *cls,
   if (NULL == buf)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Timeout trying to transmit to peer `%4s', discarding message queue.\n",
+         "Timeout trying to transmit to peer `%s', discarding message queue.\n",
          GNUNET_i2s (&session->target));
     /* timeout; cancel all messages that have already expired */
     hd = NULL;
@@ -1540,7 +2027,7 @@ do_transmit (void *cls,
       GNUNET_assert (pos->message_size <= session->bytes_in_queue);
       session->bytes_in_queue -= pos->message_size;
       LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Failed to transmit %u byte message to `%4s'.\n",
+           "Failed to transmit %u byte message to `%s'.\n",
            pos->message_size,
            GNUNET_i2s (&session->target));
       ret += pos->message_size;
@@ -1729,10 +2216,10 @@ tcp_plugin_send (void *cls,
   pm->transmit_cont = cont;
   pm->transmit_cont_cls = cont_cls;
 
-  LOG(GNUNET_ERROR_TYPE_DEBUG,
-      "Asked to transmit %u bytes to `%s', added message to list.\n",
-      msgbuf_size,
-      GNUNET_i2s (&session->target));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Asked to transmit %u bytes to `%s', added message to list.\n",
+       msgbuf_size,
+       GNUNET_i2s (&session->target));
 
   if (GNUNET_YES ==
       GNUNET_CONTAINER_multipeermap_contains_value (plugin->sessionmap,
@@ -1997,6 +2484,13 @@ tcp_plugin_get_session (void *cls,
                                          address->address_length));
       return session;
     }
+    /* This is a bit of a hack, limiting TCP to never allow more than
+       one TCP connection to any given peer at the same time.
+       Without this, peers sometimes disagree about which of the TCP
+       connections they should use, causing one side to believe that
+       they transmit successfully, while the other receives nothing. */
+    return NULL; /* Refuse to have more than one TCP connection per
+                    peer pair at the same time. */
   }
 
   if (addrlen == sizeof(struct IPv6TcpAddress))
@@ -2078,7 +2572,7 @@ tcp_plugin_get_session (void *cls,
                                                 &address->peer)))
   {
     struct sockaddr_in local_sa;
-    
+
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Found valid IPv4 NAT address (creating session)!\n");
     session = create_session (plugin,
@@ -2168,13 +2662,13 @@ tcp_plugin_get_session (void *cls,
   if (NULL == sa)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Failed to create connection to `%4s' at `%s'\n",
+         "Failed to create connection to `%s' at `%s'\n",
          GNUNET_i2s (&address->peer),
          GNUNET_a2s (sb, sbs));
     return NULL;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Asked to transmit to `%4s', creating fresh session using address `%s'.\n",
+       "Asked to transmit to `%s', creating fresh session using address `%s'.\n",
        GNUNET_i2s (&address->peer),
        GNUNET_a2s (sb, sbs));
 
@@ -2246,7 +2740,7 @@ tcp_plugin_disconnect (void *cls,
   struct Plugin *plugin = cls;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Disconnecting peer `%4s'\n",
+       "Disconnecting peer `%s'\n",
        GNUNET_i2s (target));
   GNUNET_CONTAINER_multipeermap_get_multiple (plugin->sessionmap,
                                               target,
@@ -2446,7 +2940,7 @@ tcp_plugin_check_address (void *cls,
   if (addrlen == sizeof(struct IPv4TcpAddress))
   {
     struct sockaddr_in s4;
-    
+
     v4 = (const struct IPv4TcpAddress *) addr;
     if (0 != memcmp (&v4->options,
                      &plugin->myoptions,
@@ -2462,7 +2956,7 @@ tcp_plugin_check_address (void *cls,
 #endif
     s4.sin_port = v4->t4_port;
     s4.sin_addr.s_addr = v4->ipv4_addr;
-    
+
     if (GNUNET_OK !=
         GNUNET_NAT_test_address (plugin->nat,
                                  &s4,
@@ -2472,7 +2966,7 @@ tcp_plugin_check_address (void *cls,
   else
   {
     struct sockaddr_in6 s6;
-    
+
     v6 = (const struct IPv6TcpAddress *) addr;
     if (IN6_IS_ADDR_LINKLOCAL (&v6->ipv6_addr))
     {
@@ -2686,27 +3180,37 @@ handle_tcp_welcome (void *cls,
                                           &alen))
     {
       LOG (GNUNET_ERROR_TYPE_INFO,
-           "Received WELCOME message from my own identity `%4s' on address `%s'\n",
+           "Received WELCOME message from my own identity `%s' on address `%s'\n",
            GNUNET_i2s (&wm->clientIdentity),
            GNUNET_a2s (vaddr, alen));
-      GNUNET_free(vaddr);
+      GNUNET_free (vaddr);
     }
     return;
   }
 
-  LOG(GNUNET_ERROR_TYPE_DEBUG,
-      "Received WELCOME message from `%4s' %p\n",
-      GNUNET_i2s (&wm->clientIdentity),
-      client);
+  if (GNUNET_OK ==
+      GNUNET_SERVER_client_get_address (client,
+                                        &vaddr,
+                                        &alen))
+  {
+    LOG(GNUNET_ERROR_TYPE_DEBUG,
+        "Received WELCOME message from `%s' on address `%s'\n",
+        GNUNET_i2s (&wm->clientIdentity),
+        GNUNET_a2s (vaddr, alen));
+    GNUNET_free (vaddr);
+  }
   GNUNET_STATISTICS_update (plugin->env->stats,
                             gettext_noop ("# TCP WELCOME messages received"),
                             1,
                             GNUNET_NO);
-  session = lookup_session_by_client (plugin, client);
+  session = lookup_session_by_client (plugin,
+                                      client);
   if (NULL != session)
   {
     if (GNUNET_OK ==
-        GNUNET_SERVER_client_get_address (client, &vaddr, &alen))
+        GNUNET_SERVER_client_get_address (client,
+                                          &vaddr,
+                                          &alen))
     {
       LOG (GNUNET_ERROR_TYPE_DEBUG,
            "Found existing session %p for peer `%s'\n",
@@ -2795,12 +3299,11 @@ handle_tcp_welcome (void *cls,
     }
   }
 
-  if (session->expecting_welcome != GNUNET_YES)
+  if (GNUNET_YES != session->expecting_welcome)
   {
-    GNUNET_break_op(0);
+    GNUNET_break_op (0);
     GNUNET_SERVER_receive_done (client,
                                 GNUNET_SYSERR);
-    GNUNET_break(0);
     return;
   }
   session->last_activity = GNUNET_TIME_absolute_get ();
@@ -2869,7 +3372,9 @@ handle_tcp_data (void *cls,
     void *vaddr;
     size_t alen;
 
-    GNUNET_SERVER_client_get_address (client, &vaddr, &alen);
+    GNUNET_SERVER_client_get_address (client,
+                                      &vaddr,
+                                      &alen);
     LOG (GNUNET_ERROR_TYPE_ERROR,
          "Received unexpected %u bytes of type %u from `%s'\n",
          (unsigned int) ntohs (message->size),
@@ -2883,11 +3388,21 @@ handle_tcp_data (void *cls,
   }
 
   session->last_activity = GNUNET_TIME_absolute_get ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Passing %u bytes of type %u from `%4s' to transport service.\n",
-       (unsigned int) ntohs (message->size),
-       (unsigned int) ntohs (message->type),
-       GNUNET_i2s (&session->target));
+  {
+    void *vaddr;
+    size_t alen;
+
+    GNUNET_SERVER_client_get_address (client,
+                                      &vaddr,
+                                      &alen);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Passing %u bytes of type %u from `%s' at %s to transport service.\n",
+         (unsigned int) ntohs (message->size),
+         (unsigned int) ntohs (message->type),
+         GNUNET_i2s (&session->target),
+         GNUNET_a2s (vaddr, alen));
+    GNUNET_free_non_null (vaddr);
+  }
 
   GNUNET_STATISTICS_update (plugin->env->stats,
                             gettext_noop ("# bytes received via TCP"),
@@ -2984,7 +3499,7 @@ disconnect_notify (void *cls,
   if (NULL == session)
     return; /* unknown, nothing to do */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying session of `%4s' with %s due to network-level disconnect.\n",
+       "Destroying session of `%s' with %s due to network-level disconnect.\n",
        GNUNET_i2s (&session->target),
        tcp_plugin_address_to_string (session->plugin,
                                      session->address->address,
@@ -3261,7 +3776,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   struct GNUNET_TRANSPORT_PluginEnvironment *env = cls;
   struct GNUNET_TRANSPORT_PluginFunctions *api;
   struct Plugin *plugin;
-  struct GNUNET_SERVICE_Context *service;
+  struct LEGACY_SERVICE_Context *service;
   unsigned long long aport;
   unsigned long long bport;
   unsigned long long max_connections;
@@ -3316,9 +3831,9 @@ libgnunet_plugin_transport_tcp_init (void *cls)
     aport = 0;
   if (0 != bport)
   {
-    service = GNUNET_SERVICE_start ("transport-tcp",
+    service = LEGACY_SERVICE_start ("transport-tcp",
                                     env->cfg,
-                                    GNUNET_SERVICE_OPTION_NONE);
+                                    LEGACY_SERVICE_OPTION_NONE);
     if (NULL == service)
     {
       LOG (GNUNET_ERROR_TYPE_WARNING,
@@ -3349,7 +3864,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   {
 #ifdef TCP_STEALTH
     plugin->myoptions |= TCP_OPTIONS_TCP_STEALTH;
-    lsocks = GNUNET_SERVICE_get_listen_sockets (service);
+    lsocks = LEGACY_SERVICE_get_listen_sockets (service);
     if (NULL != lsocks)
     {
       uint32_t len = sizeof (struct WelcomeMessage);
@@ -3442,7 +3957,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   plugin->service = service;
   if (NULL != service)
   {
-    plugin->server = GNUNET_SERVICE_get_server (service);
+    plugin->server = LEGACY_SERVICE_get_server (service);
   }
   else
   {
@@ -3505,7 +4020,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
     GNUNET_NAT_unregister (plugin->nat);
   GNUNET_CONTAINER_multipeermap_destroy (plugin->sessionmap);
   if (NULL != service)
-    GNUNET_SERVICE_stop (service);
+    LEGACY_SERVICE_stop (service);
   GNUNET_free (plugin);
   GNUNET_free_non_null (api);
   return NULL;
@@ -3558,7 +4073,7 @@ libgnunet_plugin_transport_tcp_done (void *cls)
   }
 
   if (NULL != plugin->service)
-    GNUNET_SERVICE_stop (plugin->service);
+    LEGACY_SERVICE_stop (plugin->service);
   else
     GNUNET_SERVER_destroy (plugin->server);
   GNUNET_free (plugin->handlers);
diff --git a/src/transport/plugin_transport_udp.c b/src/transport/plugin_transport_udp.c
index fd8493e5f..1ff962544 100644
--- a/src/transport/plugin_transport_udp.c
+++ b/src/transport/plugin_transport_udp.c
@@ -159,6 +159,11 @@ struct GNUNET_ATS_Session
   struct GNUNET_PeerIdentity target;
 
   /**
+   * Tokenizer for inbound messages.
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
    * Plugin this session belongs to.
    */
   struct Plugin *plugin;
@@ -626,6 +631,11 @@ free_session (struct GNUNET_ATS_Session *s)
     GNUNET_free (s->frag_ctx);
     s->frag_ctx = NULL;
   }
+  if (NULL != s->mst)
+  {
+    GNUNET_MST_destroy (s->mst);
+    s->mst = NULL;
+  }
   GNUNET_free (s);
 }
 
@@ -1271,10 +1281,7 @@ udp_plugin_check_address (void *cls,
 
     v6 = (const struct IPv6UdpAddress *) addr;
     if (IN6_IS_ADDR_LINKLOCAL (&v6->ipv6_addr))
-    {
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
+      return GNUNET_OK; /* plausible, if unlikely... */
     memset (&s6, 0, sizeof (s6));
     s6.sin6_family = AF_INET6;
 #if HAVE_SOCKADDR_IN_SIN_LEN
@@ -1338,10 +1345,7 @@ udp_nat_port_map_callback (void *cls,
       GNUNET_assert (sizeof(struct sockaddr_in) == addrlen);
       i4 = (const struct sockaddr_in *) addr;
       if (0 == ntohs (i4->sin_port))
-      {
-        GNUNET_break (0);
-        return;
-      }
+        return; /* Port = 0 means unmapped, ignore these for UDP. */
       memset (&u4,
               0,
               sizeof(u4));
@@ -1359,10 +1363,7 @@ udp_nat_port_map_callback (void *cls,
       GNUNET_assert (sizeof(struct sockaddr_in6) == addrlen);
       i6 = (const struct sockaddr_in6 *) addr;
       if (0 == ntohs (i6->sin6_port))
-      {
-        GNUNET_break (0);
-        return;
-      }
+        return; /* Port = 0 means unmapped, ignore these for UDP. */
       memset (&u6,
               0,
               sizeof(u6));
@@ -1632,7 +1633,7 @@ enqueue (struct Plugin *plugin,
     GNUNET_break (0);
     return;
   }
-  if (plugin->bytes_in_buffer + udpw->msg_size > INT64_MAX)
+  if (plugin->bytes_in_buffer > INT64_MAX - udpw->msg_size)
   {
     GNUNET_break (0);
   }
@@ -2060,7 +2061,7 @@ udp_plugin_send (void *cls,
   if ( (sizeof(struct IPv4UdpAddress) == s->address->address_length) &&
        (NULL == plugin->sockv4) )
     return GNUNET_SYSERR;
-  if (udpmlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (udpmlen >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
@@ -2508,18 +2509,16 @@ read_process_ack (struct Plugin *plugin,
  * Message tokenizer has broken up an incomming message. Pass it on
  * to the service.
  *
- * @param cls the `struct Plugin *`
- * @param client the `struct GNUNET_ATS_Session *`
+ * @param cls the `struct GNUNET_ATS_Session *`
  * @param hdr the actual message
  * @return #GNUNET_OK (always)
  */
 static int
 process_inbound_tokenized_messages (void *cls,
-                                    void *client,
                                     const struct GNUNET_MessageHeader *hdr)
 {
-  struct Plugin *plugin = cls;
-  struct GNUNET_ATS_Session *session = client;
+  struct GNUNET_ATS_Session *session = cls;
+  struct Plugin *plugin = session->plugin;
 
   if (GNUNET_YES == session->in_destroy)
     return GNUNET_OK;
@@ -2635,6 +2634,8 @@ udp_plugin_create_session (void *cls,
   struct GNUNET_ATS_Session *s;
 
   s = GNUNET_new (struct GNUNET_ATS_Session);
+  s->mst = GNUNET_MST_create (&process_inbound_tokenized_messages,
+                              s);
   s->plugin = plugin;
   s->address = GNUNET_HELLO_address_copy (address);
   s->target = address->peer;
@@ -2801,12 +2802,11 @@ process_udp_message (struct Plugin *plugin,
   GNUNET_free (address);
 
   s->rc++;
-  GNUNET_SERVER_mst_receive (plugin->mst,
-                             s,
-                             (const char *) &msg[1],
-                             ntohs (msg->header.size) - sizeof(struct UDPMessage),
-                             GNUNET_YES,
-                             GNUNET_NO);
+  GNUNET_MST_from_buffer (s->mst,
+                          (const char *) &msg[1],
+                          ntohs (msg->header.size) - sizeof(struct UDPMessage),
+                          GNUNET_YES,
+                          GNUNET_NO);
   s->rc--;
   if ( (0 == s->rc) &&
        (GNUNET_YES == s->in_destroy) )
@@ -3999,8 +3999,6 @@ libgnunet_plugin_transport_udp_init (void *cls)
   p->sessions = GNUNET_CONTAINER_multipeermap_create (16,
                                                       GNUNET_NO);
   p->defrag_ctxs = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  p->mst = GNUNET_SERVER_mst_create (&process_inbound_tokenized_messages,
-                                     p);
   GNUNET_BANDWIDTH_tracker_init (&p->tracker,
                                  NULL,
                                  NULL,
@@ -4017,7 +4015,6 @@ libgnunet_plugin_transport_udp_init (void *cls)
         _("Failed to create UDP network sockets\n"));
     GNUNET_CONTAINER_multipeermap_destroy (p->sessions);
     GNUNET_CONTAINER_heap_destroy (p->defrag_ctxs);
-    GNUNET_SERVER_mst_destroy (p->mst);
     if (NULL != p->nat)
       GNUNET_NAT_unregister (p->nat);
     GNUNET_free (p);
@@ -4129,11 +4126,6 @@ libgnunet_plugin_transport_udp_done (void *cls)
     GNUNET_CONTAINER_heap_destroy (plugin->defrag_ctxs);
     plugin->defrag_ctxs = NULL;
   }
-  if (NULL != plugin->mst)
-  {
-    GNUNET_SERVER_mst_destroy (plugin->mst);
-    plugin->mst = NULL;
-  }
   while (NULL != (udpw = plugin->ipv4_queue_head))
   {
     dequeue (plugin,
diff --git a/src/transport/plugin_transport_udp.h b/src/transport/plugin_transport_udp.h
index 152b16099..48c7365c7 100644
--- a/src/transport/plugin_transport_udp.h
+++ b/src/transport/plugin_transport_udp.h
@@ -164,11 +164,6 @@ struct Plugin
   struct GNUNET_SCHEDULER_Task *select_task_v6;
 
   /**
-   * Tokenizer for inbound messages.
-   */
-  struct GNUNET_SERVER_MessageStreamTokenizer *mst;
-
-  /**
    * Bandwidth tracker to limit global UDP traffic.
    */
   struct GNUNET_BANDWIDTH_Tracker tracker;
@@ -192,7 +187,7 @@ struct Plugin
    * Handle to NAT traversal support.
    */
   struct GNUNET_NAT_STUN_Handle *stun;
-  
+
   /**
    * The read socket for IPv4
    */
@@ -204,11 +199,6 @@ struct Plugin
   struct GNUNET_NETWORK_Handle *sockv6;
 
   /**
-   * Tokenizer for inbound messages.
-   */
-  struct GNUNET_SERVER_MessageStreamTokenizer *broadcast_mst;
-
-  /**
    * Head of DLL of broadcast addresses
    */
   struct BroadcastAddress *broadcast_tail;
diff --git a/src/transport/plugin_transport_udp_broadcasting.c b/src/transport/plugin_transport_udp_broadcasting.c
index a440830fd..c6ddbce9b 100644
--- a/src/transport/plugin_transport_udp_broadcasting.c
+++ b/src/transport/plugin_transport_udp_broadcasting.c
@@ -133,11 +133,10 @@ struct MstContext
  */
 static int
 broadcast_mst_cb (void *cls,
-                  void *client,
                   const struct GNUNET_MessageHeader *message)
 {
-  struct Plugin *plugin = cls;
-  struct MstContext *mc = client;
+  struct MstContext *mc = cls;
+  struct Plugin *plugin = mc->plugin;
   struct GNUNET_HELLO_Address *address;
   const struct GNUNET_MessageHeader *hello;
   const struct UDP_Beacon_Message *msg;
@@ -191,16 +190,20 @@ udp_broadcast_receive (struct Plugin *plugin,
                        size_t udp_addr_len,
                        enum GNUNET_ATS_Network_Type network_type)
 {
+  struct GNUNET_MessageStreamTokenizer *broadcast_mst;
   struct MstContext mc;
 
+  broadcast_mst = GNUNET_MST_create (&broadcast_mst_cb,
+                                     &mc);
+  mc.plugin = plugin;
   mc.udp_addr = udp_addr;
   mc.udp_addr_len = udp_addr_len;
   mc.ats_address_network_type = network_type;
-  GNUNET_SERVER_mst_receive (plugin->broadcast_mst,
-                             &mc,
-                             buf, size,
-                             GNUNET_NO,
-                             GNUNET_NO);
+  GNUNET_MST_from_buffer (broadcast_mst,
+                          buf, size,
+                          GNUNET_NO,
+                          GNUNET_NO);
+  GNUNET_MST_destroy (broadcast_mst);
 }
 
 
@@ -546,10 +549,6 @@ setup_broadcast (struct Plugin *plugin,
     return;
   }
 
-  /* always create tokenizers */
-  plugin->broadcast_mst =
-    GNUNET_SERVER_mst_create (&broadcast_mst_cb, plugin);
-
   if (GNUNET_YES != plugin->enable_broadcasting)
     return; /* We do not send, just receive */
 
@@ -636,13 +635,6 @@ stop_broadcast (struct Plugin *plugin)
       GNUNET_free (p);
     }
   }
-
-  /* Destroy MSTs */
-  if (NULL != plugin->broadcast_mst)
-  {
-    GNUNET_SERVER_mst_destroy (plugin->broadcast_mst);
-    plugin->broadcast_mst = NULL;
-  }
 }
 
 /* end of plugin_transport_udp_broadcasting.c */
diff --git a/src/transport/plugin_transport_unix.c b/src/transport/plugin_transport_unix.c
index 49a5e89ed..6bcb7290f 100644
--- a/src/transport/plugin_transport_unix.c
+++ b/src/transport/plugin_transport_unix.c
@@ -566,7 +566,7 @@ unix_address_to_sockaddr (const char *unixpath,
   GNUNET_memcpy (un->sun_path, unixpath, slen);
   un->sun_path[slen] = '\0';
   slen = sizeof (struct sockaddr_un);
-#if HAVE_SOCKADDR_IN_SIN_LEN
+#if HAVE_SOCKADDR_UN_SUN_LEN
   un->sun_len = (u_char) slen;
 #endif
   (*sock_len) = slen;
@@ -1229,7 +1229,7 @@ unix_plugin_select_read (void *cls)
 {
   struct Plugin *plugin = cls;
   const struct GNUNET_SCHEDULER_TaskContext *tc;
-  
+
   plugin->read_task = NULL;
   tc = GNUNET_SCHEDULER_get_task_context ();
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
@@ -1252,7 +1252,7 @@ unix_plugin_select_write (void *cls)
 {
   struct Plugin *plugin = cls;
   const struct GNUNET_SCHEDULER_TaskContext *tc;
-  
+
   plugin->write_task = NULL;
   tc = GNUNET_SCHEDULER_get_task_context ();
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_WRITE_READY))
diff --git a/src/transport/plugin_transport_wlan.c b/src/transport/plugin_transport_wlan.c
index 376065d24..b23739d3c 100644
--- a/src/transport/plugin_transport_wlan.c
+++ b/src/transport/plugin_transport_wlan.c
@@ -38,6 +38,7 @@
 #include "gnunet_fragmentation_lib.h"
 #include "gnunet_constants.h"
 
+
 #if BUILD_WLAN
 /* begin case wlan */
 #define PLUGIN_NAME "wlan"
@@ -48,6 +49,7 @@
 #define LIBGNUNET_PLUGIN_TRANSPORT_DONE libgnunet_plugin_transport_wlan_done
 #define LOG(kind,...) GNUNET_log_from (kind, "transport-wlan",__VA_ARGS__)
 
+
 /**
  * time out of a mac endpoint
  */
@@ -92,6 +94,30 @@
 #error need to build wlan or bluetooth
 #endif
 
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer.
+ *
+ * Do not call #GNUNET_SERVER_mst_destroy from within
+ * the scope of this callback.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ */
+typedef int
+(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
+                                           void *client,
+                                           const struct GNUNET_MessageHeader *message);
+
+
+/* Include legacy message stream tokenizer that was removed from util (for now) */
+#include "tcp_server_mst_legacy.c"
+
+
 /**
  * Max size of packet (that we give to the WLAN driver for transmission)
  */
@@ -718,7 +744,7 @@ send_ack (void *cls,
     GNUNET_break (0);
     return;
   }
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1728,11 +1754,10 @@ send_hello_beacon (void *cls)
  * Function used for to process the data from the suid process
  *
  * @param cls the plugin handle
- * @param client client that send the data (not used)
  * @param hdr header of the GNUNET_MessageHeader
  */
 static int
-handle_helper_message (void *cls, void *client,
+handle_helper_message (void *cls,
                        const struct GNUNET_MessageHeader *hdr)
 {
   struct Plugin *plugin = cls;
diff --git a/src/util/connection.c b/src/transport/tcp_connection_legacy.c
index 6d3070415..5b219a467 100644
--- a/src/util/connection.c
+++ b/src/transport/tcp_connection_legacy.c
@@ -35,9 +35,17 @@
 #include "gnunet_resolver_service.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+/**
+ * Timeout we use on TCP connect before trying another
+ * result from the DNS resolver.  Actual value used
+ * is this value divided by the number of address families.
+ * Default is 5s.
+ */
+#define CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
+
+
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-connection", syscall)
 
 
 /**
@@ -305,7 +313,7 @@ GNUNET_CONNECTION_create_from_existing (struct GNUNET_NETWORK_Handle *osSocket)
   struct GNUNET_CONNECTION_Handle *connection;
 
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->sock = osSocket;
   return connection;
@@ -335,12 +343,17 @@ GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
   struct sockaddr_in6 *v6;
   struct sockaddr *sa;
   void *uaddr;
-  struct GNUNET_CONNECTION_Credentials *gcp;
-  struct GNUNET_CONNECTION_Credentials gc;
 #ifdef SO_PEERCRED
   struct ucred uc;
   socklen_t olen;
 #endif
+  struct GNUNET_CONNECTION_Credentials *gcp;
+#if HAVE_GETPEEREID || defined(SO_PEERCRED) || HAVE_GETPEERUCRED
+  struct GNUNET_CONNECTION_Credentials gc;
+
+  gc.uid = 0;
+  gc.gid = 0;
+#endif
 
   addrlen = sizeof (addr);
   sock =
@@ -386,8 +399,6 @@ GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
     GNUNET_memcpy (uaddr, addr, addrlen);
   }
   gcp = NULL;
-  gc.uid = 0;
-  gc.gid = 0;
   if (AF_UNIX == sa->sa_family)
   {
 #if HAVE_GETPEEREID
@@ -449,7 +460,7 @@ GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
     return NULL;
   }
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->addr = uaddr;
   connection->addrlen = addrlen;
@@ -822,7 +833,7 @@ try_connect_using_address (void *cls,
     return;
   }
   GNUNET_CONTAINER_DLL_insert (connection->ap_head, connection->ap_tail, ap);
-  delay = GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT;
+  delay = CONNECT_RETRY_TIMEOUT;
   if (NULL != connection->nth.notify_ready)
     delay = GNUNET_TIME_relative_min (delay,
                                       GNUNET_TIME_absolute_get_remaining (connection->nth.transmit_timeout));
@@ -856,14 +867,14 @@ GNUNET_CONNECTION_create_from_connect (const struct GNUNET_CONFIGURATION_Handle
   GNUNET_assert (0 < strlen (hostname));        /* sanity check */
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
   connection->cfg = cfg;
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->port = port;
   connection->hostname = GNUNET_strdup (hostname);
   connection->dns_active =
       GNUNET_RESOLVER_ip_get (connection->hostname,
                               AF_UNSPEC,
-                              GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+                              CONNECT_RETRY_TIMEOUT,
                               &try_connect_using_address,
                               connection);
   return connection;
@@ -902,12 +913,12 @@ GNUNET_CONNECTION_create_from_connect_to_unixpath (const struct GNUNET_CONFIGURA
       un->sun_path[0] = '\0';
   }
 #endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
+#if HAVE_SOCKADDR_UN_SUN_LEN
   un->sun_len = (u_char) sizeof (struct sockaddr_un);
 #endif
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
   connection->cfg = cfg;
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->port = 0;
   connection->hostname = NULL;
@@ -1012,7 +1023,7 @@ GNUNET_CONNECTION_create_from_sockaddr (int af_family,
     return NULL;
   }
   return GNUNET_CONNECTION_connect_socket (s,
-                                           serv_addr, 
+                                           serv_addr,
                                            addrlen);
 }
 
@@ -1533,7 +1544,7 @@ GNUNET_CONNECTION_notify_transmit_ready (struct GNUNET_CONNECTION_Handle *connec
     return NULL;
   }
   GNUNET_assert (NULL != notify);
-  GNUNET_assert (size < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (size < GNUNET_MAX_MESSAGE_SIZE);
   GNUNET_assert (connection->write_buffer_off <= connection->write_buffer_size);
   GNUNET_assert (connection->write_buffer_pos <= connection->write_buffer_size);
   GNUNET_assert (connection->write_buffer_pos <= connection->write_buffer_off);
diff --git a/src/util/server.c b/src/transport/tcp_server_legacy.c
index 00e37c9d9..6b4daa525 100644
--- a/src/util/server.c
+++ b/src/transport/tcp_server_legacy.c
@@ -28,11 +28,7 @@
 #include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
-
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-server", syscall, filename)
 
 
 /**
@@ -945,9 +941,9 @@ GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
 
   type = ntohs (message->type);
   size = ntohs (message->size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Server schedules transmission of %u-byte message of type %u to client.\n",
-       size, type);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received message of type %u and size %u from client\n",
+       type, size);
   found = GNUNET_NO;
   for (pos = server->handlers; NULL != pos; pos = pos->next)
   {
@@ -1049,7 +1045,7 @@ process_mst (struct GNUNET_SERVER_Client *client,
            GNUNET_STRINGS_relative_time_to_string (client->idle_timeout, GNUNET_YES));
       client->receive_pending = GNUNET_YES;
       GNUNET_CONNECTION_receive (client->connection,
-                                 GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                                 GNUNET_MAX_MESSAGE_SIZE - 1,
                                  client->idle_timeout,
                                  &process_incoming,
                                  client);
@@ -1128,7 +1124,7 @@ process_incoming (void *cls,
          client);
     client->receive_pending = GNUNET_YES;
     GNUNET_CONNECTION_receive (client->connection,
-                               GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                               GNUNET_MAX_MESSAGE_SIZE - 1,
                                GNUNET_TIME_absolute_get_remaining (end),
                                &process_incoming,
                                client);
@@ -1204,7 +1200,7 @@ restart_processing (void *cls)
     LOG (GNUNET_ERROR_TYPE_DEBUG, "Server begins to read again from client.\n");
     client->receive_pending = GNUNET_YES;
     GNUNET_CONNECTION_receive (client->connection,
-                               GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                               GNUNET_MAX_MESSAGE_SIZE - 1,
                                client->idle_timeout,
                                &process_incoming,
                                client);
@@ -1240,8 +1236,8 @@ client_message_tokenizer_callback (void *cls,
   int ret;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tokenizer gives server message of type %u from client\n",
-       ntohs (message->type));
+       "Tokenizer gives server message of type %u and size %u from client\n",
+       ntohs (message->type), ntohs (message->size));
   sender->in_process_client_buffer = GNUNET_YES;
   ret = GNUNET_SERVER_inject (server, sender, message);
   sender->in_process_client_buffer = GNUNET_NO;
@@ -1292,7 +1288,7 @@ GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
     n->callback (n->callback_cls, client);
   client->receive_pending = GNUNET_YES;
   GNUNET_CONNECTION_receive (client->connection,
-                             GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                             GNUNET_MAX_MESSAGE_SIZE - 1,
                              client->idle_timeout,
                              &process_incoming,
                              client);
diff --git a/src/util/server_mst.c b/src/transport/tcp_server_mst_legacy.c
index 8c9bc4b5b..78e04c1b1 100644
--- a/src/util/server_mst.c
+++ b/src/transport/tcp_server_mst_legacy.c
@@ -34,8 +34,6 @@
 #define ALIGN_FACTOR 8
 #endif
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-
 
 /**
  * Handle to a message stream tokenizer.
@@ -91,8 +89,8 @@ GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
   struct GNUNET_SERVER_MessageStreamTokenizer *ret;
 
   ret = GNUNET_new (struct GNUNET_SERVER_MessageStreamTokenizer);
-  ret->hdr = GNUNET_malloc (GNUNET_SERVER_MIN_BUFFER_SIZE);
-  ret->curr_buf = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  ret->hdr = GNUNET_malloc (GNUNET_MIN_MESSAGE_SIZE);
+  ret->curr_buf = GNUNET_MIN_MESSAGE_SIZE;
   ret->cb = cb;
   ret->cb_cls = cb_cls;
   return ret;
diff --git a/src/transport/tcp_service_legacy.c b/src/transport/tcp_service_legacy.c
new file mode 100644
index 000000000..1f1f6c063
--- /dev/null
+++ b/src/transport/tcp_service_legacy.c
@@ -0,0 +1,1688 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2012 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file util/service.c
+ * @brief functions related to starting services
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_protocols.h"
+#include "gnunet_constants.h"
+#include "gnunet_resolver_service.h"
+
+#if HAVE_MALLINFO
+#include <malloc.h>
+#include "gauger.h"
+#endif
+
+
+/* ******************* access control ******************** */
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param add the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
+                   const struct in_addr *add)
+{
+  unsigned int i;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  i = 0;
+  while ((list[i].network.s_addr != 0) || (list[i].netmask.s_addr != 0))
+  {
+    if ((add->s_addr & list[i].netmask.s_addr) ==
+        (list[i].network.s_addr & list[i].netmask.s_addr))
+      return GNUNET_YES;
+    i++;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param ip the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
+                   const struct in6_addr *ip)
+{
+  unsigned int i;
+  unsigned int j;
+  struct in6_addr zero;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  memset (&zero, 0, sizeof (struct in6_addr));
+  i = 0;
+NEXT:
+  while (0 != memcmp (&zero, &list[i].network, sizeof (struct in6_addr)))
+  {
+    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
+      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
+          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
+      {
+        i++;
+        goto NEXT;
+      }
+    return GNUNET_YES;
+  }
+  return GNUNET_NO;
+}
+
+
+/* ****************** service struct ****************** */
+
+
+/**
+ * Context for "service_task".
+ */
+struct LEGACY_SERVICE_Context
+{
+  /**
+   * Our configuration.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Handle for the server.
+   */
+  struct GNUNET_SERVER_Handle *server;
+
+  /**
+   * NULL-terminated array of addresses to bind to, NULL if we got pre-bound
+   * listen sockets.
+   */
+  struct sockaddr **addrs;
+
+  /**
+   * Name of our service.
+   */
+  const char *service_name;
+
+  /**
+   * Main service-specific task to run.
+   */
+  LEGACY_SERVICE_Main task;
+
+  /**
+   * Closure for @e task.
+   */
+  void *task_cls;
+
+  /**
+   * IPv4 addresses that are not allowed to connect.
+   */
+  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_denied;
+
+  /**
+   * IPv6 addresses that are not allowed to connect.
+   */
+  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_denied;
+
+  /**
+   * IPv4 addresses that are allowed to connect (if not
+   * set, all are allowed).
+   */
+  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_allowed;
+
+  /**
+   * IPv6 addresses that are allowed to connect (if not
+   * set, all are allowed).
+   */
+  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
+
+  /**
+   * My (default) message handlers.  Adjusted copy
+   * of "defhandlers".
+   */
+  struct GNUNET_SERVER_MessageHandler *my_handlers;
+
+  /**
+   * Array of the lengths of the entries in addrs.
+   */
+  socklen_t *addrlens;
+
+  /**
+   * NULL-terminated array of listen sockets we should take over.
+   */
+  struct GNUNET_NETWORK_Handle **lsocks;
+
+  /**
+   * Task ID of the shutdown task.
+   */
+  struct GNUNET_SCHEDULER_Task *shutdown_task;
+
+  /**
+   * Idle timeout for server.
+   */
+  struct GNUNET_TIME_Relative timeout;
+
+  /**
+   * Overall success/failure of the service start.
+   */
+  int ret;
+
+  /**
+   * If we are daemonizing, this FD is set to the
+   * pipe to the parent.  Send '.' if we started
+   * ok, '!' if not.  -1 if we are not daemonizing.
+   */
+  int ready_confirm_fd;
+
+  /**
+   * Do we close connections if we receive messages
+   * for which we have no handler?
+   */
+  int require_found;
+
+  /**
+   * Do we require a matching UID for UNIX domain socket connections?
+   * #GNUNET_NO means that the UID does not have to match (however,
+   * @e match_gid may still impose other access control checks).
+   */
+  int match_uid;
+
+  /**
+   * Do we require a matching GID for UNIX domain socket connections?
+   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
+   * checking that the client's UID is in our group OR that the
+   * client's GID is our GID.  If both "match_gid" and @e match_uid are
+   * #GNUNET_NO, all users on the local system have access.
+   */
+  int match_gid;
+
+  /**
+   * Our options.
+   */
+  enum LEGACY_SERVICE_Options options;
+
+};
+
+
+/* ****************** message handlers ****************** */
+
+/**
+ * Send a 'TEST' message back to the client.
+ *
+ * @param cls the 'struct GNUNET_SERVER_Client' to send TEST to
+ * @param size number of bytes available in 'buf'
+ * @param buf where to copy the message
+ * @return number of bytes written to 'buf'
+ */
+static size_t
+write_test (void *cls, size_t size, void *buf)
+{
+  struct GNUNET_SERVER_Client *client = cls;
+  struct GNUNET_MessageHeader *msg;
+
+  if (size < sizeof (struct GNUNET_MessageHeader))
+  {
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return 0;                   /* client disconnected */
+  }
+  msg = (struct GNUNET_MessageHeader *) buf;
+  msg->type = htons (GNUNET_MESSAGE_TYPE_TEST);
+  msg->size = htons (sizeof (struct GNUNET_MessageHeader));
+  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  return sizeof (struct GNUNET_MessageHeader);
+}
+
+
+/**
+ * Handler for TEST message.
+ *
+ * @param cls closure (refers to service)
+ * @param client identification of the client
+ * @param message the actual message
+ */
+static void
+handle_test (void *cls, struct GNUNET_SERVER_Client *client,
+             const struct GNUNET_MessageHeader *message)
+{
+  /* simply bounce message back to acknowledge */
+  if (NULL ==
+      GNUNET_SERVER_notify_transmit_ready (client,
+                                           sizeof (struct GNUNET_MessageHeader),
+                                           GNUNET_TIME_UNIT_FOREVER_REL,
+                                           &write_test, client))
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+}
+
+
+/**
+ * Default handlers for all services.  Will be copied and the
+ * "callback_cls" fields will be replaced with the specific service
+ * struct.
+ */
+static const struct GNUNET_SERVER_MessageHandler defhandlers[] = {
+  {&handle_test, NULL, GNUNET_MESSAGE_TYPE_TEST,
+   sizeof (struct GNUNET_MessageHeader)},
+  {NULL, NULL, 0, 0}
+};
+
+
+/* ****************** service core routines ************** */
+
+
+/**
+ * Check if access to the service is allowed from the given address.
+ *
+ * @param cls closure
+ * @param uc credentials, if available, otherwise NULL
+ * @param addr address
+ * @param addrlen length of address
+ * @return #GNUNET_YES to allow, #GNUNET_NO to deny, #GNUNET_SYSERR
+ *   for unknown address family (will be denied).
+ */
+static int
+check_access (void *cls, const struct GNUNET_CONNECTION_Credentials *uc,
+              const struct sockaddr *addr, socklen_t addrlen)
+{
+  struct LEGACY_SERVICE_Context *sctx = cls;
+  const struct sockaddr_in *i4;
+  const struct sockaddr_in6 *i6;
+  int ret;
+
+  switch (addr->sa_family)
+  {
+  case AF_INET:
+    GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
+    i4 = (const struct sockaddr_in *) addr;
+    ret = ((NULL == sctx->v4_allowed) ||
+           (check_ipv4_listed (sctx->v4_allowed, &i4->sin_addr))) &&
+        ((NULL == sctx->v4_denied) ||
+         (!check_ipv4_listed (sctx->v4_denied, &i4->sin_addr)));
+    break;
+  case AF_INET6:
+    GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
+    i6 = (const struct sockaddr_in6 *) addr;
+    ret = ((NULL == sctx->v6_allowed) ||
+           (check_ipv6_listed (sctx->v6_allowed, &i6->sin6_addr))) &&
+        ((NULL == sctx->v6_denied) ||
+         (!check_ipv6_listed (sctx->v6_denied, &i6->sin6_addr)));
+    break;
+#ifndef WINDOWS
+  case AF_UNIX:
+    ret = GNUNET_OK;            /* controlled using file-system ACL now */
+    break;
+#endif
+  default:
+    LOG (GNUNET_ERROR_TYPE_WARNING, _("Unknown address family %d\n"),
+         addr->sa_family);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK != ret)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Access from `%s' denied to service `%s'\n"),
+         GNUNET_a2s (addr, addrlen),
+         sctx->service_name);
+  }
+  return ret;
+}
+
+
+/**
+ * Get the name of the file where we will
+ * write the PID of the service.
+ *
+ * @param sctx service context
+ * @return name of the file for the process ID
+ */
+static char *
+get_pid_file_name (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *pif;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
+                                               "PIDFILE", &pif))
+    return NULL;
+  return pif;
+}
+
+
+/**
+ * Parse an IPv4 access control list.
+ *
+ * @param ret location where to write the ACL (set)
+ * @param sctx service context to use to get the configuration
+ * @param option name of the ACL option to parse
+ * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
+ *         no ACL configured)
+ */
+static int
+process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
+              struct LEGACY_SERVICE_Context *sctx,
+              const char *option)
+{
+  char *opt;
+
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  {
+    *ret = NULL;
+    return GNUNET_OK;
+  }
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
+                                                       sctx->service_name,
+                                                       option, &opt));
+  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
+         opt, sctx->service_name, option);
+    GNUNET_free (opt);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (opt);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Parse an IPv6 access control list.
+ *
+ * @param ret location where to write the ACL (set)
+ * @param sctx service context to use to get the configuration
+ * @param option name of the ACL option to parse
+ * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
+ *         no ACL configured)
+ */
+static int
+process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
+              struct LEGACY_SERVICE_Context *sctx,
+              const char *option)
+{
+  char *opt;
+
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  {
+    *ret = NULL;
+    return GNUNET_OK;
+  }
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
+                                                       sctx->service_name,
+                                                       option, &opt));
+  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
+         opt, sctx->service_name, option);
+    GNUNET_free (opt);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (opt);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Add the given UNIX domain path as an address to the
+ * list (as the first entry).
+ *
+ * @param saddrs array to update
+ * @param saddrlens where to store the address length
+ * @param unixpath path to add
+ * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
+ *          parameter is ignore on systems other than LINUX
+ */
+static void
+add_unixpath (struct sockaddr **saddrs,
+              socklen_t *saddrlens,
+              const char *unixpath,
+              int abstract)
+{
+#ifdef AF_UNIX
+  struct sockaddr_un *un;
+
+  un = GNUNET_new (struct sockaddr_un);
+  un->sun_family = AF_UNIX;
+  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
+#ifdef LINUX
+  if (GNUNET_YES == abstract)
+    un->sun_path[0] = '\0';
+#endif
+#if HAVE_SOCKADDR_UN_SUN_LEN
+  un->sun_len = (u_char) sizeof (struct sockaddr_un);
+#endif
+  *saddrs = (struct sockaddr *) un;
+  *saddrlens = sizeof (struct sockaddr_un);
+#else
+  /* this function should never be called
+   * unless AF_UNIX is defined! */
+  GNUNET_assert (0);
+#endif
+}
+
+
+/**
+ * Get the list of addresses that a server for the given service
+ * should bind to.
+ *
+ * @param service_name name of the service
+ * @param cfg configuration (which specifies the addresses)
+ * @param addrs set (call by reference) to an array of pointers to the
+ *              addresses the server should bind to and listen on; the
+ *              array will be NULL-terminated (on success)
+ * @param addr_lens set (call by reference) to an array of the lengths
+ *              of the respective `struct sockaddr` struct in the @a addrs
+ *              array (on success)
+ * @return number of addresses found on success,
+ *              #GNUNET_SYSERR if the configuration
+ *              did not specify reasonable finding information or
+ *              if it specified a hostname that could not be resolved;
+ *              #GNUNET_NO if the number of addresses configured is
+ *              zero (in this case, `*addrs` and `*addr_lens` will be
+ *              set to NULL).
+ */
+int
+LEGACY_SERVICE_get_server_addresses (const char *service_name,
+                                     const struct GNUNET_CONFIGURATION_Handle *cfg,
+                                     struct sockaddr ***addrs,
+                                     socklen_t ** addr_lens)
+{
+  int disablev6;
+  struct GNUNET_NETWORK_Handle *desc;
+  unsigned long long port;
+  char *unixpath;
+  struct addrinfo hints;
+  struct addrinfo *res;
+  struct addrinfo *pos;
+  struct addrinfo *next;
+  unsigned int i;
+  int resi;
+  int ret;
+  int abstract;
+  struct sockaddr **saddrs;
+  socklen_t *saddrlens;
+  char *hostname;
+
+  *addrs = NULL;
+  *addr_lens = NULL;
+  desc = NULL;
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "DISABLEV6"))
+  {
+    if (GNUNET_SYSERR ==
+        (disablev6 =
+         GNUNET_CONFIGURATION_get_value_yesno (cfg, service_name, "DISABLEV6")))
+      return GNUNET_SYSERR;
+  }
+  else
+    disablev6 = GNUNET_NO;
+
+  if (! disablev6)
+  {
+    /* probe IPv6 support */
+    desc = GNUNET_NETWORK_socket_create (PF_INET6, SOCK_STREAM, 0);
+    if (NULL == desc)
+    {
+      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+          (EACCES == errno))
+      {
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        return GNUNET_SYSERR;
+      }
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
+           service_name, STRERROR (errno));
+      disablev6 = GNUNET_YES;
+    }
+    else
+    {
+      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      desc = NULL;
+    }
+  }
+
+  port = 0;
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "PORT"))
+  {
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_number (cfg, service_name,
+                                               "PORT", &port))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Require valid port number for service `%s' in configuration!\n"),
+           service_name);
+    }
+    if (port > 65535)
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Require valid port number for service `%s' in configuration!\n"),
+           service_name);
+      return GNUNET_SYSERR;
+    }
+  }
+
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "BINDTO"))
+  {
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_CONFIGURATION_get_value_string (cfg, service_name,
+                                                         "BINDTO", &hostname));
+  }
+  else
+    hostname = NULL;
+
+  unixpath = NULL;
+  abstract = GNUNET_NO;
+#ifdef AF_UNIX
+  if ((GNUNET_YES ==
+       GNUNET_CONFIGURATION_have_value (cfg, service_name, "UNIXPATH")) &&
+      (GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_filename (cfg, service_name, "UNIXPATH",
+                                              &unixpath)) &&
+      (0 < strlen (unixpath)))
+  {
+    /* probe UNIX support */
+    struct sockaddr_un s_un;
+
+    if (strlen (unixpath) >= sizeof (s_un.sun_path))
+    {
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           _("UNIXPATH `%s' too long, maximum length is %llu\n"), unixpath,
+           (unsigned long long) sizeof (s_un.sun_path));
+      unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Using `%s' instead\n"),
+           unixpath);
+    }
+#ifdef LINUX
+    abstract = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                                     "TESTING",
+                                                     "USE_ABSTRACT_SOCKETS");
+    if (GNUNET_SYSERR == abstract)
+      abstract = GNUNET_NO;
+#endif
+    if ((GNUNET_YES != abstract)
+        && (GNUNET_OK !=
+            GNUNET_DISK_directory_create_for_file (unixpath)))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "mkdir",
+                                unixpath);
+  }
+  if (NULL != unixpath)
+  {
+    desc = GNUNET_NETWORK_socket_create (AF_UNIX, SOCK_STREAM, 0);
+    if (NULL == desc)
+    {
+      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+          (EACCES == errno))
+      {
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        GNUNET_free_non_null (hostname);
+        GNUNET_free (unixpath);
+        return GNUNET_SYSERR;
+      }
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Disabling UNIX domain socket support for service `%s', failed to create UNIX domain socket: %s\n"),
+           service_name,
+           STRERROR (errno));
+      GNUNET_free (unixpath);
+      unixpath = NULL;
+    }
+    else
+    {
+      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      desc = NULL;
+    }
+  }
+#endif
+
+  if ((0 == port) && (NULL == unixpath))
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Have neither PORT nor UNIXPATH for service `%s', but one is required\n"),
+         service_name);
+    GNUNET_free_non_null (hostname);
+    return GNUNET_SYSERR;
+  }
+  if (0 == port)
+  {
+    saddrs = GNUNET_malloc (2 * sizeof (struct sockaddr *));
+    saddrlens = GNUNET_malloc (2 * sizeof (socklen_t));
+    add_unixpath (saddrs, saddrlens, unixpath, abstract);
+    GNUNET_free_non_null (unixpath);
+    GNUNET_free_non_null (hostname);
+    *addrs = saddrs;
+    *addr_lens = saddrlens;
+    return 1;
+  }
+
+  if (NULL != hostname)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Resolving `%s' since that is where `%s' will bind to.\n",
+         hostname,
+         service_name);
+    memset (&hints, 0, sizeof (struct addrinfo));
+    if (disablev6)
+      hints.ai_family = AF_INET;
+    hints.ai_protocol = IPPROTO_TCP;
+    if ((0 != (ret = getaddrinfo (hostname, NULL, &hints, &res))) ||
+        (NULL == res))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Failed to resolve `%s': %s\n"),
+           hostname,
+           gai_strerror (ret));
+      GNUNET_free (hostname);
+      GNUNET_free_non_null (unixpath);
+      return GNUNET_SYSERR;
+    }
+    next = res;
+    i = 0;
+    while (NULL != (pos = next))
+    {
+      next = pos->ai_next;
+      if ((disablev6) && (pos->ai_family == AF_INET6))
+        continue;
+      i++;
+    }
+    if (0 == i)
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Failed to find %saddress for `%s'.\n"),
+           disablev6 ? "IPv4 " : "",
+           hostname);
+      freeaddrinfo (res);
+      GNUNET_free (hostname);
+      GNUNET_free_non_null (unixpath);
+      return GNUNET_SYSERR;
+    }
+    resi = i;
+    if (NULL != unixpath)
+      resi++;
+    saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+    saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+    i = 0;
+    if (NULL != unixpath)
+    {
+      add_unixpath (saddrs, saddrlens, unixpath, abstract);
+      i++;
+    }
+    next = res;
+    while (NULL != (pos = next))
+    {
+      next = pos->ai_next;
+      if ((disablev6) && (AF_INET6 == pos->ai_family))
+        continue;
+      if ((IPPROTO_TCP != pos->ai_protocol) && (0 != pos->ai_protocol))
+        continue;               /* not TCP */
+      if ((SOCK_STREAM != pos->ai_socktype) && (0 != pos->ai_socktype))
+        continue;               /* huh? */
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "Service `%s' will bind to `%s'\n",
+           service_name, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
+      if (AF_INET == pos->ai_family)
+      {
+        GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
+        saddrlens[i] = pos->ai_addrlen;
+        saddrs[i] = GNUNET_malloc (saddrlens[i]);
+        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+      }
+      else
+      {
+        GNUNET_assert (AF_INET6 == pos->ai_family);
+        GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
+        saddrlens[i] = pos->ai_addrlen;
+        saddrs[i] = GNUNET_malloc (saddrlens[i]);
+        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
+      }
+      i++;
+    }
+    GNUNET_free (hostname);
+    freeaddrinfo (res);
+    resi = i;
+  }
+  else
+  {
+    /* will bind against everything, just set port */
+    if (disablev6)
+    {
+      /* V4-only */
+      resi = 1;
+      if (NULL != unixpath)
+        resi++;
+      i = 0;
+      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      if (NULL != unixpath)
+      {
+        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        i++;
+      }
+      saddrlens[i] = sizeof (struct sockaddr_in);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[i];
+#endif
+      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
+      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+    }
+    else
+    {
+      /* dual stack */
+      resi = 2;
+      if (NULL != unixpath)
+        resi++;
+      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      i = 0;
+      if (NULL != unixpath)
+      {
+        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        i++;
+      }
+      saddrlens[i] = sizeof (struct sockaddr_in6);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_len = saddrlens[0];
+#endif
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_family = AF_INET6;
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
+      i++;
+      saddrlens[i] = sizeof (struct sockaddr_in);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[1];
+#endif
+      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
+      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+    }
+  }
+  GNUNET_free_non_null (unixpath);
+  *addrs = saddrs;
+  *addr_lens = saddrlens;
+  return resi;
+}
+
+
+#ifdef MINGW
+/**
+ * Read listen sockets from the parent process (ARM).
+ *
+ * @param sctx service context to initialize
+ * @return #GNUNET_YES if ok, #GNUNET_NO if not ok (must bind yourself),
+ * and #GNUNET_SYSERR on error.
+ */
+static int
+receive_sockets_from_parent (struct LEGACY_SERVICE_Context *sctx)
+{
+  const char *env_buf;
+  int fail;
+  uint64_t count;
+  uint64_t i;
+  HANDLE lsocks_pipe;
+
+  env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
+  if ((NULL == env_buf) || (strlen (env_buf) <= 0))
+    return GNUNET_NO;
+  /* Using W32 API directly here, because this pipe will
+   * never be used outside of this function, and it's just too much of a bother
+   * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
+   */
+  lsocks_pipe = (HANDLE) strtoul (env_buf, NULL, 10);
+  if ( (0 == lsocks_pipe) || (INVALID_HANDLE_VALUE == lsocks_pipe))
+    return GNUNET_NO;
+  fail = 1;
+  do
+  {
+    int ret;
+    int fail2;
+    DWORD rd;
+
+    ret = ReadFile (lsocks_pipe, &count, sizeof (count), &rd, NULL);
+    if ((0 == ret) || (sizeof (count) != rd) || (0 == count))
+      break;
+    sctx->lsocks =
+        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (count + 1));
+
+    fail2 = 1;
+    for (i = 0; i < count; i++)
+    {
+      WSAPROTOCOL_INFOA pi;
+      uint64_t size;
+      SOCKET s;
+
+      ret = ReadFile (lsocks_pipe, &size, sizeof (size), &rd, NULL);
+      if ( (0 == ret) || (sizeof (size) != rd) || (sizeof (pi) != size) )
+        break;
+      ret = ReadFile (lsocks_pipe, &pi, sizeof (pi), &rd, NULL);
+      if ( (0 == ret) || (sizeof (pi) != rd))
+        break;
+      s = WSASocketA (pi.iAddressFamily, pi.iSocketType, pi.iProtocol, &pi, 0, WSA_FLAG_OVERLAPPED);
+      sctx->lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
+      if (NULL == sctx->lsocks[i])
+        break;
+      else if (i == count - 1)
+        fail2 = 0;
+    }
+    if (fail2)
+      break;
+    sctx->lsocks[count] = NULL;
+    fail = 0;
+  }
+  while (fail);
+
+  CloseHandle (lsocks_pipe);
+
+  if (fail)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Could not access a pre-bound socket, will try to bind myself\n"));
+    for (i = 0; (i < count) && (NULL != sctx->lsocks[i]); i++)
+      GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[i]));
+    GNUNET_free_non_null (sctx->lsocks);
+    sctx->lsocks = NULL;
+    return GNUNET_NO;
+  }
+  return GNUNET_YES;
+}
+#endif
+
+
+/**
+ * Setup addr, addrlen, idle_timeout
+ * based on configuration!
+ *
+ * Configuration may specify:
+ * - PORT (where to bind to for TCP)
+ * - UNIXPATH (where to bind to for UNIX domain sockets)
+ * - TIMEOUT (after how many ms does an inactive service timeout);
+ * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
+ * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
+ * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
+ * - ACCEPT_FROM6 (only allow connections from specified IPv6 subnets)
+ * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
+ * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
+ *
+ * @param sctx service context to initialize
+ * @return #GNUNET_OK if configuration succeeded
+ */
+static int
+setup_service (struct LEGACY_SERVICE_Context *sctx)
+{
+  struct GNUNET_TIME_Relative idleout;
+  int tolerant;
+
+#ifndef MINGW
+  const char *nfds;
+  unsigned int cnt;
+  int flags;
+#endif
+
+  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, "TIMEOUT"))
+  {
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->service_name,
+                                             "TIMEOUT", &idleout))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Specified value for `%s' of service `%s' is invalid\n"),
+           "TIMEOUT", sctx->service_name);
+      return GNUNET_SYSERR;
+    }
+    sctx->timeout = idleout;
+  }
+  else
+    sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+
+  if (GNUNET_CONFIGURATION_have_value
+      (sctx->cfg, sctx->service_name, "TOLERANT"))
+  {
+    if (GNUNET_SYSERR ==
+        (tolerant =
+         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                               "TOLERANT")))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Specified value for `%s' of service `%s' is invalid\n"),
+           "TOLERANT", sctx->service_name);
+      return GNUNET_SYSERR;
+    }
+  }
+  else
+    tolerant = GNUNET_NO;
+
+#ifndef MINGW
+  errno = 0;
+  if ((NULL != (nfds = getenv ("LISTEN_FDS"))) &&
+      (1 == SSCANF (nfds, "%u", &cnt)) && (cnt > 0) && (cnt < FD_SETSIZE) &&
+      (cnt + 4 < FD_SETSIZE))
+  {
+    sctx->lsocks =
+        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (cnt + 1));
+    while (0 < cnt--)
+    {
+      flags = fcntl (3 + cnt, F_GETFD);
+      if ((flags < 0) || (0 != (flags & FD_CLOEXEC)) ||
+          (NULL ==
+           (sctx->lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
+      {
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             _
+             ("Could not access pre-bound socket %u, will try to bind myself\n"),
+             (unsigned int) 3 + cnt);
+        cnt++;
+        while (sctx->lsocks[cnt] != NULL)
+          GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[cnt++]));
+        GNUNET_free (sctx->lsocks);
+        sctx->lsocks = NULL;
+        break;
+      }
+    }
+    unsetenv ("LISTEN_FDS");
+  }
+#else
+  if (getenv ("GNUNET_OS_READ_LSOCKS") != NULL)
+  {
+    receive_sockets_from_parent (sctx);
+    putenv ("GNUNET_OS_READ_LSOCKS=");
+  }
+#endif
+
+  if ((NULL == sctx->lsocks) &&
+      (GNUNET_SYSERR ==
+       LEGACY_SERVICE_get_server_addresses (sctx->service_name, sctx->cfg,
+                                            &sctx->addrs, &sctx->addrlens)))
+    return GNUNET_SYSERR;
+  sctx->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
+  sctx->match_uid =
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                            "UNIX_MATCH_UID");
+  sctx->match_gid =
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                            "UNIX_MATCH_GID");
+  process_acl4 (&sctx->v4_denied, sctx, "REJECT_FROM");
+  process_acl4 (&sctx->v4_allowed, sctx, "ACCEPT_FROM");
+  process_acl6 (&sctx->v6_denied, sctx, "REJECT_FROM6");
+  process_acl6 (&sctx->v6_allowed, sctx, "ACCEPT_FROM6");
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Get the name of the user that'll be used
+ * to provide the service.
+ *
+ * @param sctx service context
+ * @return value of the 'USERNAME' option
+ */
+static char *
+get_user_name (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *un;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
+                                               "USERNAME", &un))
+    return NULL;
+  return un;
+}
+
+
+/**
+ * Write PID file.
+ *
+ * @param sctx service context
+ * @param pid PID to write (should be equal to 'getpid()'
+ * @return  #GNUNET_OK on success (including no work to be done)
+ */
+static int
+write_pid_file (struct LEGACY_SERVICE_Context *sctx, pid_t pid)
+{
+  FILE *pidfd;
+  char *pif;
+  char *user;
+  char *rdir;
+  int len;
+
+  if (NULL == (pif = get_pid_file_name (sctx)))
+    return GNUNET_OK;           /* no file desired */
+  user = get_user_name (sctx);
+  rdir = GNUNET_strdup (pif);
+  len = strlen (rdir);
+  while ((len > 0) && (rdir[len] != DIR_SEPARATOR))
+    len--;
+  rdir[len] = '\0';
+  if (0 != ACCESS (rdir, F_OK))
+  {
+    /* we get to create a directory -- and claim it
+     * as ours! */
+    (void) GNUNET_DISK_directory_create (rdir);
+    if ((NULL != user) && (0 < strlen (user)))
+      GNUNET_DISK_file_change_owner (rdir, user);
+  }
+  if (0 != ACCESS (rdir, W_OK | X_OK))
+  {
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "access", rdir);
+    GNUNET_free (rdir);
+    GNUNET_free_non_null (user);
+    GNUNET_free (pif);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (rdir);
+  pidfd = FOPEN (pif, "w");
+  if (NULL == pidfd)
+  {
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "fopen", pif);
+    GNUNET_free (pif);
+    GNUNET_free_non_null (user);
+    return GNUNET_SYSERR;
+  }
+  if (0 > FPRINTF (pidfd, "%u", pid))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "fprintf", pif);
+  GNUNET_break (0 == FCLOSE (pidfd));
+  if ((NULL != user) && (0 < strlen (user)))
+    GNUNET_DISK_file_change_owner (pif, user);
+  GNUNET_free_non_null (user);
+  GNUNET_free (pif);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Task run during shutdown.  Stops the server/service.
+ *
+ * @param cls the `struct LEGACY_SERVICE_Context`
+ */
+static void
+shutdown_task (void *cls)
+{
+  struct LEGACY_SERVICE_Context *service = cls;
+  struct GNUNET_SERVER_Handle *server = service->server;
+
+  service->shutdown_task = NULL;
+  if (0 != (service->options & LEGACY_SERVICE_OPTION_SOFT_SHUTDOWN))
+    GNUNET_SERVER_stop_listening (server);
+  else
+    GNUNET_SERVER_destroy (server);
+}
+
+
+/**
+ * Initial task for the service.
+ *
+ * @param cls service context
+ */
+static void
+service_task (void *cls)
+{
+  struct LEGACY_SERVICE_Context *sctx = cls;
+  unsigned int i;
+
+  GNUNET_RESOLVER_connect (sctx->cfg);
+  if (NULL != sctx->lsocks)
+    sctx->server
+      = GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
+                                           sctx->timeout, sctx->require_found);
+  else
+    sctx->server
+      = GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
+                              sctx->timeout, sctx->require_found);
+  if (NULL == sctx->server)
+  {
+    if (NULL != sctx->addrs)
+      for (i = 0; NULL != sctx->addrs[i]; i++)
+        LOG (GNUNET_ERROR_TYPE_INFO,
+             _("Failed to start `%s' at `%s'\n"),
+             sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+    sctx->ret = GNUNET_SYSERR;
+    return;
+  }
+#ifndef WINDOWS
+  if (NULL != sctx->addrs)
+    for (i = 0; NULL != sctx->addrs[i]; i++)
+      if ((AF_UNIX == sctx->addrs[i]->sa_family)
+          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
+        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
+                                     sctx->match_uid,
+                                     sctx->match_gid);
+#endif
+
+
+  if (0 == (sctx->options & LEGACY_SERVICE_OPTION_MANUAL_SHUTDOWN))
+  {
+    /* install a task that will kill the server
+     * process if the scheduler ever gets a shutdown signal */
+    sctx->shutdown_task = GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                                         sctx);
+  }
+  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
+  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
+  i = 0;
+  while (NULL != sctx->my_handlers[i].callback)
+    sctx->my_handlers[i++].callback_cls = sctx;
+  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
+  if (-1 != sctx->ready_confirm_fd)
+  {
+    GNUNET_break (1 == WRITE (sctx->ready_confirm_fd, ".", 1));
+    GNUNET_break (0 == CLOSE (sctx->ready_confirm_fd));
+    sctx->ready_confirm_fd = -1;
+    write_pid_file (sctx, getpid ());
+  }
+  if (NULL != sctx->addrs)
+  {
+    i = 0;
+    while (NULL != sctx->addrs[i])
+    {
+      LOG (GNUNET_ERROR_TYPE_INFO, _("Service `%s' runs at %s\n"),
+           sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+      i++;
+    }
+  }
+  sctx->task (sctx->task_cls, sctx->server, sctx->cfg);
+}
+
+
+/**
+ * Detach from terminal.
+ *
+ * @param sctx service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+detach_terminal (struct LEGACY_SERVICE_Context *sctx)
+{
+#ifndef MINGW
+  pid_t pid;
+  int nullfd;
+  int filedes[2];
+
+  if (0 != PIPE (filedes))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "pipe");
+    return GNUNET_SYSERR;
+  }
+  pid = fork ();
+  if (pid < 0)
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "fork");
+    return GNUNET_SYSERR;
+  }
+  if (0 != pid)
+  {
+    /* Parent */
+    char c;
+
+    GNUNET_break (0 == CLOSE (filedes[1]));
+    c = 'X';
+    if (1 != READ (filedes[0], &c, sizeof (char)))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "read");
+    fflush (stdout);
+    switch (c)
+    {
+    case '.':
+      exit (0);
+    case 'I':
+      LOG (GNUNET_ERROR_TYPE_INFO, _("Service process failed to initialize\n"));
+      break;
+    case 'S':
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process could not initialize server function\n"));
+      break;
+    case 'X':
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process failed to report status\n"));
+      break;
+    }
+    exit (1);                   /* child reported error */
+  }
+  GNUNET_break (0 == CLOSE (0));
+  GNUNET_break (0 == CLOSE (1));
+  GNUNET_break (0 == CLOSE (filedes[0]));
+  nullfd = OPEN ("/dev/null", O_RDWR | O_APPEND);
+  if (nullfd < 0)
+    return GNUNET_SYSERR;
+  /* set stdin/stdout to /dev/null */
+  if ((dup2 (nullfd, 0) < 0) || (dup2 (nullfd, 1) < 0))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "dup2");
+    (void) CLOSE (nullfd);
+    return GNUNET_SYSERR;
+  }
+  (void) CLOSE (nullfd);
+  /* Detach from controlling terminal */
+  pid = setsid ();
+  if (-1 == pid)
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "setsid");
+  sctx->ready_confirm_fd = filedes[1];
+#else
+  /* FIXME: we probably need to do something else
+   * elsewhere in order to fork the process itself... */
+  FreeConsole ();
+#endif
+  return GNUNET_OK;
+}
+
+
+/**
+ * Set user ID.
+ *
+ * @param sctx service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+set_user_id (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *user;
+
+  if (NULL == (user = get_user_name (sctx)))
+    return GNUNET_OK;           /* keep */
+#ifndef MINGW
+  struct passwd *pws;
+
+  errno = 0;
+  pws = getpwnam (user);
+  if (NULL == pws)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Cannot obtain information about user `%s': %s\n"), user,
+         errno == 0 ? _("No such user") : STRERROR (errno));
+    GNUNET_free (user);
+    return GNUNET_SYSERR;
+  }
+  if ((0 != setgid (pws->pw_gid)) || (0 != setegid (pws->pw_gid)) ||
+#if HAVE_INITGROUPS
+      (0 != initgroups (user, pws->pw_gid)) ||
+#endif
+      (0 != setuid (pws->pw_uid)) || (0 != seteuid (pws->pw_uid)))
+  {
+    if ((0 != setregid (pws->pw_gid, pws->pw_gid)) ||
+        (0 != setreuid (pws->pw_uid, pws->pw_uid)))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR, _("Cannot change user/group to `%s': %s\n"),
+           user, STRERROR (errno));
+      GNUNET_free (user);
+      return GNUNET_SYSERR;
+    }
+  }
+#endif
+  GNUNET_free (user);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Delete the PID file that was created by our parent.
+ *
+ * @param sctx service context
+ */
+static void
+pid_file_delete (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *pif = get_pid_file_name (sctx);
+
+  if (NULL == pif)
+    return;                     /* no PID file */
+  if (0 != UNLINK (pif))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", pif);
+  GNUNET_free (pif);
+}
+
+
+/**
+ * Run a standard GNUnet service startup sequence (initialize loggers
+ * and configuration, parse options).
+ *
+ * @param argc number of command line arguments
+ * @param argv command line arguments
+ * @param service_name our service name
+ * @param options service options
+ * @param task main task of the service
+ * @param task_cls closure for @a task
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK
+ *         if we shutdown nicely
+ */
+int
+LEGACY_SERVICE_run (int argc, char *const *argv,
+                    const char *service_name,
+                    enum LEGACY_SERVICE_Options options,
+                    LEGACY_SERVICE_Main task,
+                    void *task_cls)
+{
+#define HANDLE_ERROR do { GNUNET_break (0); goto shutdown; } while (0)
+
+  int err;
+  int ret;
+  char *cfg_fn;
+  char *opt_cfg_fn;
+  char *loglev;
+  char *logfile;
+  int do_daemonize;
+  unsigned int i;
+  unsigned long long skew_offset;
+  unsigned long long skew_variance;
+  long long clock_offset;
+  struct LEGACY_SERVICE_Context sctx;
+  struct GNUNET_CONFIGURATION_Handle *cfg;
+  const char *xdg;
+
+  struct GNUNET_GETOPT_CommandLineOption service_options[] = {
+    GNUNET_GETOPT_option_cfgfile (&opt_cfg_fn),
+    GNUNET_GETOPT_option_flag ('d',
+                                  "daemonize",
+                                  gettext_noop ("do daemonize (detach from terminal)"),
+                                  &do_daemonize),
+    GNUNET_GETOPT_option_help (NULL),
+    GNUNET_GETOPT_option_loglevel (&loglev),
+    GNUNET_GETOPT_option_logfile (&logfile),
+    GNUNET_GETOPT_option_version (PACKAGE_VERSION " " VCS_VERSION),
+    GNUNET_GETOPT_OPTION_END
+  };
+  err = 1;
+  do_daemonize = 0;
+  logfile = NULL;
+  loglev = NULL;
+  opt_cfg_fn = NULL;
+  xdg = getenv ("XDG_CONFIG_HOME");
+  if (NULL != xdg)
+    GNUNET_asprintf (&cfg_fn,
+                     "%s%s%s",
+                     xdg,
+                     DIR_SEPARATOR_STR,
+                     GNUNET_OS_project_data_get ()->config_file);
+  else
+    cfg_fn = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
+  memset (&sctx, 0, sizeof (sctx));
+  sctx.options = options;
+  sctx.ready_confirm_fd = -1;
+  sctx.ret = GNUNET_OK;
+  sctx.timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+  sctx.task = task;
+  sctx.task_cls = task_cls;
+  sctx.service_name = service_name;
+  sctx.cfg = cfg = GNUNET_CONFIGURATION_create ();
+
+  /* setup subsystems */
+  ret = GNUNET_GETOPT_run (service_name, service_options, argc, argv);
+  if (GNUNET_SYSERR == ret)
+    goto shutdown;
+  if (GNUNET_NO == ret)
+  {
+    err = 0;
+    goto shutdown;
+  }
+  if (GNUNET_OK != GNUNET_log_setup (service_name, loglev, logfile))
+    HANDLE_ERROR;
+  if (NULL == opt_cfg_fn)
+    opt_cfg_fn = GNUNET_strdup (cfg_fn);
+  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_fn))
+  {
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, opt_cfg_fn))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Malformed configuration file `%s', exit ...\n"),
+                  opt_cfg_fn);
+      goto shutdown;
+    }
+  }
+  else
+  {
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, NULL))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Malformed configuration, exit ...\n"));
+      goto shutdown;
+    }
+    if (0 != strcmp (opt_cfg_fn, cfg_fn))
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Could not access configuration file `%s'\n"),
+                  opt_cfg_fn);
+  }
+  if (GNUNET_OK != setup_service (&sctx))
+    goto shutdown;
+  if ((1 == do_daemonize) && (GNUNET_OK != detach_terminal (&sctx)))
+    HANDLE_ERROR;
+  if (GNUNET_OK != set_user_id (&sctx))
+    goto shutdown;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Service `%s' runs with configuration from `%s'\n",
+       service_name,
+       opt_cfg_fn);
+  if ((GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
+                                              "SKEW_OFFSET", &skew_offset)) &&
+      (GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
+                                              "SKEW_VARIANCE", &skew_variance)))
+  {
+    clock_offset = skew_offset - skew_variance;
+    GNUNET_TIME_set_offset (clock_offset);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "Skewing clock by %dll ms\n", clock_offset);
+  }
+  /* actually run service */
+  err = 0;
+  GNUNET_SCHEDULER_run (&service_task, &sctx);
+  /* shutdown */
+  if ((1 == do_daemonize) && (NULL != sctx.server))
+    pid_file_delete (&sctx);
+  GNUNET_free_non_null (sctx.my_handlers);
+
+shutdown:
+  if (-1 != sctx.ready_confirm_fd)
+  {
+    if (1 != WRITE (sctx.ready_confirm_fd, err ? "I" : "S", 1))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "write");
+    GNUNET_break (0 == CLOSE (sctx.ready_confirm_fd));
+  }
+#if HAVE_MALLINFO
+  {
+    char *counter;
+
+    if ( (GNUNET_YES ==
+          GNUNET_CONFIGURATION_have_value (sctx.cfg, service_name,
+                                           "GAUGER_HEAP")) &&
+         (GNUNET_OK ==
+          GNUNET_CONFIGURATION_get_value_string (sctx.cfg, service_name,
+                                                 "GAUGER_HEAP",
+                                                 &counter)) )
+    {
+      struct mallinfo mi;
+
+      mi = mallinfo ();
+      GAUGER (service_name, counter, mi.usmblks, "blocks");
+      GNUNET_free (counter);
+    }
+  }
+#endif
+  GNUNET_CONFIGURATION_destroy (cfg);
+  i = 0;
+  if (NULL != sctx.addrs)
+    while (NULL != sctx.addrs[i])
+      GNUNET_free (sctx.addrs[i++]);
+  GNUNET_free_non_null (sctx.addrs);
+  GNUNET_free_non_null (sctx.addrlens);
+  GNUNET_free_non_null (logfile);
+  GNUNET_free_non_null (loglev);
+  GNUNET_free (cfg_fn);
+  GNUNET_free_non_null (opt_cfg_fn);
+  GNUNET_free_non_null (sctx.v4_denied);
+  GNUNET_free_non_null (sctx.v6_denied);
+  GNUNET_free_non_null (sctx.v4_allowed);
+  GNUNET_free_non_null (sctx.v6_allowed);
+
+  return err ? GNUNET_SYSERR : sctx.ret;
+}
+
+
+/**
+ * Run a service startup sequence within an existing
+ * initialized system.
+ *
+ * @param service_name our service name
+ * @param cfg configuration to use
+ * @param options service options
+ * @return NULL on error, service handle
+ */
+struct LEGACY_SERVICE_Context *
+LEGACY_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      enum LEGACY_SERVICE_Options options)
+{
+  int i;
+  struct LEGACY_SERVICE_Context *sctx;
+
+  sctx = GNUNET_new (struct LEGACY_SERVICE_Context);
+  sctx->ready_confirm_fd = -1;  /* no daemonizing */
+  sctx->ret = GNUNET_OK;
+  sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+  sctx->service_name = service_name;
+  sctx->cfg = cfg;
+  sctx->options = options;
+
+  /* setup subsystems */
+  if (GNUNET_OK != setup_service (sctx))
+  {
+    LEGACY_SERVICE_stop (sctx);
+    return NULL;
+  }
+  if (NULL != sctx->lsocks)
+    sctx->server =
+        GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
+                                           sctx->timeout, sctx->require_found);
+  else
+    sctx->server =
+        GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
+                              sctx->timeout, sctx->require_found);
+
+  if (NULL == sctx->server)
+  {
+    LEGACY_SERVICE_stop (sctx);
+    return NULL;
+  }
+#ifndef WINDOWS
+  if (NULL != sctx->addrs)
+    for (i = 0; NULL != sctx->addrs[i]; i++)
+      if ((AF_UNIX == sctx->addrs[i]->sa_family)
+          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
+        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
+                                     sctx->match_uid,
+                                     sctx->match_gid);
+#endif
+  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
+  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
+  i = 0;
+  while ((sctx->my_handlers[i].callback != NULL))
+    sctx->my_handlers[i++].callback_cls = sctx;
+  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
+  return sctx;
+}
+
+
+/**
+ * Obtain the server used by a service.  Note that the server must NOT
+ * be destroyed by the caller.
+ *
+ * @param ctx the service context returned from the start function
+ * @return handle to the server for this service, NULL if there is none
+ */
+struct GNUNET_SERVER_Handle *
+LEGACY_SERVICE_get_server (struct LEGACY_SERVICE_Context *ctx)
+{
+  return ctx->server;
+}
+
+
+/**
+ * Get the NULL-terminated array of listen sockets for this service.
+ *
+ * @param ctx service context to query
+ * @return NULL if there are no listen sockets, otherwise NULL-terminated
+ *              array of listen sockets.
+ */
+struct GNUNET_NETWORK_Handle *const*
+LEGACY_SERVICE_get_listen_sockets (struct LEGACY_SERVICE_Context *ctx)
+{
+  return ctx->lsocks;
+}
+
+
+/**
+ * Stop a service that was started with "LEGACY_SERVICE_start".
+ *
+ * @param sctx the service context returned from the start function
+ */
+void
+LEGACY_SERVICE_stop (struct LEGACY_SERVICE_Context *sctx)
+{
+  unsigned int i;
+
+#if HAVE_MALLINFO
+  {
+    char *counter;
+
+    if ( (GNUNET_YES ==
+          GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name,
+                                           "GAUGER_HEAP")) &&
+         (GNUNET_OK ==
+          GNUNET_CONFIGURATION_get_value_string (sctx->cfg, sctx->service_name,
+                                                 "GAUGER_HEAP",
+                                                 &counter)) )
+    {
+      struct mallinfo mi;
+
+      mi = mallinfo ();
+      GAUGER (sctx->service_name, counter, mi.usmblks, "blocks");
+      GNUNET_free (counter);
+    }
+  }
+#endif
+  if (NULL != sctx->shutdown_task)
+  {
+    GNUNET_SCHEDULER_cancel (sctx->shutdown_task);
+    sctx->shutdown_task = NULL;
+  }
+  if (NULL != sctx->server)
+    GNUNET_SERVER_destroy (sctx->server);
+  GNUNET_free_non_null (sctx->my_handlers);
+  if (NULL != sctx->addrs)
+  {
+    i = 0;
+    while (NULL != sctx->addrs[i])
+      GNUNET_free (sctx->addrs[i++]);
+    GNUNET_free (sctx->addrs);
+  }
+  GNUNET_free_non_null (sctx->addrlens);
+  GNUNET_free_non_null (sctx->v4_denied);
+  GNUNET_free_non_null (sctx->v6_denied);
+  GNUNET_free_non_null (sctx->v4_allowed);
+  GNUNET_free_non_null (sctx->v6_allowed);
+  GNUNET_free (sctx);
+}
+
+
+/* end of service.c */
diff --git a/src/transport/test_plugin_transport.c b/src/transport/test_plugin_transport.c
index be79d5499..1d92588ea 100644
--- a/src/transport/test_plugin_transport.c
+++ b/src/transport/test_plugin_transport.c
@@ -552,7 +552,7 @@ setup_plugin_environment ()
 
 
 static int
-handle_helper_message (void *cls, void *client,
+handle_helper_message (void *cls,
                        const struct GNUNET_MessageHeader *hdr)
 {
   return GNUNET_OK;
diff --git a/src/transport/test_transport_address_switch.c b/src/transport/test_transport_address_switch.c
index 77bc9aef8..e076d3501 100644
--- a/src/transport/test_transport_address_switch.c
+++ b/src/transport/test_transport_address_switch.c
@@ -49,9 +49,9 @@
 
 
 /**
- * Testcase timeout
+ * Testcase timeout (set aggressively as we know this test doesn't work right now)
  */
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
 
 
 static struct GNUNET_TRANSPORT_TESTING_ConnectCheckContext *ccc;
@@ -199,7 +199,7 @@ custom_shutdown (void *cls)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Fail (timeout)! No transmission after switch! Stopping peers\n");
-    ccc->global_ret = GNUNET_SYSERR;
+    ccc->global_ret = 77; /* GNUNET_SYSERR; */
   }
 
   /* stop statistics */
@@ -277,8 +277,11 @@ custom_shutdown (void *cls)
     GNUNET_break (0);
     result++;
   }
+#if 0
+  /* This test is not really expected to pass right now... */
   if (0 != result)
     ccc->global_ret = GNUNET_SYSERR;
+#endif
 }
 
 
@@ -298,7 +301,7 @@ notify_receive (void *cls,
                 "Peer %u (`%s') got message %u of size %u from peer (`%s')\n",
                 receiver->no,
                 ps,
-                ntohl (hdr->num),
+                (uint32_t) ntohl (hdr->num),
                 ntohs (hdr->header.size),
                 GNUNET_i2s (sender));
     GNUNET_free (ps);
diff --git a/src/transport/test_transport_api_reliability.c b/src/transport/test_transport_api_reliability.c
index d6702cc25..e5ba2831b 100644
--- a/src/transport/test_transport_api_reliability.c
+++ b/src/transport/test_transport_api_reliability.c
@@ -91,7 +91,7 @@ get_size (unsigned int iter)
 #ifndef LINUX
   /* FreeBSD/OSX etc. Unix DGRAMs do not work
    * with large messages */
-  if (0 == strcmp ("unix", test_plugin))
+  if (0 == strcmp ("unix", ccc->test_plugin))
     ret = sizeof (struct GNUNET_TRANSPORT_TESTING_TestMessage) + (ret % 1024);
 #endif
   ret = sizeof (struct GNUNET_TRANSPORT_TESTING_TestMessage) + (ret % 60000);
@@ -217,7 +217,7 @@ notify_receive (void *cls,
 {
   static int n;
   unsigned int s;
-  char cbuf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
+  char cbuf[GNUNET_MAX_MESSAGE_SIZE - 1];
 
   if (GNUNET_TRANSPORT_TESTING_SIMPLE_MTYPE != ntohs (hdr->header.type))
     return;
@@ -228,10 +228,10 @@ notify_receive (void *cls,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Expected message %u of size %u, got %u bytes of message %u\n",
-                ntohl (hdr->num),
+                (uint32_t) ntohl (hdr->num),
                 s,
                 ntohs (hdr->header.size),
-                ntohl (hdr->num));
+                (uint32_t) ntohl (hdr->num));
     ccc->global_ret = GNUNET_SYSERR;
     GNUNET_SCHEDULER_shutdown ();
     return;
@@ -247,7 +247,7 @@ notify_receive (void *cls,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Expected message %u with bits %u, but body did not match\n",
-                ntohl (hdr->num),
+                (uint32_t) ntohl (hdr->num),
                 (unsigned char) ntohl (hdr->num));
     ccc->global_ret = GNUNET_SYSERR;
     GNUNET_SCHEDULER_shutdown ();
@@ -258,7 +258,7 @@ notify_receive (void *cls,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Got message %u of size %u\n",
-                ntohl (hdr->num),
+                (uint32_t) ntohl (hdr->num),
                 ntohs (hdr->header.size));
   }
 #endif
@@ -267,7 +267,7 @@ notify_receive (void *cls,
   {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Message id %u is bigger than maxmimum number of messages %u expected\n",
-                  ntohl (hdr->num),
+                  (uint32_t) ntohl (hdr->num),
                   TOTAL_MSGS / xhdr);
   }
   if (0 == (n % (TOTAL_MSGS / xhdr / 100)))
diff --git a/src/transport/transport-testing-loggers.c b/src/transport/transport-testing-loggers.c
index de9fa91c1..0ebb07d74 100644
--- a/src/transport/transport-testing-loggers.c
+++ b/src/transport/transport-testing-loggers.c
@@ -71,6 +71,7 @@ GNUNET_TRANSPORT_TESTING_log_disconnect (void *cls,
               GNUNET_i2s (other),
               me->no,
               ps);
+  GNUNET_free (ps);
 }
 
 /* end of transport-testing-loggers.c */
diff --git a/src/transport/transport-testing-main.c b/src/transport/transport-testing-main.c
index a79d81cb9..81a66e113 100644
--- a/src/transport/transport-testing-main.c
+++ b/src/transport/transport-testing-main.c
@@ -146,7 +146,7 @@ struct GNUNET_TRANSPORT_TESTING_InternalPeerContext
 
 /**
  * Information tracked per connected peer.
- */ 
+ */
 struct ConnectPairInfo
 {
   /**
@@ -235,7 +235,7 @@ my_nc (void *cls,
   struct GNUNET_TRANSPORT_TESTING_InternalPeerContext *ipi = cls;
   struct GNUNET_TRANSPORT_TESTING_ConnectCheckContext *ccc = ipi->ccc;
   struct ConnectPairInfo *cpi;
-  
+
   if (NULL != ccc->nc)
     ccc->nc (ccc->cls,
              ccc->p[ipi->off],
@@ -262,7 +262,7 @@ my_nd (void *cls,
   struct GNUNET_TRANSPORT_TESTING_InternalPeerContext *ipi = cls;
   struct GNUNET_TRANSPORT_TESTING_ConnectCheckContext *ccc = ipi->ccc;
   struct ConnectPairInfo *cpi = custom_cls;
-  
+
   if (NULL != ccc->nd)
     ccc->nd (ccc->cls,
              ccc->p[ipi->off],
@@ -535,13 +535,15 @@ GNUNET_TRANSPORT_TESTING_connect_check (void *cls,
     ip[i].off = i;
     ip[i].ccc = ccc;
   }
-  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
-                      argv,
-                      test_name_,
-                      "nohelp",
-                      options,
-                      &connect_check_run,
-                      ccc);
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
+                          argv,
+                          test_name_,
+                          "nohelp",
+                          options,
+                          &connect_check_run,
+                          ccc))
+    return GNUNET_SYSERR;
   return ccc->global_ret;
 }
 
diff --git a/src/transport/transport-testing.c b/src/transport/transport-testing.c
index c0775a135..53a44f338 100644
--- a/src/transport/transport-testing.c
+++ b/src/transport/transport-testing.c
@@ -49,7 +49,7 @@ find_peer_context (struct GNUNET_TRANSPORT_TESTING_Handle *tth,
  *
  * @param p1 first peer
  * @param p2 second peer
- * @param cb function to call 
+ * @param cb function to call
  * @param cb_cls closure for @a cb
  */
 void
@@ -66,7 +66,7 @@ GNUNET_TRANSPORT_TESTING_find_connecting_context (struct GNUNET_TRANSPORT_TESTIN
   {
     ccn = cc->next;
     if ( (cc->p1 == p1) &&
-         (cc->p2 == p2) ) 
+         (cc->p2 == p2) )
       cb (cb_cls,
           cc);
   }
@@ -74,7 +74,7 @@ GNUNET_TRANSPORT_TESTING_find_connecting_context (struct GNUNET_TRANSPORT_TESTIN
 
 
 static void
-set_p1c (void *cls,   
+set_p1c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -86,7 +86,7 @@ set_p1c (void *cls,
 
 
 static void
-set_mq (void *cls,   
+set_mq (void *cls,
         struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   struct GNUNET_MQ_Handle *mq = cls;
@@ -96,7 +96,7 @@ set_mq (void *cls,
 
 
 static void
-set_p2c (void *cls,   
+set_p2c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -108,7 +108,7 @@ set_p2c (void *cls,
 
 
 static void
-clear_p1c (void *cls,   
+clear_p1c (void *cls,
            struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -120,7 +120,7 @@ clear_p1c (void *cls,
 
 
 static void
-clear_p2c (void *cls,   
+clear_p2c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -154,7 +154,7 @@ notify_connect (void *cls,
   else
     ret = NULL;
 
-  if (p2 != NULL)
+  if (NULL != p2)
     GNUNET_asprintf (&p2_s,
                      "%u (`%s')",
                      p2->no,
@@ -267,7 +267,7 @@ notify_disconnect (void *cls,
   int no = 0;
   struct GNUNET_TRANSPORT_TESTING_PeerContext *p2 = NULL;
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cc;
-  
+
   p2 = find_peer_context (p->tth,
                           peer);
   no = p->no;
@@ -386,7 +386,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth
   struct GNUNET_TRANSPORT_TESTING_PeerContext *p;
   struct GNUNET_PeerIdentity *dummy;
   unsigned int i;
-  
+
   if (GNUNET_NO == GNUNET_DISK_file_test (cfgname))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
@@ -407,7 +407,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth
     GNUNET_memcpy (p->handlers,
                    handlers,
                    i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }    
+  }
   if (NULL != cb_cls)
     p->cb_cls = cb_cls;
   else
@@ -532,7 +532,7 @@ GNUNET_TRANSPORT_TESTING_restart_peer (struct GNUNET_TRANSPORT_TESTING_PeerConte
 {
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cc;
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *ccn;
- 
+
   /* shutdown */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Stopping peer %u (`%s')\n",
@@ -770,7 +770,7 @@ GNUNET_TRANSPORT_TESTING_connect_peers (struct GNUNET_TRANSPORT_TESTING_PeerCont
       break;
     }
   }
-          
+
   cc = GNUNET_new (struct GNUNET_TRANSPORT_TESTING_ConnectRequest);
   cc->p1 = p1;
   cc->p2 = p2;
diff --git a/src/transport/transport.conf.in b/src/transport/transport.conf.in
index 7b5413bbe..2c99af000 100644
--- a/src/transport/transport.conf.in
+++ b/src/transport/transport.conf.in
@@ -4,6 +4,8 @@ AUTOSTART = @AUTOSTART@
 HOSTNAME = localhost
 BINARY = gnunet-service-transport
 # PREFIX = valgrind
+
+# Maximum number of neighbours PER PLUGIN (not in total).
 NEIGHBOUR_LIMIT = 50
 ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
@@ -65,8 +67,6 @@ MAX_CONNECTIONS = 128
 # Enable TCP stealth?
 TCP_STEALTH = NO
 
-# Configuration for manually punched holes in NAT.
-# HOLE_EXTERNAL = auto:2086
 
 [transport-udp]
 # Use PORT = 0 to autodetect a port available
@@ -110,6 +110,7 @@ TESTING_IGNORE_KEYS = ACCEPT_FROM;
 # Experimental, default: NO
 # PROXY_HTTP_TUNNELING = NO
 
+
 [transport-http_server]
 #EXTERNAL_HOSTNAME = <your hostname/path>
 PORT = 1080
@@ -122,8 +123,7 @@ ADVERTISED_PORT = 1080
 # Can use IPv6 addresses ([fefc::]:PORT).
 # Use "AUTO" for the hostname to automatically detect external IP.
 # Do not set if NAT is not manually punched.
-# HOLE_EXTERNAL = AUTO:2086
-
+# HOLE_EXTERNAL = AUTO:1080
 
 MAX_CONNECTIONS = 128
 TESTING_IGNORE_KEYS = ACCEPT_FROM;
@@ -131,8 +131,6 @@ TESTING_IGNORE_KEYS = ACCEPT_FROM;
 # Enable TCP stealth?
 TCP_STEALTH = NO
 
-# Configuration for manually punched holes in NAT.
-# HOLE_EXTERNAL = auto:2086
 
 [transport-https_client]
 MAX_CONNECTIONS = 128
@@ -155,7 +153,6 @@ TESTING_IGNORE_KEYS = ACCEPT_FROM;
 # PROXY_HTTP_TUNNELING = NO
 
 
-
 [transport-https_server]
 # EXTERNAL_HOSTNAME = <your hostname/path>
 # EXTERNAL_HOSTNAME_ONLY = YES
@@ -174,7 +171,7 @@ ADVERTISED_PORT = 4433
 # Can use IPv6 addresses ([fefc::]:PORT).
 # Use "AUTO" for the hostname to automatically detect external IP.
 # Do not set if NAT is not manually punched.
-# HOLE_EXTERNAL = AUTO:2086
+# HOLE_EXTERNAL = AUTO:4433
 
 CRYPTO_INIT = NORMAL
 KEY_FILE = $GNUNET_DATA_HOME/transport/https.key
@@ -185,9 +182,6 @@ TESTING_IGNORE_KEYS = ACCEPT_FROM;
 # Enable TCP stealth?
 TCP_STEALTH = NO
 
-# Configuration for manually punched holes in NAT.
-# HOLE_EXTERNAL = auto:2086
-
 
 [transport-wlan]
 # Name of the interface in monitor mode (typically monX)
@@ -196,6 +190,7 @@ INTERFACE = mon0
 TESTMODE = 0
 TESTING_IGNORE_KEYS = ACCEPT_FROM;
 
+
 [transport-bluetooth]
 # Name of the interface (typically hciX)
 INTERFACE = hci0
diff --git a/src/transport/transport_api_address_to_string.c b/src/transport/transport_api_address_to_string.c
index b9c72dcb3..902764a8f 100644
--- a/src/transport/transport_api_address_to_string.c
+++ b/src/transport/transport_api_address_to_string.c
@@ -199,10 +199,10 @@ GNUNET_TRANSPORT_address_to_string (const struct GNUNET_CONFIGURATION_Handle *cf
 
   alen = address->address_length;
   slen = strlen (address->transport_name) + 1;
-  if ( (alen + slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE
+  if ( (alen + slen >= GNUNET_MAX_MESSAGE_SIZE
         - sizeof (struct AddressLookupMessage)) ||
-       (alen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) )
+       (alen >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen >= GNUNET_MAX_MESSAGE_SIZE) )
   {
     GNUNET_break (0);
     GNUNET_free (alc);
diff --git a/src/transport/transport_api_core.c b/src/transport/transport_api_core.c
index a693cb889..9c29d4908 100644
--- a/src/transport/transport_api_core.c
+++ b/src/transport/transport_api_core.c
@@ -425,7 +425,7 @@ mq_send_impl (struct GNUNET_MQ_Handle *mq,
 
   GNUNET_assert (GNUNET_YES == n->is_ready);
   msize = ntohs (msg->size);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*obm))
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE - sizeof (*obm))
   {
     GNUNET_break (0);
     GNUNET_MQ_impl_send_continue (mq);
@@ -447,7 +447,8 @@ mq_send_impl (struct GNUNET_MQ_Handle *mq,
   GNUNET_MQ_send (h->mq,
                   n->env);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Queued message for neighbour `%s'.\n",
+       "Queued message of type %u for neighbour `%s'.\n",
+       ntohs (msg->type),
        GNUNET_i2s (&n->id));
 }
 
diff --git a/src/tun/Makefile.am b/src/tun/Makefile.am
index d8eb2767e..c741f5654 100644
--- a/src/tun/Makefile.am
+++ b/src/tun/Makefile.am
@@ -28,7 +28,7 @@ check_PROGRAMS = \
  test_regex
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
diff --git a/src/util/.gitignore b/src/util/.gitignore
index f207e07bf..3576a2134 100644
--- a/src/util/.gitignore
+++ b/src/util/.gitignore
@@ -22,6 +22,7 @@ test_connection_timeout.nc
 test_connection_timeout_no_connect.nc
 test_connection_transmit_cancel.nc
 test_container_bloomfilter
+test_container_dll
 test_container_heap
 test_container_meta_data
 test_container_multihashmap
@@ -63,3 +64,6 @@ test_strings
 test_strings_to_data
 test_time
 test_socks.nc
+perf_crypto_asymmetric
+perf_crypto_hash
+perf_crypto_symmetric
diff --git a/src/util/Makefile.am b/src/util/Makefile.am
index f49aee17f..9be572bb6 100644
--- a/src/util/Makefile.am
+++ b/src/util/Makefile.am
@@ -30,7 +30,9 @@ W32CONSOLEHELPER = gnunet-helper-w32-console
 endif
 
 if !MINGW
- SERVER_CLIENT_UNIX = test_server_with_client_unix
+ TEST_CLIENT_UNIX_NC = test_client_unix.nc
+else
+ TEST_CLIENT_UNIX_NC =
 endif
 
 if USE_COVERAGE
@@ -65,7 +67,6 @@ libgnunetutil_la_SOURCES = \
   common_logging.c \
   configuration.c \
   configuration_loader.c \
-  connection.c \
   container_bloomfilter.c \
   container_heap.c \
   container_meta_data.c \
@@ -105,16 +106,10 @@ libgnunetutil_la_SOURCES = \
   program.c \
   resolver_api.c resolver.h \
   scheduler.c \
-  server.c \
-  server_mst.c \
-  server_nc.c \
-  server_tc.c \
   service.c \
-  service_new.c \
   signal.c \
   strings.c \
   time.c \
-  socks.c \
   speedup.c speedup.h
 
 libgnunetutil_la_LIBADD = \
@@ -187,7 +182,7 @@ noinst_PROGRAMS = \
 
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = $(check_PROGRAMS)
 endif
 
@@ -260,19 +255,19 @@ if HAVE_BENCHMARKS
 endif
 
 if HAVE_SSH_KEY
- SSH_USING_TESTS = test_socks.nc
+# SSH_USING_TESTS = test_socks.nc
 endif
 
 check_PROGRAMS = \
  test_bio \
  test_client.nc \
- test_client_unix.nc \
- $(SSH_USING_TESTS) \
+ $(TEST_CLIENT_UNIX_NC) \
  test_common_allocation \
  test_common_endian \
  test_common_logging \
  test_configuration \
  test_container_bloomfilter \
+ test_container_dll \
  test_container_meta_data \
  test_container_multihashmap \
  test_container_multihashmap32 \
@@ -294,12 +289,6 @@ check_PROGRAMS = \
  test_crypto_rsa \
  test_disk \
  test_getopt \
- test_connection.nc \
- test_connection_addressing.nc \
- test_connection_receive_cancel.nc \
- test_connection_timeout.nc \
- test_connection_timeout_no_connect.nc \
- test_connection_transmit_cancel.nc \
  test_mq \
  test_os_network \
  test_peer \
@@ -308,11 +297,6 @@ check_PROGRAMS = \
  test_resolver_api.nc \
  test_scheduler \
  test_scheduler_delay \
- test_server.nc \
- test_server_disconnect.nc \
- test_server_with_client.nc \
- test_server_mst_interrupt.nc \
- $(SERVER_CLIENT_UNIX) \
  test_service \
  test_strings \
  test_strings_to_data \
@@ -326,18 +310,7 @@ check_PROGRAMS = \
 # Declare .nc (NO-CONCURRENCY) as a test extension so that we can impart
 # sequential execution order for them
 TEST_EXTENSIONS = .nc
-test_connection.log: test_client.log
-test_connection_addressing.log: test_connection.log
-test_connection_timeout_no_connect.log: test_connection_addressing.log
-test_connection_transmit_cancel.log: test_connection_timeout_no_connect.log
-test_connection_receive_cancel.log: test_connection_transmit_cancel.log
-test_connection_timeout.log: test_connection_receive_cancel.log
-test_resolver_api.log: test_connection_timeout.log
-test_server.log: test_resolver_api.log
-test_server_disconnect.log: test_server.log
-test_server_with_client.log: test_server_disconnect.log
-test_server_mst_interrupt.log: test_server_with_client.log
-test_client_unix.log: test_server_mst_interrupt.log
+test_test_client_unix.log: test_client.log
 
 test_bio_SOURCES = \
  test_bio.c
@@ -398,10 +371,15 @@ test_container_bloomfilter_SOURCES = \
 test_container_bloomfilter_LDADD = \
  libgnunetutil.la
 
+test_container_dll_SOURCES = \
+ test_container_dll.c
+test_container_dll_LDADD = \
+ libgnunetutil.la
+
 test_container_meta_data_SOURCES = \
  test_container_meta_data.c
 test_container_meta_data_LDADD = \
- libgnunetutil.la -lextractor
+ libgnunetutil.la
 
 test_container_multihashmap_SOURCES = \
  test_container_multihashmap.c
@@ -509,36 +487,6 @@ test_getopt_SOURCES = \
 test_getopt_LDADD = \
  libgnunetutil.la
 
-test_connection_nc_SOURCES = \
- test_connection.c
-test_connection_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_addressing_nc_SOURCES = \
- test_connection_addressing.c
-test_connection_addressing_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_receive_cancel_nc_SOURCES = \
- test_connection_receive_cancel.c
-test_connection_receive_cancel_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_timeout_nc_SOURCES = \
- test_connection_timeout.c
-test_connection_timeout_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_timeout_no_connect_nc_SOURCES = \
- test_connection_timeout_no_connect.c
-test_connection_timeout_no_connect_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_transmit_cancel_nc_SOURCES = \
- test_connection_transmit_cancel.c
-test_connection_transmit_cancel_nc_LDADD = \
- libgnunetutil.la
-
 test_mq_SOURCES = \
  test_mq.c
 test_mq_LDADD = \
@@ -579,32 +527,6 @@ test_scheduler_delay_SOURCES = \
 test_scheduler_delay_LDADD = \
  libgnunetutil.la
 
-test_server_mst_interrupt_nc_SOURCES = \
- test_server_mst_interrupt.c
-test_server_mst_interrupt_nc_LDADD = \
- libgnunetutil.la
-
-test_server_nc_SOURCES = \
- test_server.c
-test_server_nc_LDADD = \
- libgnunetutil.la
-
-test_server_disconnect_nc_SOURCES = \
- test_server_disconnect.c
-test_server_disconnect_nc_LDADD = \
- libgnunetutil.la
-
-test_server_with_client_nc_SOURCES = \
- test_server_with_client.c
-test_server_with_client_nc_LDADD = \
- libgnunetutil.la
-
-test_server_with_client_unix_SOURCES = \
- test_server_with_client_unix.c
-test_server_with_client_unix_LDADD = \
- libgnunetutil.la
-
-
 test_service_SOURCES = \
  test_service.c
 test_service_LDADD = \
@@ -615,7 +537,6 @@ test_strings_SOURCES = \
 test_strings_LDADD = \
  libgnunetutil.la
 
-
 test_strings_to_data_SOURCES = \
  test_strings_to_data.c
 test_strings_to_data_LDADD = \
diff --git a/src/util/bandwidth.c b/src/util/bandwidth.c
index a059fc738..bc0c3b9b4 100644
--- a/src/util/bandwidth.c
+++ b/src/util/bandwidth.c
@@ -184,8 +184,8 @@ update_excess (struct GNUNET_BANDWIDTH_Tracker *av)
   }
   /* negative current_consumption means that we have savings */
   max_carry = ((uint64_t) av->available_bytes_per_s__) * av->max_carry_s__;
-  if (max_carry < GNUNET_SERVER_MAX_MESSAGE_SIZE)
-    max_carry = GNUNET_SERVER_MAX_MESSAGE_SIZE;
+  if (max_carry < GNUNET_MAX_MESSAGE_SIZE)
+    max_carry = GNUNET_MAX_MESSAGE_SIZE;
   if (max_carry > INT64_MAX)
     max_carry = INT64_MAX;
   left_bytes = current_consumption + max_carry;
@@ -224,10 +224,10 @@ update_excess (struct GNUNET_BANDWIDTH_Tracker *av)
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * To stop notifications about updates and excess callbacks use
@@ -271,10 +271,10 @@ GNUNET_BANDWIDTH_tracker_init2 (struct GNUNET_BANDWIDTH_Tracker *av,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
@@ -345,8 +345,8 @@ update_tracker (struct GNUNET_BANDWIDTH_Tracker *av)
     left_bytes = - av->consumption_since_last_update__;
     max_carry = ((unsigned long long) av->available_bytes_per_s__) *
                 av->max_carry_s__;
-    if (max_carry < GNUNET_SERVER_MAX_MESSAGE_SIZE)
-      max_carry = GNUNET_SERVER_MAX_MESSAGE_SIZE;
+    if (max_carry < GNUNET_MAX_MESSAGE_SIZE)
+      max_carry = GNUNET_MAX_MESSAGE_SIZE;
     if (max_carry > INT64_MAX)
       max_carry = INT64_MAX;
     if (max_carry > left_bytes)
diff --git a/src/util/bio.c b/src/util/bio.c
index 62f4904f7..08e30dc01 100644
--- a/src/util/bio.c
+++ b/src/util/bio.c
@@ -25,7 +25,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util",__VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-bio",__VA_ARGS__)
 
 #ifndef PATH_MAX
 /**
diff --git a/src/util/client.c b/src/util/client.c
index 4fd971040..3d74bff33 100644
--- a/src/util/client.c
+++ b/src/util/client.c
@@ -33,7 +33,16 @@
 #include "gnunet_socks.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util",__VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-client",__VA_ARGS__)
+
+/**
+ * Timeout we use on TCP connect before trying another
+ * result from the DNS resolver.  Actual value used
+ * is this value divided by the number of address families.
+ * Default is 5s.
+ */
+#define CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
+
 
 
 /**
@@ -298,6 +307,11 @@ recv_message (void *cls,
 
   if (GNUNET_YES == cstate->in_destroy)
     return GNUNET_SYSERR;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received message of type %u and size %u from %s\n",
+       ntohs (msg->type),
+       ntohs (msg->size),
+       cstate->service_name);
   GNUNET_MQ_inject_message (cstate->mq,
                             msg);
   if (GNUNET_YES == cstate->in_destroy)
@@ -491,8 +505,8 @@ try_unixpath (const char *service_name,
         s_un.sun_path[0] = '\0';
     }
 #endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
-    un.sun_len = (u_char) sizeof (struct sockaddr_un);
+#if HAVE_SOCKADDR_UN_SUN_LEN
+    s_un.sun_len = (u_char) sizeof (struct sockaddr_un);
 #endif
     sock = GNUNET_NETWORK_socket_create (AF_UNIX,
                                          SOCK_STREAM,
@@ -510,6 +524,8 @@ try_unixpath (const char *service_name,
       GNUNET_free (unixpath);
       return sock;
     }
+    if (NULL != sock)
+      GNUNET_NETWORK_socket_close (sock);
   }
   GNUNET_free_non_null (unixpath);
 #endif
@@ -649,7 +665,7 @@ try_connect_using_address (void *cls,
   GNUNET_CONTAINER_DLL_insert (cstate->ap_head,
                                cstate->ap_tail,
                                ap);
-  ap->task = GNUNET_SCHEDULER_add_write_net (GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+  ap->task = GNUNET_SCHEDULER_add_write_net (CONNECT_RETRY_TIMEOUT,
                                              ap->sock,
                                              &connect_probe_continuation,
                                              ap);
@@ -753,7 +769,7 @@ start_connect (void *cls)
   cstate->dns_active
     = GNUNET_RESOLVER_ip_get (cstate->hostname,
                               AF_UNSPEC,
-                              GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+                              CONNECT_RETRY_TIMEOUT,
                               &try_connect_using_address,
                               cstate);
 }
@@ -877,4 +893,4 @@ GNUNET_CLIENT_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
   return cstate->mq;
 }
 
-/* end of client_new.c */
+/* end of client.c */
diff --git a/src/util/common_allocation.c b/src/util/common_allocation.c
index 71a2221ee..737584815 100644
--- a/src/util/common_allocation.c
+++ b/src/util/common_allocation.c
@@ -32,9 +32,9 @@
 #include <malloc/malloc.h>
 #endif
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util",__VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-common-allocation",__VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-common-allocation", syscall)
 
 #ifndef INT_MAX
 #define INT_MAX 0x7FFFFFFF
diff --git a/src/util/common_endian.c b/src/util/common_endian.c
index 4c8ad7182..f29e42c98 100644
--- a/src/util/common_endian.c
+++ b/src/util/common_endian.c
@@ -28,7 +28,7 @@
 #include "platform.h"
 #include "gnunet_crypto_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util",__VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-common-endian",__VA_ARGS__)
 
 
 uint64_t
diff --git a/src/util/configuration_loader.c b/src/util/configuration_loader.c
index 07eeb98f0..ceaf2a6ea 100644
--- a/src/util/configuration_loader.c
+++ b/src/util/configuration_loader.c
@@ -27,7 +27,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-configuration", __VA_ARGS__)
 
 
 /**
diff --git a/src/util/container_bloomfilter.c b/src/util/container_bloomfilter.c
index 58725dcc5..aedca232d 100644
--- a/src/util/container_bloomfilter.c
+++ b/src/util/container_bloomfilter.c
@@ -42,11 +42,11 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-bloomfilter", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-container-bloomfilter", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-container-bloomfilter", syscall, filename)
 
 struct GNUNET_CONTAINER_BloomFilter
 {
diff --git a/src/util/container_heap.c b/src/util/container_heap.c
index 1ead5ec6d..21bdee834 100644
--- a/src/util/container_heap.c
+++ b/src/util/container_heap.c
@@ -28,7 +28,7 @@
 #include "platform.h"
 #include "gnunet_container_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-heap", __VA_ARGS__)
 
 #define EXTRA_CHECKS 0
 
diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c
index b4d231dae..ec527005a 100644
--- a/src/util/container_meta_data.c
+++ b/src/util/container_meta_data.c
@@ -31,7 +31,7 @@
 #endif
 #include <zlib.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-meta-data", __VA_ARGS__)
 
 
 
diff --git a/src/util/container_multihashmap.c b/src/util/container_multihashmap.c
index d85562579..ffeb4a71f 100644
--- a/src/util/container_multihashmap.c
+++ b/src/util/container_multihashmap.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_container_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-multihashmap", __VA_ARGS__)
 
 /**
  * An entry in the hash map with the full key.
diff --git a/src/util/container_multihashmap32.c b/src/util/container_multihashmap32.c
index 4cc9b7ebd..d33c3c2d9 100644
--- a/src/util/container_multihashmap32.c
+++ b/src/util/container_multihashmap32.c
@@ -28,7 +28,7 @@
 #include "platform.h"
 #include "gnunet_container_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-multihashmap32", __VA_ARGS__)
 
 /**
  * An entry in the hash map.
diff --git a/src/util/container_multipeermap.c b/src/util/container_multipeermap.c
index 6c62e7403..7830771d8 100644
--- a/src/util/container_multipeermap.c
+++ b/src/util/container_multipeermap.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-multipeermap", __VA_ARGS__)
 
 /**
  * An entry in the hash map with the full key.
diff --git a/src/util/container_multishortmap.c b/src/util/container_multishortmap.c
index 5e8a47b09..cfa82ca20 100644
--- a/src/util/container_multishortmap.c
+++ b/src/util/container_multishortmap.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-container-multishortmap", __VA_ARGS__)
 
 /**
  * An entry in the hash map with the full key.
diff --git a/src/util/crypto_crc.c b/src/util/crypto_crc.c
index d7f5f7fc8..03b24d9f7 100644
--- a/src/util/crypto_crc.c
+++ b/src/util/crypto_crc.c
@@ -30,7 +30,7 @@
 #include "platform.h"
 #include "gnunet_crypto_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-crc", __VA_ARGS__)
 
 /* Avoid wasting space on 8-byte longs. */
 #if UINT_MAX >= 0xffffffff
diff --git a/src/util/crypto_ecc.c b/src/util/crypto_ecc.c
index 4bba395b3..eaa49a991 100644
--- a/src/util/crypto_ecc.c
+++ b/src/util/crypto_ecc.c
@@ -38,11 +38,11 @@
  */
 #define CURVE "Ed25519"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-ecc", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-crypto-ecc", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-crypto-ecc", syscall, filename)
 
 /**
  * Log an error message at log-level 'level' that indicates
@@ -488,6 +488,28 @@ struct GNUNET_CRYPTO_EcdhePrivateKey *
 GNUNET_CRYPTO_ecdhe_key_create ()
 {
   struct GNUNET_CRYPTO_EcdhePrivateKey *priv;
+
+  priv = GNUNET_new (struct GNUNET_CRYPTO_EcdhePrivateKey);
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_ecdhe_key_create2 (priv))
+  {
+    GNUNET_free (priv);
+    return NULL;
+  }
+  return priv;
+}
+
+
+/**
+ * @ingroup crypto
+ * Create a new private key.  Clear with #GNUNET_CRYPTO_ecdhe_key_clear().
+ *
+ * @param[out] pk set to fresh private key;
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+int
+GNUNET_CRYPTO_ecdhe_key_create2 (struct GNUNET_CRYPTO_EcdhePrivateKey *pk)
+{
   gcry_sexp_t priv_sexp;
   gcry_sexp_t s_keyparam;
   gcry_mpi_t d;
@@ -503,13 +525,13 @@ GNUNET_CRYPTO_ecdhe_key_create ()
                                   "(flags eddsa no-keytest)))")))
   {
     LOG_GCRY (GNUNET_ERROR_TYPE_ERROR, "gcry_sexp_build", rc);
-    return NULL;
+    return GNUNET_SYSERR;
   }
   if (0 != (rc = gcry_pk_genkey (&priv_sexp, s_keyparam)))
   {
     LOG_GCRY (GNUNET_ERROR_TYPE_ERROR, "gcry_pk_genkey", rc);
     gcry_sexp_release (s_keyparam);
-    return NULL;
+    return GNUNET_SYSERR;
   }
   gcry_sexp_release (s_keyparam);
 #if EXTRA_CHECKS
@@ -517,20 +539,19 @@ GNUNET_CRYPTO_ecdhe_key_create ()
   {
     LOG_GCRY (GNUNET_ERROR_TYPE_ERROR, "gcry_pk_testkey", rc);
     gcry_sexp_release (priv_sexp);
-    return NULL;
+    return GNUNET_SYSERR;
   }
 #endif
   if (0 != (rc = key_from_sexp (&d, priv_sexp, "private-key", "d")))
   {
     LOG_GCRY (GNUNET_ERROR_TYPE_ERROR, "key_from_sexp", rc);
     gcry_sexp_release (priv_sexp);
-    return NULL;
+    return GNUNET_SYSERR;
   }
   gcry_sexp_release (priv_sexp);
-  priv = GNUNET_new (struct GNUNET_CRYPTO_EcdhePrivateKey);
-  GNUNET_CRYPTO_mpi_print_unsigned (priv->d, sizeof (priv->d), d);
+  GNUNET_CRYPTO_mpi_print_unsigned (pk->d, sizeof (pk->d), d);
   gcry_mpi_release (d);
-  return priv;
+  return GNUNET_OK;
 }
 
 
diff --git a/src/util/crypto_ecc_setup.c b/src/util/crypto_ecc_setup.c
index 0ce55ddbf..2f2e2f122 100644
--- a/src/util/crypto_ecc_setup.c
+++ b/src/util/crypto_ecc_setup.c
@@ -27,11 +27,11 @@
 #include <gcrypt.h>
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-ecc", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-crypto-ecc", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-crypto-ecc", syscall, filename)
 
 /**
  * Log an error message at log-level 'level' that indicates
diff --git a/src/util/crypto_hash.c b/src/util/crypto_hash.c
index 31824e72a..49dbacd0b 100644
--- a/src/util/crypto_hash.c
+++ b/src/util/crypto_hash.c
@@ -28,9 +28,9 @@
 #include "gnunet_strings_lib.h"
 #include <gcrypt.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-hash", __VA_ARGS__)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-crypto-hash", syscall, filename)
 
 /**
  * Hash block of given size.
diff --git a/src/util/crypto_hash_file.c b/src/util/crypto_hash_file.c
index ace5212c9..3e5900200 100644
--- a/src/util/crypto_hash_file.c
+++ b/src/util/crypto_hash_file.c
@@ -27,9 +27,9 @@
 #include "gnunet_util_lib.h"
 #include <gcrypt.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-hash-file", __VA_ARGS__)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-crypto-hash-file", syscall, filename)
 
 
 /**
diff --git a/src/util/crypto_hkdf.c b/src/util/crypto_hkdf.c
index c6c43f800..f04d3e675 100644
--- a/src/util/crypto_hkdf.c
+++ b/src/util/crypto_hkdf.c
@@ -36,7 +36,7 @@
  * - Matthias Wachs (08.10.2010)
  */
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-hkdf", __VA_ARGS__)
 
 /**
  * Set this to 0 if you compile this code outside of GNUnet.
diff --git a/src/util/crypto_kdf.c b/src/util/crypto_kdf.c
index 78fb1911a..6d7c5a096 100644
--- a/src/util/crypto_kdf.c
+++ b/src/util/crypto_kdf.c
@@ -30,7 +30,7 @@
 #include "platform.h"
 #include "gnunet_crypto_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-kdf", __VA_ARGS__)
 
 /**
  * @brief Derive key
diff --git a/src/util/crypto_mpi.c b/src/util/crypto_mpi.c
index 668d5e602..ff3e9a8a7 100644
--- a/src/util/crypto_mpi.c
+++ b/src/util/crypto_mpi.c
@@ -29,7 +29,7 @@
 #include "gnunet_crypto_lib.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-mpi", __VA_ARGS__)
 
 /**
  * Log an error message at log-level 'level' that indicates
diff --git a/src/util/crypto_random.c b/src/util/crypto_random.c
index 8897ae24f..d5b5eb9ec 100644
--- a/src/util/crypto_random.c
+++ b/src/util/crypto_random.c
@@ -28,9 +28,9 @@
 #include "gnunet_crypto_lib.h"
 #include <gcrypt.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-random", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-crypto-random", syscall)
 
 
 /* TODO: ndurner, move this to plibc? */
diff --git a/src/util/crypto_rsa.c b/src/util/crypto_rsa.c
index 443d597e4..7a108c21b 100644
--- a/src/util/crypto_rsa.c
+++ b/src/util/crypto_rsa.c
@@ -25,7 +25,7 @@
 #include <gcrypt.h>
 #include "gnunet_crypto_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-rsa", __VA_ARGS__)
 
 
 /**
@@ -430,7 +430,7 @@ rsa_blinding_key_derive (const struct GNUNET_CRYPTO_RsaPublicKey *pkey,
   char *xts = "Blinding KDF extrator HMAC key";  /* Trusts bks' randomness more */
   struct RsaBlindingKey *blind;
   gcry_mpi_t n;
- 
+
   blind = GNUNET_new (struct RsaBlindingKey);
   GNUNET_assert( NULL != blind );
 
@@ -454,25 +454,25 @@ rsa_blinding_key_derive (const struct GNUNET_CRYPTO_RsaPublicKey *pkey,
 }
 
 
-/* 
+/*
 We originally added GNUNET_CRYPTO_kdf_mod_mpi for the benifit of the
 previous routine.
 
-There was previously a call to GNUNET_CRYPTO_kdf in 
+There was previously a call to GNUNET_CRYPTO_kdf in
   bkey = rsa_blinding_key_derive (len, bks);
-that gives exactly len bits where 
+that gives exactly len bits where
   len = GNUNET_CRYPTO_rsa_public_key_len (pkey);
 
 Now r = 2^(len-1)/pkey.n is the probability that a set high bit being
 okay, meaning bkey < pkey.n.  It follows that (1-r)/2 of the time bkey >
-pkey.n making the effective bkey be 
+pkey.n making the effective bkey be
   bkey mod pkey.n = bkey - pkey.n
 so the effective bkey has its high bit set with probability r/2.
 
 We expect r to be close to 1/2 if the exchange is honest, but the
 exchange can choose r otherwise.
 
-In blind signing, the exchange sees  
+In blind signing, the exchange sees
   B = bkey * S mod pkey.n
 On deposit, the exchange sees S so they can compute bkey' = B/S mod
 pkey.n for all B they recorded to see if bkey' has it's high bit set.
@@ -489,7 +489,7 @@ the wrong and right probabilities 1/3 and 1/4, respectively.
 I feared this gives the exchange a meaningful fraction of a bit of
 information per coin involved in the transaction.  It sounds damaging if
 numerous coins were involved.  And it could run across transactions in
-some scenarios. 
+some scenarios.
 
 We fixed this by using a more uniform deterministic pseudo-random number
 generator for blinding factors.  I do not believe this to be a problem
@@ -748,7 +748,7 @@ GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash,
   }
 
   data = rsa_full_domain_hash (pkey, hash);
-  if (NULL == data) 
+  if (NULL == data)
     goto rsa_gcd_validate_failure;
 
   bkey = rsa_blinding_key_derive (pkey, bks);
@@ -771,7 +771,7 @@ GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash,
   gcry_mpi_release (ne[0]);
   gcry_mpi_release (ne[1]);
   gcry_mpi_release (r_e);
-  rsa_blinding_key_free (bkey);  
+  rsa_blinding_key_free (bkey);
 
   *buf_size = numeric_mpi_alloc_n_print (data_r_e, buf);
   gcry_mpi_release (data_r_e);
@@ -917,7 +917,7 @@ GNUNET_CRYPTO_rsa_sign_fdh (const struct GNUNET_CRYPTO_RsaPrivateKey *key,
   GNUNET_CRYPTO_rsa_public_key_free (pkey);
   if (NULL == v)   /* rsa_gcd_validate failed meaning */
     return NULL;   /* our *own* RSA key is malicious. */
- 
+
   sig = rsa_sign_mpi (key, v);
   gcry_mpi_release (v);
   return sig;
@@ -1077,11 +1077,11 @@ GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig,
   }
 
   bkey = rsa_blinding_key_derive (pkey, bks);
-  if (NULL == bkey) 
+  if (NULL == bkey)
   {
-    /* RSA key is malicious since rsa_gcd_validate failed here. 
+    /* RSA key is malicious since rsa_gcd_validate failed here.
      * It should have failed during GNUNET_CRYPTO_rsa_blind too though,
-     * so the exchange is being malicious in an unfamilair way, maybe 
+     * so the exchange is being malicious in an unfamilair way, maybe
      * just trying to crash us.  */
     GNUNET_break_op (0);
     gcry_mpi_release (n);
@@ -1096,10 +1096,10 @@ GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig,
                      n))
   {
     /* We cannot find r mod n, so gcd(r,n) != 1, which should get *
-     * caught above, but we handle it the same here.              */  
+     * caught above, but we handle it the same here.              */
     GNUNET_break_op (0);
     gcry_mpi_release (r_inv);
-    rsa_blinding_key_free (bkey);  
+    rsa_blinding_key_free (bkey);
     gcry_mpi_release (n);
     gcry_mpi_release (s);
     return NULL;
@@ -1144,11 +1144,11 @@ GNUNET_CRYPTO_rsa_verify (const struct GNUNET_HashCode *hash,
   r = rsa_full_domain_hash (pkey, hash);
   if (NULL == r) {
     GNUNET_break_op (0);
-    /* RSA key is malicious since rsa_gcd_validate failed here. 
+    /* RSA key is malicious since rsa_gcd_validate failed here.
      * It should have failed during GNUNET_CRYPTO_rsa_blind too though,
-     * so the exchange is being malicious in an unfamilair way, maybe 
+     * so the exchange is being malicious in an unfamilair way, maybe
      * just trying to crash us.  Arguably, we've only an internal error
-     * though because we should've detected this in our previous call 
+     * though because we should've detected this in our previous call
      * to GNUNET_CRYPTO_rsa_unblind. */
     return GNUNET_NO;
   }
diff --git a/src/util/crypto_symmetric.c b/src/util/crypto_symmetric.c
index 381a5d2f8..e25e2f1dd 100644
--- a/src/util/crypto_symmetric.c
+++ b/src/util/crypto_symmetric.c
@@ -29,7 +29,7 @@
 #include "gnunet_crypto_lib.h"
 #include <gcrypt.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-crypto-symmetric", __VA_ARGS__)
 
 /**
  * Create a new SessionKey (for symmetric encryption).
diff --git a/src/util/disk.c b/src/util/disk.c
index 40043549b..d536ec897 100644
--- a/src/util/disk.c
+++ b/src/util/disk.c
@@ -28,11 +28,11 @@
 #include "gnunet_strings_lib.h"
 #include "gnunet_disk_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-disk", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-disk", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-disk", syscall, filename)
 
 /**
  * Block size for IO for copying files.
@@ -329,8 +329,10 @@ GNUNET_DISK_file_get_identifiers (const char *filename, uint64_t * dev,
     BY_HANDLE_FILE_INFORMATION info;
     int succ;
 
-    fh = GNUNET_DISK_file_open (filename, GNUNET_DISK_OPEN_READ, 0);
-    if (fh == NULL)
+    fh = GNUNET_DISK_file_open (filename,
+                                GNUNET_DISK_OPEN_READ,
+                                GNUNET_DISK_PERM_NONE);
+    if (NULL == fh)
       return GNUNET_SYSERR;
     succ = GetFileInformationByHandle (fh->h, &info);
     GNUNET_DISK_file_close (fh);
@@ -1191,7 +1193,7 @@ GNUNET_DISK_fn_write (const char *fn,
   fh = GNUNET_DISK_file_open (fn,
                               GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE
                               | GNUNET_DISK_OPEN_CREATE, mode);
-  if (!fh)
+  if (! fh)
     return GNUNET_SYSERR;
   ret = GNUNET_DISK_file_write (fh, buffer, n);
   GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh));
@@ -1756,9 +1758,10 @@ GNUNET_DISK_file_open (const char *fn,
 
 
 /**
- * Close an open file
+ * Close an open file.
+ *
  * @param h file handle
- * @return GNUNET_OK on success, GNUNET_SYSERR otherwise
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise
  */
 int
 GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
@@ -1773,7 +1776,7 @@ GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
   ret = GNUNET_OK;
 
 #if MINGW
-  if (!CloseHandle (h->h))
+  if (! CloseHandle (h->h))
   {
     SetErrnoFromWinError (GetLastError ());
     LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "close");
@@ -1781,7 +1784,7 @@ GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
   }
   if (h->oOverlapRead)
   {
-    if (!CloseHandle (h->oOverlapRead->hEvent))
+    if (! CloseHandle (h->oOverlapRead->hEvent))
     {
       SetErrnoFromWinError (GetLastError ());
       LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "close");
@@ -1822,7 +1825,6 @@ struct GNUNET_DISK_FileHandle *
 GNUNET_DISK_get_handle_from_w32_handle (HANDLE osfh)
 {
   struct GNUNET_DISK_FileHandle *fh;
-
   DWORD dwret;
   enum GNUNET_FILE_Type ftype;
 
@@ -1836,10 +1838,13 @@ GNUNET_DISK_get_handle_from_w32_handle (HANDLE osfh)
     ftype = GNUNET_DISK_HANLDE_TYPE_PIPE;
     break;
   case FILE_TYPE_UNKNOWN:
-    if (GetLastError () == NO_ERROR || GetLastError () == ERROR_INVALID_HANDLE)
+    if ( (GetLastError () == NO_ERROR) ||
+         (GetLastError () == ERROR_INVALID_HANDLE) )
     {
       if (0 != ResetEvent (osfh))
         ftype = GNUNET_DISK_HANLDE_TYPE_EVENT;
+      else
+        return NULL;
     }
     else
       return NULL;
diff --git a/src/util/getopt.c b/src/util/getopt.c
index e150496ae..036e0f4be 100644
--- a/src/util/getopt.c
+++ b/src/util/getopt.c
@@ -26,7 +26,7 @@ USA.
 
 
 This code was heavily modified for GNUnet.
-Copyright Copyright (C) 2006 Christian Grothoff
+Copyright Copyright (C) 2006, 2017 Christian Grothoff
 */
 
 /**
@@ -47,9 +47,9 @@ Copyright Copyright (C) 2006 Christian Grothoff
 #endif
 #endif
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-getopt", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-getopt", syscall)
 
 #if defined (WIN32) && !defined (__CYGWIN32__)
 /* It's not Unix, really.  See?  Capital letters.  */
@@ -845,9 +845,13 @@ GN_getopt_internal (int argc, char *const *argv, const char *optstring,
   }
 }
 
+
 static int
-GNgetopt_long (int argc, char *const *argv, const char *options,
-               const struct GNoption *long_options, int *opt_index)
+GNgetopt_long (int argc,
+               char *const *argv,
+               const char *options,
+               const struct GNoption *long_options,
+               int *opt_index)
 {
   return GN_getopt_internal (argc, argv, options, long_options, opt_index, 0);
 }
@@ -867,16 +871,17 @@ GNgetopt_long (int argc, char *const *argv, const char *options,
 int
 GNUNET_GETOPT_run (const char *binaryOptions,
                    const struct GNUNET_GETOPT_CommandLineOption *allOptions,
-                   unsigned int argc, char *const *argv)
+                   unsigned int argc,
+                   char *const *argv)
 {
   struct GNoption *long_options;
   struct GNUNET_GETOPT_CommandLineProcessorContext clpc;
   int count;
-  int i;
   char *shorts;
   int spos;
   int cont;
   int c;
+  uint8_t *seen;
 
   GNUNET_assert (argc > 0);
   GNoptind = 0;
@@ -885,13 +890,15 @@ GNUNET_GETOPT_run (const char *binaryOptions,
   clpc.allOptions = allOptions;
   clpc.argv = argv;
   clpc.argc = argc;
-  count = 0;
-  while (allOptions[count].name != NULL)
-    count++;
-  long_options = GNUNET_malloc (sizeof (struct GNoption) * (count + 1));
+  for (count = 0; NULL != allOptions[count].name; count++) ;
+
+  long_options = GNUNET_new_array (count + 1,
+                                   struct GNoption);
+  seen = GNUNET_new_array (count,
+                           uint8_t);
   shorts = GNUNET_malloc (count * 2 + 1);
   spos = 0;
-  for (i = 0; i < count; i++)
+  for (unsigned i = 0; i < count; i++)
   {
     long_options[i].name = allOptions[i].name;
     long_options[i].has_arg = allOptions[i].require_argument;
@@ -907,13 +914,17 @@ GNUNET_GETOPT_run (const char *binaryOptions,
   long_options[count].val = '\0';
   shorts[spos] = '\0';
   cont = GNUNET_OK;
+
   /* main getopt loop */
-  while (cont == GNUNET_OK)
+  while (GNUNET_OK == cont)
   {
     int option_index = 0;
+    unsigned int i;
 
-    c = GNgetopt_long (argc, argv, shorts, long_options, &option_index);
-
+    c = GNgetopt_long (argc, argv,
+                       shorts,
+                       long_options,
+                       &option_index);
     if (c == GNUNET_SYSERR)
       break;                    /* No more flags to process */
 
@@ -922,25 +933,46 @@ GNUNET_GETOPT_run (const char *binaryOptions,
       clpc.currentArgument = GNoptind - 1;
       if ((char) c == allOptions[i].shortName)
       {
-        cont =
-            allOptions[i].processor (&clpc, allOptions[i].scls,
-                                     allOptions[i].name, GNoptarg);
+        cont = allOptions[i].processor (&clpc,
+                                        allOptions[i].scls,
+                                        allOptions[i].name,
+                                        GNoptarg);
+        seen[i] = 1;
         break;
       }
     }
     if (i == count)
     {
-      FPRINTF (stderr, _("Use %s to get a list of options.\n"), "--help");
+      FPRINTF (stderr,
+               _("Use %s to get a list of options.\n"),
+               "--help");
       cont = GNUNET_SYSERR;
     }
   }
-
   GNUNET_free (shorts);
   GNUNET_free (long_options);
-  if (cont != GNUNET_OK)
+
+  if (GNUNET_YES == cont)
   {
-    return cont;
+    for (count = 0; NULL != allOptions[count].name; count++)
+      if ( (0 == seen[count]) &&
+           (allOptions[count].option_mandatory) )
+      {
+        FPRINTF (stderr,
+                 _("Missing mandatory option `%s'.\n"),
+                 allOptions[count].name);
+        cont = GNUNET_SYSERR;
+      }
   }
+  GNUNET_free (seen);
+
+  /* call cleaners, if available */
+  for (count = 0; NULL != allOptions[count].name; count++)
+    if (NULL != allOptions[count].cleaner)
+      allOptions[count].cleaner (allOptions[count].scls);
+
+  if (GNUNET_OK != cont)
+    return cont;
   return GNoptind;
 }
 
diff --git a/src/util/getopt_helpers.c b/src/util/getopt_helpers.c
index ab0b67412..c3d0e4c7c 100644
--- a/src/util/getopt_helpers.c
+++ b/src/util/getopt_helpers.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-getopt", __VA_ARGS__)
 
 
 /**
@@ -38,11 +38,11 @@
  * @param value not used (NULL)
  * @return #GNUNET_NO (do not continue, not an error)
  */
-int
-GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                              void *scls,
-                              const char *option,
-                              const char *value)
+static int
+print_version (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+               void *scls,
+               const char *option,
+               const char *value)
 {
   const char *version = scls;
 
@@ -54,6 +54,26 @@ GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *
 
 
 /**
+ * Define the option to print the version of
+ * the application (-v option)
+ *
+ * @param version string with the version number
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_version (const char *version)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'v',
+    .name = "version",
+    .description = gettext_noop("print the version number"),
+    .processor = &print_version,
+    .scls = (void *) version
+  };
+  return clo;
+}
+
+
+/**
  * At what offset does the help text start?
  */
 #define BORDER 29
@@ -67,11 +87,11 @@ GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *
  * @param value not used (NULL)
  * @return #GNUNET_NO (do not continue, not an error)
  */
-int
-GNUNET_GETOPT_format_help_ (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls,
-                            const char *option,
-                            const char *value)
+static int
+format_help (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+             void *scls,
+             const char *option,
+             const char *value)
 {
   const char *about = scls;
   size_t slen;
@@ -165,6 +185,27 @@ OUTER:
 
 
 /**
+ * Defining the option to print the command line
+ * help text (-h option).
+ *
+ * @param about string with brief description of the application
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_help (const char *about)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'h',
+    .name = "help",
+    .description = gettext_noop("print this help"),
+    .processor = format_help,
+    .scls = (void *) about
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'unsigned int' from the command line. Each
  * time the option flag is given, the value is incremented by one.
  * A pointer to this function should be passed as part of the
@@ -173,17 +214,18 @@ OUTER:
  * type 'int'.
  *
  * @param ctx command line processing context
- * @param scls additional closure (will point to the 'int')
+ * @param scls additional closure (will point to the 'unsigned int')
  * @param option name of the option
  * @param value not used (NULL)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value)
+static int
+increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                 void *scls,
+                 const char *option,
+                 const char *value)
 {
-  int *val = scls;
+  unsigned int *val = scls;
 
   (*val)++;
   return GNUNET_OK;
@@ -191,6 +233,54 @@ GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
 
 
 /**
+ * Increment @a val each time the option flag is given by one.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val increment by 1 each time the option is present
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_increment_uint (char shortName,
+                                      const char *name,
+                                      const char *description,
+                                      unsigned int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .description = description,
+    .processor = &increment_value,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Define the '-V' verbosity option.  Using the option more
+ * than once increments @a level each time.
+ *
+ * @param[out] level set to the verbosity level
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_verbose (unsigned int *level)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'V',
+    .name = "verbose",
+    .description = gettext_noop("be verbose"),
+    .processor = &increment_value,
+    .scls = (void *) level
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'int' from the command line to 1 if the
  * given option is present.
  * A pointer to this function should be passed as part of the
@@ -204,9 +294,11 @@ GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
  * @param value not used (NULL)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                       void *scls, const char *option, const char *value)
+static int
+set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+         void *scls,
+         const char *option,
+         const char *value)
 {
   int *val = scls;
 
@@ -216,6 +308,34 @@ GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
 
 
 /**
+ * Allow user to specify a flag (which internally means setting
+ * an integer to 1/#GNUNET_YES/#GNUNET_OK.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_flag (char shortName,
+                              const char *name,
+                              const char *description,
+                              int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .description = description,
+    .processor = &set_one,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'char *' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -229,31 +349,174 @@ GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
  * @param value actual value of the option (a string)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value)
+static int
+set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+            void *scls,
+            const char *option,
+            const char *value)
 {
   char **val = scls;
 
-  GNUNET_assert (value != NULL);
+  GNUNET_assert (NULL != value);
   GNUNET_free_non_null (*val);
   *val = GNUNET_strdup (value);
   return GNUNET_OK;
 }
 
 
-int
-GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls, const char *option, const char *value)
+/**
+ * Allow user to specify a string.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_string (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_string,
+    .scls = (void *) str
+  };
+
+  return clo;
+}
+
+
+/**
+ * Define the '-L' log level option.  Note that we do not check
+ * that the log level is valid here.
+ *
+ * @param[out] level set to the log level
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_loglevel (char **level)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'L',
+    .name = "log",
+    .argumentHelp = "LOGLEVEL",
+    .description = gettext_noop("configure logging to use LOGLEVEL"),
+    .require_argument = 1,
+    .processor = &set_string,
+    .scls = (void *) level
+  };
+
+  return clo;
+}
+
+
+/**
+ * Set an option of type 'char *' from the command line with
+ * filename expansion a la #GNUNET_STRINGS_filename_expand().
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the `char *`,
+ *             which will be allocated)
+ * @param option name of the option
+ * @param value actual value of the option (a string)
+ * @return #GNUNET_OK
+ */
+static int
+set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+              void *scls,
+              const char *option,
+              const char *value)
 {
   char **val = scls;
 
-  GNUNET_assert (value != NULL);
+  GNUNET_assert (NULL != value);
   GNUNET_free_non_null (*val);
   *val = GNUNET_STRINGS_filename_expand (value);
   return GNUNET_OK;
 }
 
+
+/**
+ * Allow user to specify a filename (automatically path expanded).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_filename (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) str
+  };
+
+  return clo;
+}
+
+
+/**
+ * Allow user to specify log file name (-l option)
+ *
+ * @param[out] logfn set to the name of the logfile
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_logfile (char **logfn)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'l',
+    .name = "logfile",
+    .argumentHelp = "FILENAME",
+    .description = gettext_noop ("configure logging to write logs to FILENAME"),
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) logfn
+  };
+
+  return clo;
+}
+
+
+/**
+ * Allow user to specify configuration file name (-c option)
+ *
+ * @param[out] fn set to the name of the configuration file
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_cfgfile (char **fn)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'c',
+    .name = "config",
+    .argumentHelp = "FILENAME",
+    .description = gettext_noop("use configuration file FILENAME"),
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) fn
+  };
+
+  return clo;
+}
+
+
 /**
  * Set an option of type 'unsigned long long' from the command line.
  * A pointer to this function should be passed as part of the
@@ -267,15 +530,21 @@ GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ct
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                         void *scls, const char *option, const char *value)
+static int
+set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+           void *scls,
+           const char *option,
+           const char *value)
 {
   unsigned long long *val = scls;
 
-  if (1 != SSCANF (value, "%llu", val))
+  if (1 != SSCANF (value,
+                   "%llu",
+                   val))
   {
-    FPRINTF (stderr, _("You must pass a number to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass a number to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -283,6 +552,36 @@ GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
 
 
 /**
+ * Allow user to specify an `unsigned long long`
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_ulong (char shortName,
+                                const char *name,
+                                const char *argumentHelp,
+                                const char *description,
+                                unsigned long long *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_ulong,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'struct GNUNET_TIME_Relative' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -295,9 +594,11 @@ GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                                 void *scls, const char *option, const char *value)
+static int
+set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                   void *scls,
+                   const char *option,
+                   const char *value)
 {
   struct GNUNET_TIME_Relative *val = scls;
 
@@ -305,7 +606,74 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
       GNUNET_STRINGS_fancy_time_to_relative (value,
                                              val))
   {
-    FPRINTF (stderr, _("You must pass relative time to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass relative time to the `%s' option.\n"),
+             option);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify a `struct GNUNET_TIME_Relative`
+ * (using human-readable "fancy" time).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_relative_time (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Relative *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_relative_time,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Set an option of type 'struct GNUNET_TIME_Absolute' from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.  It should be followed by a pointer to a value of
+ * type 'struct GNUNET_TIME_Absolute'.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the `struct GNUNET_TIME_Absolute`)
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_absolute_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                   void *scls,
+                   const char *option,
+                   const char *value)
+{
+  struct GNUNET_TIME_Absolute *val = scls;
+
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_fancy_time_to_absolute (value,
+                                             val))
+  {
+    FPRINTF (stderr,
+             _("You must pass absolute time to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -313,6 +681,37 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
 
 
 /**
+ * Allow user to specify a `struct GNUNET_TIME_Absolute`
+ * (using human-readable "fancy" time).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_absolute_time (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Absolute *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_absolute_time,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'unsigned int' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -325,19 +724,172 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                        void *scls, const char *option, const char *value)
+static int
+set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+          void *scls,
+          const char *option,
+          const char *value)
 {
   unsigned int *val = scls;
 
-  if (1 != SSCANF (value, "%u", val))
+  if (1 != SSCANF (value,
+                   "%u",
+                   val))
   {
-    FPRINTF (stderr, _("You must pass a number to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass a number to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
 }
 
 
+/**
+ * Allow user to specify an unsigned integer.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_uint (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               unsigned int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_uint,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Closure for #set_base32().
+ */
+struct Base32Context
+{
+  /**
+   * Value to initialize (already allocated)
+   */
+  void *val;
+
+  /**
+   * Number of bytes expected for @e val.
+   */
+  size_t val_size;
+};
+
+
+/**
+ * Set an option of type 'unsigned int' from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.  It should be followed by a pointer to a value of
+ * type 'unsigned int'.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the 'unsigned int')
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_base32 (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+            void *scls,
+            const char *option,
+            const char *value)
+{
+  struct Base32Context *bc = scls;
+
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_string_to_data (value,
+                                     strlen (value),
+                                     bc->val,
+                                     bc->val_size))
+  {
+    fprintf (stderr,
+             _("Argument `%s' malformed. Expected base32 (Crockford) encoded value.\n"),
+             option);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Helper function to clean up after
+ * #GNUNET_GETOPT_option_base32_fixed_size.
+ *
+ * @param cls value to GNUNET_free()
+ */
+static void
+free_bc (void *cls)
+{
+  GNUNET_free (cls);
+}
+
+
+/**
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument
+ * @param val_size size of @a val in bytes
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_base32_fixed_size (char shortName,
+                                            const char *name,
+                                            const char *argumentHelp,
+                                            const char *description,
+                                            void *val,
+                                            size_t val_size)
+{
+  struct Base32Context *bc = GNUNET_new (struct Base32Context);
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_base32,
+    .cleaner = &free_bc,
+    .scls = (void *) bc
+  };
+
+  bc->val = val;
+  bc->val_size = val_size;
+  return clo;
+}
+
+
+/**
+ * Make the given option mandatory.
+ *
+ * @param opt option to modify
+ * @return @a opt with the mandatory flag set.
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_option_mandatory (struct GNUNET_GETOPT_CommandLineOption opt)
+{
+  opt.option_mandatory = 1;
+  return opt;
+}
+
+
 /* end of getopt_helpers.c */
diff --git a/src/util/gnunet-config.c b/src/util/gnunet-config.c
index 7ec7162f1..fb3b9ebf9 100644
--- a/src/util/gnunet-config.c
+++ b/src/util/gnunet-config.c
@@ -223,34 +223,48 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'f', "filename", NULL,
-      gettext_noop ("obtain option of value as a filename (with $-expansion)"),
-      0, &GNUNET_GETOPT_set_one, &is_filename },
-    { 's', "section", "SECTION",
-      gettext_noop ("name of the section to access"),
-      1, &GNUNET_GETOPT_set_string, &section },
-    { 'o', "option", "OPTION",
-      gettext_noop ("name of the option to access"),
-      1, &GNUNET_GETOPT_set_string, &option },
-    { 'V', "value", "VALUE",
-      gettext_noop ("value to set"),
-      1, &GNUNET_GETOPT_set_string, &value },
-    { 'S', "list-sections", NULL,
-      gettext_noop ("print available configuration sections"),
-      0, &GNUNET_GETOPT_set_one, &list_sections },
-    { 'w', "rewrite", NULL,
-      gettext_noop ("write configuration file that only contains delta to defaults"),
-      0, &GNUNET_GETOPT_set_one, &rewrite },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('f',
+                                  "filename",
+                                  gettext_noop ("obtain option of value as a filename (with $-expansion)"),
+                                  &is_filename),
+    GNUNET_GETOPT_option_string ('s',
+                                     "section",
+                                     "SECTION",
+                                     gettext_noop ("name of the section to access"),
+                                     &section),
+    GNUNET_GETOPT_option_string ('o',
+                                     "option",
+                                     "OPTION",
+                                     gettext_noop ("name of the option to access"),
+                                     &option),
+    GNUNET_GETOPT_option_string ('V',
+                                     "value",
+                                     "VALUE",
+                                     gettext_noop ("value to set"),
+                                     &value),
+    GNUNET_GETOPT_option_flag ('S',
+                                  "list-sections",
+                                  gettext_noop ("print available configuration sections"),
+                                  &list_sections),
+    GNUNET_GETOPT_option_flag ('w',
+                                  "rewrite",
+                                  gettext_noop ("write configuration file that only contains delta to defaults"),
+                                  &rewrite),
     GNUNET_GETOPT_OPTION_END
   };
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
 
   ret = (GNUNET_OK ==
-         GNUNET_PROGRAM_run (argc, argv, "gnunet-config [OPTIONS]",
+         GNUNET_PROGRAM_run (argc,
+                             argv,
+                             "gnunet-config [OPTIONS]",
                              gettext_noop ("Manipulate GNUnet configuration files"),
-                             options, &run, NULL)) ? 0 : ret;
+                             options,
+                             &run, NULL)) ? 0 : ret;
   GNUNET_free ((void*) argv);
   return ret;
 }
diff --git a/src/util/gnunet-ecc.c b/src/util/gnunet-ecc.c
index ddfd9b1c3..2a712f4eb 100644
--- a/src/util/gnunet-ecc.c
+++ b/src/util/gnunet-ecc.c
@@ -41,7 +41,7 @@ static int list_keys;
 /**
  * Flag for listing public key.
  */
-static int list_keys_count;
+static unsigned int list_keys_count;
 
 /**
  * Flag for printing public key.
@@ -406,36 +406,50 @@ run (void *cls, char *const *args, const char *cfgfile,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
   list_keys_count = UINT32_MAX;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'i', "iterate", "FILE",
-      gettext_noop ("list keys included in a file (for testing)"),
-      0, &GNUNET_GETOPT_set_one, &list_keys },
-    { 'e', "end=", "COUNT",
-      gettext_noop ("number of keys to list included in a file (for testing)"),
-      1, &GNUNET_GETOPT_set_uint, &list_keys_count },
-    { 'g', "generate-keys", "COUNT",
-      gettext_noop ("create COUNT public-private key pairs (for testing)"),
-      1, &GNUNET_GETOPT_set_uint, &make_keys },
-    { 'p', "print-public-key", NULL,
-      gettext_noop ("print the public key in ASCII format"),
-      0, &GNUNET_GETOPT_set_one, &print_public_key },
-    { 'E', "examples", NULL,
-      gettext_noop ("print examples of ECC operations (used for compatibility testing)"),
-      0, &GNUNET_GETOPT_set_one, &print_examples_flag },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('i',
+                                  "iterate",
+                                  gettext_noop ("list keys included in a file (for testing)"),
+                                  &list_keys),
+    GNUNET_GETOPT_option_uint ('e',
+                                   "end=",
+                                   "COUNT",
+                                   gettext_noop ("number of keys to list included in a file (for testing)"),
+                                   &list_keys_count),
+    GNUNET_GETOPT_option_uint ('g',
+                                   "generate-keys",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT public-private key pairs (for testing)"),
+                                   &make_keys),
+    GNUNET_GETOPT_option_flag ('p',
+                                  "print-public-key",
+                                  gettext_noop ("print the public key in ASCII format"),
+                                  &print_public_key),
+    GNUNET_GETOPT_option_flag ('E',
+                                  "examples",
+                                  gettext_noop ("print examples of ECC operations (used for compatibility testing)"),
+                                  &print_examples_flag),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
 
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
 
   ret = (GNUNET_OK ==
-         GNUNET_PROGRAM_run (argc, argv, "gnunet-ecc [OPTIONS] keyfile [VANITY_PREFIX]",
+         GNUNET_PROGRAM_run (argc,
+                             argv,
+                             "gnunet-ecc [OPTIONS] keyfile [VANITY_PREFIX]",
                              gettext_noop ("Manipulate GNUnet private ECC key files"),
-                             options, &run, NULL)) ? 0 : 1;
+                             options,
+                             &run,
+                             NULL)) ? 0 : 1;
   GNUNET_free ((void*) argv);
   return ret;
 }
diff --git a/src/util/gnunet-resolver.c b/src/util/gnunet-resolver.c
index e84a2332f..7ffafee32 100644
--- a/src/util/gnunet-resolver.c
+++ b/src/util/gnunet-resolver.c
@@ -144,10 +144,11 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'r', "reverse", NULL,
-      gettext_noop ("perform a reverse lookup"),
-      0, &GNUNET_GETOPT_set_one, &reverse },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('r',
+                                  "reverse",
+                                  gettext_noop ("perform a reverse lookup"),
+                                  &reverse),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/util/gnunet-scrypt.c b/src/util/gnunet-scrypt.c
index ab0cf92e0..3b3b7d695 100644
--- a/src/util/gnunet-scrypt.c
+++ b/src/util/gnunet-scrypt.c
@@ -307,21 +307,30 @@ run (void *cls,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'b', "bits", "BITS",
-      gettext_noop ("number of bits to require for the proof of work"),
-      1, &GNUNET_GETOPT_set_ulong, &nse_work_required },
-    { 'k', "keyfile", "FILE",
-      gettext_noop ("file with private key, otherwise default is used"),
-      1, &GNUNET_GETOPT_set_filename, &pkfn },
-    { 'o', "outfile", "FILE",
-      gettext_noop ("file with proof of work, otherwise default is used"),
-      1, &GNUNET_GETOPT_set_filename, &pwfn },
-    { 't', "timeout", "TIME",
-      gettext_noop ("time to wait between calculations"),
-      1, &GNUNET_GETOPT_set_relative_time, &proof_find_delay },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_ulong ('b',
+                                    "bits",
+                                    "BITS",
+                                    gettext_noop ("number of bits to require for the proof of work"),
+                                    &nse_work_required),
+    GNUNET_GETOPT_option_filename ('k',
+                                   "keyfile",
+                                   "FILE",
+                                   gettext_noop ("file with private key, otherwise default is used"),
+                                   &pkfn),
+    GNUNET_GETOPT_option_filename ('o',
+                                   "outfile",
+                                   "FILE",
+                                   gettext_noop ("file with proof of work, otherwise default is used"),
+                                   &pwfn),
+    GNUNET_GETOPT_option_relative_time ('t',
+                                            "timeout",
+                                            "TIME",
+                                            gettext_noop ("time to wait between calculations"),
+                                            &proof_find_delay),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
@@ -334,7 +343,9 @@ main (int argc, char *const *argv)
          GNUNET_PROGRAM_run (argc, argv,
                              "gnunet-scrypt [OPTIONS] prooffile",
                              gettext_noop ("Manipulate GNUnet proof of work files"),
-                             options, &run, NULL)) ? 0 : 1;
+                             options,
+                             &run,
+                             NULL)) ? 0 : 1;
   GNUNET_free ((void*) argv);
   GNUNET_free_non_null (pwfn);
   return ret;
diff --git a/src/util/helper.c b/src/util/helper.c
index cdb1b01d4..a84b06e66 100644
--- a/src/util/helper.c
+++ b/src/util/helper.c
@@ -27,6 +27,7 @@
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_mst_lib.h"
 
 
 /**
@@ -107,7 +108,7 @@ struct GNUNET_HELPER_Handle
   /**
    * The Message-Tokenizer that tokenizes the messages comming from the helper
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *mst;
+  struct GNUNET_MessageStreamTokenizer *mst;
 
   /**
    * The exception callback
@@ -272,7 +273,10 @@ GNUNET_HELPER_wait (struct GNUNET_HELPER_Handle *h)
   }
   /* purge MST buffer */
   if (NULL != h->mst)
-    (void) GNUNET_SERVER_mst_receive (h->mst, NULL, NULL, 0, GNUNET_YES, GNUNET_NO);
+    (void) GNUNET_MST_from_buffer (h->mst,
+                                   NULL, 0,
+                                   GNUNET_YES,
+                                   GNUNET_NO);
   return ret;
 }
 
@@ -319,7 +323,7 @@ static void
 helper_read (void *cls)
 {
   struct GNUNET_HELPER_Handle *h = cls;
-  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE] GNUNET_ALIGN;
+  char buf[GNUNET_MAX_MESSAGE_SIZE] GNUNET_ALIGN;
   ssize_t t;
 
   h->read_task = NULL;
@@ -373,10 +377,10 @@ helper_read (void *cls)
                                                  h->fh_from_helper,
                                                  &helper_read, h);
   if (GNUNET_SYSERR ==
-      GNUNET_SERVER_mst_receive (h->mst,
-                                 NULL,
-                                 buf, t,
-                                 GNUNET_NO, GNUNET_NO))
+      GNUNET_MST_from_buffer (h->mst,
+                              buf, t,
+                              GNUNET_NO,
+                              GNUNET_NO))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to parse inbound message from helper `%s'\n"),
@@ -487,7 +491,7 @@ struct GNUNET_HELPER_Handle *
 GNUNET_HELPER_start (int with_control_pipe,
                      const char *binary_name,
                      char *const binary_argv[],
-                     GNUNET_SERVER_MessageTokenizerCallback cb,
+                     GNUNET_MessageTokenizerCallback cb,
                      GNUNET_HELPER_ExceptionCallback exp_cb,
                      void *cb_cls)
 {
@@ -508,7 +512,8 @@ GNUNET_HELPER_start (int with_control_pipe,
   h->binary_argv[c] = NULL;
   h->cb_cls = cb_cls;
   if (NULL != cb)
-    h->mst = GNUNET_SERVER_mst_create (cb, h->cb_cls);
+    h->mst = GNUNET_MST_create (cb,
+                                h->cb_cls);
   h->exp_cb = exp_cb;
   h->retry_back_off = 0;
   start_helper (h);
@@ -544,7 +549,7 @@ GNUNET_HELPER_destroy (struct GNUNET_HELPER_Handle *h)
     GNUNET_free (sh);
   }
   if (NULL != h->mst)
-    GNUNET_SERVER_mst_destroy (h->mst);
+    GNUNET_MST_destroy (h->mst);
   GNUNET_free (h->binary_name);
   for (c = 0; h->binary_argv[c] != NULL; c++)
     GNUNET_free (h->binary_argv[c]);
diff --git a/src/util/load.c b/src/util/load.c
index d374d7a17..d1de6aa36 100644
--- a/src/util/load.c
+++ b/src/util/load.c
@@ -27,7 +27,7 @@
 #include "gnunet_util_lib.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-load", __VA_ARGS__)
 
 /**
  * Values we track for load calculations.
diff --git a/src/util/mq.c b/src/util/mq.c
index d12f69e5f..90b2aa968 100644
--- a/src/util/mq.c
+++ b/src/util/mq.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012-2014 GNUnet e.V.
+     Copyright (C) 2012-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "mq",__VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-mq",__VA_ARGS__)
 
 
 struct GNUNET_MQ_Envelope
@@ -202,24 +202,6 @@ struct GNUNET_MQ_Handle
 
 
 /**
- * Implementation-specific state for connection to
- * client (MQ for server).
- */
-struct ServerClientSocketState
-{
-  /**
-   * Handle of the client that connected to the server.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Active transmission request to the client.
-   */
-  struct GNUNET_SERVER_TransmitHandle *th;
-};
-
-
-/**
  * Call the message message handler that was registered
  * for the type of the given message in the given message queue.
  *
@@ -235,24 +217,29 @@ GNUNET_MQ_inject_message (struct GNUNET_MQ_Handle *mq,
 {
   const struct GNUNET_MQ_MessageHandler *handler;
   int handled = GNUNET_NO;
-  uint16_t ms = ntohs (mh->size);
+  uint16_t msize = ntohs (mh->size);
+  uint16_t mtype = ntohs (mh->type);
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received message of type %u and size %u\n",
+       mtype, msize);
 
   if (NULL == mq->handlers)
     goto done;
   for (handler = mq->handlers; NULL != handler->cb; handler++)
   {
-    if (handler->type == ntohs (mh->type))
+    if (handler->type == mtype)
     {
       handled = GNUNET_YES;
-      if ( (handler->expected_size > ms) ||
-           ( (handler->expected_size != ms) &&
+      if ( (handler->expected_size > msize) ||
+           ( (handler->expected_size != msize) &&
              (NULL == handler->mv) ) )
       {
         /* Too small, or not an exact size and
            no 'mv' handler to check rest */
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Received malformed message of type %u\n",
-                    (unsigned int) handler->type);
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "Received malformed message of type %u\n",
+             (unsigned int) handler->type);
         GNUNET_MQ_inject_error (mq,
                                 GNUNET_MQ_ERROR_MALFORMED);
         break;
@@ -267,9 +254,9 @@ GNUNET_MQ_inject_message (struct GNUNET_MQ_Handle *mq,
       else
       {
         /* Message rejected by check routine */
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Received malformed message of type %u\n",
-                    (unsigned int) handler->type);
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "Received malformed message of type %u\n",
+             (unsigned int) handler->type);
         GNUNET_MQ_inject_error (mq,
                                 GNUNET_MQ_ERROR_MALFORMED);
       }
@@ -279,9 +266,8 @@ GNUNET_MQ_inject_message (struct GNUNET_MQ_Handle *mq,
  done:
   if (GNUNET_NO == handled)
     LOG (GNUNET_ERROR_TYPE_INFO,
-         "No handler for message of type %d and size %d\n",
-         ntohs (mh->type),
-         ntohs (mh->size));
+         "No handler for message of type %u and size %u\n",
+         mtype, msize);
 }
 
 
@@ -358,6 +344,7 @@ GNUNET_MQ_send (struct GNUNET_MQ_Handle *mq,
   GNUNET_assert (NULL == ev->parent_queue);
 
   mq->queue_length++;
+  GNUNET_break (mq->queue_length < 10000); /* This would seem like a bug... */
   ev->parent_queue = mq;
   /* is the implementation busy? queue it! */
   if ( (NULL != mq->current_envelope) ||
@@ -368,6 +355,7 @@ GNUNET_MQ_send (struct GNUNET_MQ_Handle *mq,
                                       ev);
     return;
   }
+  GNUNET_assert (NULL == mq->envelope_head);
   mq->current_envelope = ev;
   mq->send_impl (mq,
                  ev->mh,
@@ -376,6 +364,46 @@ GNUNET_MQ_send (struct GNUNET_MQ_Handle *mq,
 
 
 /**
+ * Remove the first envelope that has not yet been sent from the message
+ * queue and return it.
+ *
+ * @param mq queue to remove envelope from
+ * @return NULL if queue is empty (or has no envelope that is not under transmission)
+ */
+struct GNUNET_MQ_Envelope *
+GNUNET_MQ_unsent_head (struct GNUNET_MQ_Handle *mq)
+{
+  struct GNUNET_MQ_Envelope *env;
+
+  env = mq->envelope_head;
+  GNUNET_CONTAINER_DLL_remove (mq->envelope_head,
+                               mq->envelope_tail,
+                               env);
+  mq->queue_length--;
+  env->parent_queue = NULL;
+  return env;
+}
+
+
+/**
+ * Function to copy an envelope.  The envelope must not yet
+ * be in any queue or have any options or callbacks set.
+ *
+ * @param env envelope to copy
+ * @return copy of @a env
+ */
+struct GNUNET_MQ_Envelope *
+GNUNET_MQ_env_copy (struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_assert (NULL == env->next);
+  GNUNET_assert (NULL == env->parent_queue);
+  GNUNET_assert (NULL == env->sent_cb);
+  GNUNET_assert (GNUNET_NO == env->have_custom_options);
+  return GNUNET_MQ_msg_copy (env->mh);
+}
+
+
+/**
  * Send a copy of a message with the given message queue.
  * Can be called repeatedly on the same envelope.
  *
@@ -514,21 +542,12 @@ GNUNET_MQ_queue_for_callbacks (GNUNET_MQ_SendImpl send,
                                void *error_handler_cls)
 {
   struct GNUNET_MQ_Handle *mq;
-  unsigned int i;
 
   mq = GNUNET_new (struct GNUNET_MQ_Handle);
   mq->send_impl = send;
   mq->destroy_impl = destroy;
   mq->cancel_impl = cancel;
-  if (NULL != handlers)
-  {
-    for (i=0;NULL != handlers[i].cb; i++) ;
-    mq->handlers = GNUNET_new_array (i + 1,
-                                     struct GNUNET_MQ_MessageHandler);
-    GNUNET_memcpy (mq->handlers,
-            handlers,
-            i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }
+  mq->handlers = GNUNET_MQ_copy_handlers (handlers);
   mq->error_handler = error_handler;
   mq->error_handler_cls = error_handler_cls;
   mq->impl_state = impl_state;
@@ -671,87 +690,6 @@ GNUNET_MQ_msg_nested_mh_ (struct GNUNET_MessageHeader **mhp,
 
 
 /**
- * Transmit a queued message to the session's client.
- *
- * @param cls consensus session
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to @a buf
- */
-static size_t
-transmit_queued (void *cls,
-                 size_t size,
-                 void *buf)
-{
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct ServerClientSocketState *state = GNUNET_MQ_impl_state (mq);
-  const struct GNUNET_MessageHeader *msg = GNUNET_MQ_impl_current (mq);
-  size_t msg_size;
-
-  GNUNET_assert (NULL != buf);
-  msg_size = ntohs (msg->size);
-  GNUNET_assert (size >= msg_size);
-  GNUNET_memcpy (buf, msg, msg_size);
-  state->th = NULL;
-
-  GNUNET_MQ_impl_send_continue (mq);
-
-  return msg_size;
-}
-
-
-static void
-server_client_destroy_impl (struct GNUNET_MQ_Handle *mq,
-                            void *impl_state)
-{
-  struct ServerClientSocketState *state = impl_state;
-
-  if (NULL != state->th)
-  {
-    GNUNET_SERVER_notify_transmit_ready_cancel (state->th);
-    state->th = NULL;
-  }
-
-  GNUNET_assert (NULL != mq);
-  GNUNET_assert (NULL != state);
-  GNUNET_SERVER_client_drop (state->client);
-  GNUNET_free (state);
-}
-
-
-static void
-server_client_send_impl (struct GNUNET_MQ_Handle *mq,
-                         const struct GNUNET_MessageHeader *msg,
-                         void *impl_state)
-{
-  struct ServerClientSocketState *state = impl_state;
-
-  GNUNET_assert (NULL != mq);
-  state->th = GNUNET_SERVER_notify_transmit_ready (state->client,
-                                                   ntohs (msg->size),
-                                                   GNUNET_TIME_UNIT_FOREVER_REL,
-                                                   &transmit_queued, mq);
-}
-
-
-struct GNUNET_MQ_Handle *
-GNUNET_MQ_queue_for_server_client (struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_MQ_Handle *mq;
-  struct ServerClientSocketState *scss;
-
-  mq = GNUNET_new (struct GNUNET_MQ_Handle);
-  scss = GNUNET_new (struct ServerClientSocketState);
-  mq->impl_state = scss;
-  scss->client = client;
-  GNUNET_SERVER_client_keep (client);
-  mq->send_impl = &server_client_send_impl;
-  mq->destroy_impl = &server_client_destroy_impl;
-  return mq;
-}
-
-
-/**
  * Associate the assoc_data in mq with a unique request id.
  *
  * @param mq message queue, id will be unique for the queue
@@ -769,22 +707,40 @@ GNUNET_MQ_assoc_add (struct GNUNET_MQ_Handle *mq,
     mq->assoc_id = 1;
   }
   id = mq->assoc_id++;
-  GNUNET_CONTAINER_multihashmap32_put (mq->assoc_map, id, assoc_data,
-                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (mq->assoc_map,
+                                                      id,
+                                                      assoc_data,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   return id;
 }
 
 
+/**
+ * Get the data associated with a @a request_id in a queue
+ *
+ * @param mq the message queue with the association
+ * @param request_id the request id we are interested in
+ * @return the associated data
+ */
 void *
 GNUNET_MQ_assoc_get (struct GNUNET_MQ_Handle *mq,
                      uint32_t request_id)
 {
   if (NULL == mq->assoc_map)
     return NULL;
-  return GNUNET_CONTAINER_multihashmap32_get (mq->assoc_map, request_id);
+  return GNUNET_CONTAINER_multihashmap32_get (mq->assoc_map,
+                                              request_id);
 }
 
 
+/**
+ * Remove the association for a @a request_id
+ *
+ * @param mq the message queue with the association
+ * @param request_id the request id we want to remove
+ * @return the associated data
+ */
 void *
 GNUNET_MQ_assoc_remove (struct GNUNET_MQ_Handle *mq,
                         uint32_t request_id)
@@ -959,20 +915,16 @@ GNUNET_MQ_send_cancel (struct GNUNET_MQ_Envelope *ev)
 
   if (mq->current_envelope == ev)
   {
-    // complex case, we already started with transmitting
-    // the message
+    /* complex case, we already started with transmitting
+       the message using the callbacks. */
     GNUNET_assert (0 < mq->queue_length);
     mq->queue_length--;
     mq->cancel_impl (mq,
                      mq->impl_state);
-    // continue sending the next message, if any
-    if (NULL == mq->envelope_head)
-    {
-      mq->current_envelope = NULL;
-    }
-    else
+    /* continue sending the next message, if any */
+    mq->current_envelope = mq->envelope_head;
+    if (NULL != mq->current_envelope)
     {
-      mq->current_envelope = mq->envelope_head;
       GNUNET_CONTAINER_DLL_remove (mq->envelope_head,
                                    mq->envelope_tail,
                                    mq->current_envelope);
@@ -983,7 +935,7 @@ GNUNET_MQ_send_cancel (struct GNUNET_MQ_Envelope *ev)
   }
   else
   {
-    // simple case, message is still waiting in the queue
+    /* simple case, message is still waiting in the queue */
     GNUNET_CONTAINER_DLL_remove (mq->envelope_head,
                                  mq->envelope_tail,
                                  ev);
@@ -1140,4 +1092,100 @@ GNUNET_MQ_destroy_notify_cancel (struct GNUNET_MQ_DestroyNotificationHandle *dnh
 }
 
 
+/**
+ * Insert @a env into the envelope DLL starting at @a env_head
+ * Note that @a env must not be in any MQ while this function
+ * is used with DLLs defined outside of the MQ module.  This
+ * is just in case some application needs to also manage a
+ * FIFO of envelopes independent of MQ itself and wants to
+ * re-use the pointers internal to @a env.  Use with caution.
+ *
+ * @param[in|out] env_head of envelope DLL
+ * @param[in|out] env_tail tail of envelope DLL
+ * @param[in|out] env element to insert at the tail
+ */
+void
+GNUNET_MQ_dll_insert_tail (struct GNUNET_MQ_Envelope **env_head,
+                           struct GNUNET_MQ_Envelope **env_tail,
+                           struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_CONTAINER_DLL_insert_tail (*env_head,
+                                    *env_tail,
+                                    env);
+}
+
+
+/**
+ * Remove @a env from the envelope DLL starting at @a env_head.
+ * Note that @a env must not be in any MQ while this function
+ * is used with DLLs defined outside of the MQ module. This
+ * is just in case some application needs to also manage a
+ * FIFO of envelopes independent of MQ itself and wants to
+ * re-use the pointers internal to @a env.  Use with caution.
+ *
+ * @param[in|out] env_head of envelope DLL
+ * @param[in|out] env_tail tail of envelope DLL
+ * @param[in|out] env element to remove from the DLL
+ */
+void
+GNUNET_MQ_dll_remove (struct GNUNET_MQ_Envelope **env_head,
+                      struct GNUNET_MQ_Envelope **env_tail,
+                      struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_CONTAINER_DLL_remove (*env_head,
+                               *env_tail,
+                               env);
+}
+
+
+/**
+ * Copy an array of handlers.
+ *
+ * Useful if the array has been delared in local memory and needs to be
+ * persisted for future use.
+ *
+ * @param handlers Array of handlers to be copied. Can be NULL (nothing done).
+ * @return A newly allocated array of handlers.
+ *         Needs to be freed with #GNUNET_free.
+ */
+struct GNUNET_MQ_MessageHandler *
+GNUNET_MQ_copy_handlers (const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  struct GNUNET_MQ_MessageHandler *copy;
+  unsigned int count;
+
+  if (NULL == handlers)
+    return NULL;
+
+  count = GNUNET_MQ_count_handlers (handlers);
+  copy = GNUNET_new_array (count + 1,
+                           struct GNUNET_MQ_MessageHandler);
+  GNUNET_memcpy (copy,
+                 handlers,
+                 count * sizeof (struct GNUNET_MQ_MessageHandler));
+  return copy;
+}
+
+
+/**
+ * Count the handlers in a handler array.
+ *
+ * @param handlers Array of handlers to be counted.
+ * @return The number of handlers in the array.
+ */
+unsigned int
+GNUNET_MQ_count_handlers (const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  unsigned int i;
+
+  if (NULL == handlers)
+    return 0;
+
+  for (i=0; NULL != handlers[i].cb; i++) ;
+
+  return i;
+}
+
+
+
 /* end of mq.c */
diff --git a/src/util/mst.c b/src/util/mst.c
index 82a21b880..0d90c5d10 100644
--- a/src/util/mst.c
+++ b/src/util/mst.c
@@ -34,7 +34,7 @@
 #define ALIGN_FACTOR 8
 #endif
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-mst", __VA_ARGS__)
 
 
 /**
@@ -90,8 +90,8 @@ GNUNET_MST_create (GNUNET_MessageTokenizerCallback cb,
   struct GNUNET_MessageStreamTokenizer *ret;
 
   ret = GNUNET_new (struct GNUNET_MessageStreamTokenizer);
-  ret->hdr = GNUNET_malloc (GNUNET_SERVER_MIN_BUFFER_SIZE);
-  ret->curr_buf = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  ret->hdr = GNUNET_malloc (GNUNET_MIN_MESSAGE_SIZE);
+  ret->curr_buf = GNUNET_MIN_MESSAGE_SIZE;
   ret->cb = cb;
   ret->cb_cls = cb_cls;
   return ret;
@@ -130,7 +130,7 @@ GNUNET_MST_from_buffer (struct GNUNET_MessageStreamTokenizer *mst,
   GNUNET_assert (mst->off <= mst->pos);
   GNUNET_assert (mst->pos <= mst->curr_buf);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Server-mst receives %u bytes with %u bytes already in private buffer\n",
+       "MST receives %u bytes with %u bytes already in private buffer\n",
        (unsigned int) size,
        (unsigned int) (mst->pos - mst->off));
   ret = GNUNET_OK;
@@ -151,7 +151,7 @@ do_align:
     }
     if (mst->pos - mst->off < sizeof (struct GNUNET_MessageHeader))
     {
-      delta 
+      delta
         = GNUNET_MIN (sizeof (struct GNUNET_MessageHeader)
                       - (mst->pos - mst->off),
                       size);
@@ -229,8 +229,8 @@ do_align:
     if (one_shot == GNUNET_YES)
       one_shot = GNUNET_SYSERR;
     mst->off += want;
-    if (GNUNET_SYSERR == mst->cb (mst->cb_cls,
-                                  hdr))
+  if (GNUNET_SYSERR == mst->cb (mst->cb_cls,
+                                hdr))
       return GNUNET_SYSERR;
     if (mst->off == mst->pos)
     {
diff --git a/src/util/network.c b/src/util/network.c
index 67f2801c5..66a468e45 100644
--- a/src/util/network.c
+++ b/src/util/network.c
@@ -28,9 +28,9 @@
 #include "gnunet_util_lib.h"
 #include "disk.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-network", __VA_ARGS__)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-network", syscall, filename)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-network", syscall)
 
 #define DEBUG_NETWORK GNUNET_EXTRA_LOGGING
 
@@ -1697,6 +1697,64 @@ initialize_select_thread ()
 
 #endif
 
+/**
+ * Test if the given @a port is available.
+ *
+ * @param ipproto transport protocol to test (i.e. IPPROTO_TCP)
+ * @param port port number to test
+ * @return #GNUNET_OK if the port is available, #GNUNET_NO if not
+ */
+int
+GNUNET_NETWORK_test_port_free (int ipproto,
+                               uint16_t port)
+{
+  struct GNUNET_NETWORK_Handle *socket;
+  int bind_status;
+  int socktype;
+  char open_port_str[6];
+  struct addrinfo hint;
+  struct addrinfo *ret;
+  struct addrinfo *ai;
+
+  GNUNET_snprintf (open_port_str,
+                   sizeof (open_port_str),
+                   "%u",
+                   (unsigned int) port);
+  socktype = (IPPROTO_TCP == ipproto)
+    ? SOCK_STREAM
+    : SOCK_DGRAM;
+  ret = NULL;
+  memset (&hint, 0, sizeof (hint));
+  hint.ai_family = AF_UNSPEC;   /* IPv4 and IPv6 */
+  hint.ai_socktype = socktype;
+  hint.ai_protocol = ipproto;
+  hint.ai_addrlen = 0;
+  hint.ai_addr = NULL;
+  hint.ai_canonname = NULL;
+  hint.ai_next = NULL;
+  hint.ai_flags = AI_PASSIVE | AI_NUMERICSERV;  /* Wild card address */
+  GNUNET_assert (0 == getaddrinfo (NULL,
+                                   open_port_str,
+                                   &hint,
+                                   &ret));
+  for (ai = ret; NULL != ai; ai = ai->ai_next)
+  {
+    socket = GNUNET_NETWORK_socket_create (ai->ai_family,
+                                           ai->ai_socktype,
+                                           ai->ai_protocol);
+    if (NULL == socket)
+      continue;
+    bind_status = GNUNET_NETWORK_socket_bind (socket,
+                                              ai->ai_addr,
+                                              ai->ai_addrlen);
+    GNUNET_NETWORK_socket_close (socket);
+    if (GNUNET_OK != bind_status)
+      break;
+  }
+  freeaddrinfo (ret);
+  return bind_status;
+}
+
 
 #ifndef MINGW
 /**
diff --git a/src/util/os_installation.c b/src/util/os_installation.c
index ea3ae3e79..1226c5966 100644
--- a/src/util/os_installation.c
+++ b/src/util/os_installation.c
@@ -44,9 +44,9 @@
 #endif
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-os-installation", __VA_ARGS__)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-os-installation", syscall, filename)
 
 
 /**
@@ -823,9 +823,13 @@ GNUNET_OS_check_helper_binary (const char *binary,
 #ifdef MINGW
   char *binaryexe;
 
-  GNUNET_asprintf (&binaryexe, "%s.exe", binary);
-  if ( (GNUNET_YES == GNUNET_STRINGS_path_is_absolute (binaryexe, GNUNET_NO,
-                                                       NULL, NULL)) ||
+  GNUNET_asprintf (&binaryexe,
+                   "%s.exe",
+                   binary);
+  if ( (GNUNET_YES ==
+        GNUNET_STRINGS_path_is_absolute (binaryexe,
+                                         GNUNET_NO,
+                                         NULL, NULL)) ||
        (0 == strncmp (binary, "./", 2)) )
     p = GNUNET_strdup (binaryexe);
   else
@@ -840,16 +844,24 @@ GNUNET_OS_check_helper_binary (const char *binary,
   }
   GNUNET_free (binaryexe);
 #else
-  if ( (GNUNET_YES == GNUNET_STRINGS_path_is_absolute (binary, GNUNET_NO,
-                                                       NULL, NULL)) ||
+  if ( (GNUNET_YES ==
+        GNUNET_STRINGS_path_is_absolute (binary,
+                                         GNUNET_NO,
+                                         NULL,
+                                         NULL)) ||
        (0 == strncmp (binary, "./", 2)) )
+  {
     p = GNUNET_strdup (binary);
+  }
   else
   {
     p = get_path_from_PATH (binary);
     if (NULL != p)
     {
-      GNUNET_asprintf (&pf, "%s/%s", p, binary);
+      GNUNET_asprintf (&pf,
+                       "%s/%s",
+                       p,
+                       binary);
       GNUNET_free (p);
       p = pf;
     }
@@ -862,9 +874,12 @@ GNUNET_OS_check_helper_binary (const char *binary,
          binary);
     return GNUNET_SYSERR;
   }
-  if (0 != ACCESS (p, X_OK))
+  if (0 != ACCESS (p,
+                   X_OK))
   {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "access", p);
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
+                       "access",
+                       p);
     GNUNET_free (p);
     return GNUNET_SYSERR;
   }
@@ -873,22 +888,30 @@ GNUNET_OS_check_helper_binary (const char *binary,
   {
     /* as we run as root, we don't insist on SUID */
     GNUNET_free (p);
-    return GNUNET_OK;
+    return GNUNET_YES;
   }
 #endif
-  if (0 != STAT (p, &statbuf))
+  if (0 != STAT (p,
+                 &statbuf))
   {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "stat", p);
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
+                       "stat",
+                       p);
     GNUNET_free (p);
     return GNUNET_SYSERR;
   }
-  if (check_suid){
+  if (check_suid)
+  {
 #ifndef MINGW
-    if ((0 != (statbuf.st_mode & S_ISUID)) && (0 == statbuf.st_uid))
+    if ( (0 != (statbuf.st_mode & S_ISUID)) &&
+         (0 == statbuf.st_uid) )
     {
       GNUNET_free (p);
       return GNUNET_YES;
     }
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Binary `%s' exists, but is not SUID\n"),
+                p);
     /* binary exists, but not SUID */
 #else
     STARTUPINFO start;
diff --git a/src/util/os_network.c b/src/util/os_network.c
index 5e4546d08..5cb2b6864 100644
--- a/src/util/os_network.c
+++ b/src/util/os_network.c
@@ -31,11 +31,11 @@
 #include "gnunet_util_lib.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-os-network", __VA_ARGS__)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-os-network", syscall, filename)
 
 
-#if ! (HAVE_GETIFADDRS && HAVE_FREEIFADDRS)
+#if ! (HAVE_GETIFADDRS && HAVE_FREEIFADDRS) && !MINGW
 /**
  * Try to enumerate all network interfaces using 'ifconfig'.
  *
diff --git a/src/util/os_priority.c b/src/util/os_priority.c
index 4b1dbd491..2c4f7ca09 100644
--- a/src/util/os_priority.c
+++ b/src/util/os_priority.c
@@ -29,11 +29,11 @@
 #include "disk.h"
 #include <unistr.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-os-priority", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-os-priority", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-os-priority", syscall, filename)
 
 #define GNUNET_OS_CONTROL_PIPE "GNUNET_OS_CONTROL_PIPE"
 
@@ -1091,7 +1091,10 @@ start_process (int pipe_control,
                                        &lsocks_read, sizeof (HANDLE));
   }
   else
+  {
     lsocks_pipe = NULL;
+    lsocks_write_fd = NULL;
+  }
 
   env_off = 0;
   if (GNUNET_YES == pipe_control)
@@ -1229,7 +1232,7 @@ start_process (int pipe_control,
     if (sizeof (count) != wrote)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  "Failed to write %u count bytes to the child: %u\n",
+                  "Failed to write %u count bytes to the child: %lu\n",
                   sizeof (count), GetLastError ());
       break;
     }
@@ -1240,7 +1243,7 @@ start_process (int pipe_control,
       if (SOCKET_ERROR == WSADuplicateSocketA (lsocks[i], gnunet_proc->pid, &pi))
       {
         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Failed to duplicate an socket[%llu]: %u\n", i,
+                    "Failed to duplicate an socket[%u]: %lu\n", i,
                     GetLastError ());
         break;
       }
@@ -1257,7 +1260,7 @@ start_process (int pipe_control,
       if (sizeof (size) != wrote)
       {
         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Failed to write %u size[%llu] bytes to the child: %u\n",
+                    "Failed to write %u size[%u] bytes to the child: %lu\n",
                     sizeof (size), i, GetLastError ());
         break;
       }
@@ -1266,7 +1269,7 @@ start_process (int pipe_control,
       if (sizeof (pi) != wrote)
       {
         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Failed to write %u socket[%llu] bytes to the child: %u\n",
+                    "Failed to write %u socket[%u] bytes to the child: %lu\n",
                     sizeof (pi), i, GetLastError ());
         break;
       }
diff --git a/src/util/peer.c b/src/util/peer.c
index 5d54a4301..b637dc229 100644
--- a/src/util/peer.c
+++ b/src/util/peer.c
@@ -26,7 +26,7 @@
 #include "platform.h"
 #include "gnunet_peer_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-peer", __VA_ARGS__)
 
 
 struct PeerEntry
diff --git a/src/util/plugin.c b/src/util/plugin.c
index c7ac47a7c..fb296f80d 100644
--- a/src/util/plugin.c
+++ b/src/util/plugin.c
@@ -28,7 +28,7 @@
 #include <ltdl.h>
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-plugin", __VA_ARGS__)
 
 /**
  * Linked list of active plugins.
diff --git a/src/util/program.c b/src/util/program.c
index d0dd49909..92a9750f3 100644
--- a/src/util/program.c
+++ b/src/util/program.c
@@ -31,9 +31,9 @@
 #include "speedup.h"
 #include <gcrypt.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-program", __VA_ARGS__)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-program", syscall, filename)
 
 /**
  * Context for the command.
@@ -148,11 +148,11 @@ GNUNET_PROGRAM_run2 (int argc, char *const *argv, const char *binaryName,
   struct GNUNET_CONFIGURATION_Handle *cfg;
 
   struct GNUNET_GETOPT_CommandLineOption defoptions[] = {
-    GNUNET_GETOPT_OPTION_CFG_FILE (&cc.cfgfile),
-    GNUNET_GETOPT_OPTION_HELP (binaryHelp),
-    GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
-    GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
-    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION)
+    GNUNET_GETOPT_option_cfgfile (&cc.cfgfile),
+    GNUNET_GETOPT_option_help (binaryHelp),
+    GNUNET_GETOPT_option_loglevel (&loglev),
+    GNUNET_GETOPT_option_logfile (&logfile),
+    GNUNET_GETOPT_option_version (PACKAGE_VERSION " " VCS_VERSION)
   };
   struct GNUNET_GETOPT_CommandLineOption *allopts;
   const char *gargs;
diff --git a/src/util/resolver_api.c b/src/util/resolver_api.c
index fdeaed006..0c915932c 100644
--- a/src/util/resolver_api.c
+++ b/src/util/resolver_api.c
@@ -29,9 +29,9 @@
 #include "gnunet_resolver_service.h"
 #include "resolver.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "resolver-api", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-resolver-api", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "resolver-api", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-resolver-api", syscall)
 
 /**
  * Maximum supported length for a hostname
@@ -250,7 +250,7 @@ void
 GNUNET_RESOLVER_disconnect ()
 {
   struct GNUNET_RESOLVER_RequestHandle *rh;
-  
+
   while (NULL != (rh = req_head))
   {
     GNUNET_assert (GNUNET_SYSERR == rh->was_transmitted);
@@ -298,7 +298,7 @@ static void
 check_disconnect ()
 {
   struct GNUNET_RESOLVER_RequestHandle *rh;
-  
+
   for (rh = req_head; NULL != rh; rh = rh->next)
     if (GNUNET_SYSERR != rh->was_transmitted)
       return;
@@ -876,7 +876,7 @@ GNUNET_RESOLVER_ip_get (const char *hostname,
 
   slen = strlen (hostname) + 1;
   if (slen + sizeof (struct GNUNET_RESOLVER_GetMessage) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/util/scheduler.c b/src/util/scheduler.c
index 409a0942f..a7b1d8e2a 100644
--- a/src/util/scheduler.c
+++ b/src/util/scheduler.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2009-2016 GNUnet e.V.
+      Copyright (C) 2009-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -17,7 +17,6 @@
       Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
       Boston, MA 02110-1301, USA.
  */
-
 /**
  * @file util/scheduler.c
  * @brief schedule computations using continuation passing style
@@ -71,6 +70,35 @@
 
 
 /**
+ * Argument to be passed from the driver to
+ * #GNUNET_SCHEDULER_run_from_driver().  Contains the
+ * scheduler's internal state.
+ */ 
+struct GNUNET_SCHEDULER_Handle
+{
+  /**
+   * Passed here to avoid constantly allocating/deallocating
+   * this element, but generally we want to get rid of this.
+   * @deprecated
+   */
+  struct GNUNET_NETWORK_FDSet *rs;
+
+  /**
+   * Passed here to avoid constantly allocating/deallocating
+   * this element, but generally we want to get rid of this.
+   * @deprecated
+   */
+  struct GNUNET_NETWORK_FDSet *ws;
+
+  /**
+   * Driver we used for the event loop.
+   */
+  const struct GNUNET_SCHEDULER_Driver *driver;
+  
+};
+
+
+/**
  * Entry in list of pending tasks.
  */
 struct GNUNET_SCHEDULER_Task
@@ -96,6 +124,11 @@ struct GNUNET_SCHEDULER_Task
   void *callback_cls;
 
   /**
+   * Handle to the scheduler's state.
+   */
+  const struct GNUNET_SCHEDULER_Handle *sh;
+  
+  /**
    * Set of file descriptors this task is waiting
    * for for reading.  Once ready, this is updated
    * to reflect the set of file descriptors ready
@@ -111,6 +144,18 @@ struct GNUNET_SCHEDULER_Task
   struct GNUNET_NETWORK_FDSet *write_set;
 
   /**
+   * Information about which FDs are ready for this task (and why).
+   */
+  const struct GNUNET_SCHEDULER_FdInfo *fds;
+
+  /**
+   * Storage location used for @e fds if we want to avoid
+   * a separate malloc() call in the common case that this
+   * task is only about a single FD.
+   */
+  struct GNUNET_SCHEDULER_FdInfo fdx;
+
+  /**
    * Absolute timeout value for the task, or
    * #GNUNET_TIME_UNIT_FOREVER_ABS for "no timeout".
    */
@@ -124,6 +169,11 @@ struct GNUNET_SCHEDULER_Task
 #endif
 
   /**
+   * Size of the @e fds array.
+   */
+  unsigned int fds_len;
+  
+  /**
    * Why is the task ready?  Set after task is added to ready queue.
    * Initially set to zero.  All reasons that have already been
    * satisfied (i.e.  read or write ready) will be set over time.
@@ -1742,7 +1792,8 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio,
   GNUNET_CONTAINER_DLL_insert (pending_head,
                                pending_tail,
                                t);
-  max_priority_added = GNUNET_MAX (max_priority_added, t->priority);
+  max_priority_added = GNUNET_MAX (max_priority_added,
+                                   t->priority);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Adding task %p\n",
        t);
@@ -1750,4 +1801,275 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio,
   return t;
 }
 
+
+/**
+ * Function used by event-loop implementations to signal the scheduler
+ * that a particular @a task is ready due to an event of type @a et.
+ *
+ * This function will then queue the task to notify the application
+ * that the task is ready (with the respective priority).
+ *
+ * @param task the task that is ready, NULL for wake up calls
+ * @param et information about why the task is ready
+ */
+void
+GNUNET_SCHEDULER_task_ready (struct GNUNET_SCHEDULER_Task *task,
+                             enum GNUNET_SCHEDULER_EventType et)
+{
+  enum GNUNET_SCHEDULER_Reason reason;
+  struct GNUNET_TIME_Absolute now;
+
+  now = GNUNET_TIME_absolute_get ();
+  reason = task->reason;
+  if (now.abs_value_us >= task->timeout.abs_value_us)
+    reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
+  if ( (0 == (reason & GNUNET_SCHEDULER_REASON_READ_READY)) &&
+       (0 != (GNUNET_SCHEDULER_ET_IN & et)) )
+    reason |= GNUNET_SCHEDULER_REASON_READ_READY;
+  if ( (0 == (reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) &&
+       (0 != (GNUNET_SCHEDULER_ET_OUT & et)) )
+    reason |= GNUNET_SCHEDULER_REASON_WRITE_READY;
+  reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE;
+  task->reason = reason;
+  task->fds = &task->fdx;
+  task->fdx.et = et;
+  task->fds_len = 1;
+  queue_ready_task (task);
+}
+
+
+/**
+ * Function called by the driver to tell the scheduler to run some of
+ * the tasks that are ready.  This function may return even though
+ * there are tasks left to run just to give other tasks a chance as
+ * well.  If we return #GNUNET_YES, the driver should call this
+ * function again as soon as possible, while if we return #GNUNET_NO
+ * it must block until the operating system has more work as the
+ * scheduler has no more work to do right now.
+ *
+ * @param sh scheduler handle that was given to the `loop`
+ * @return #GNUNET_OK if there are more tasks that are ready,
+ *          and thus we would like to run more (yield to avoid 
+ *          blocking other activities for too long)
+ *         #GNUNET_NO if we are done running tasks (yield to block)
+ *         #GNUNET_SYSERR on error
+ */
+int
+GNUNET_SCHEDULER_run_from_driver (struct GNUNET_SCHEDULER_Handle *sh)
+{
+  enum GNUNET_SCHEDULER_Priority p;
+  struct GNUNET_SCHEDULER_Task *pos;
+  struct GNUNET_TIME_Absolute now;
+
+  /* check for tasks that reached the timeout! */
+  now = GNUNET_TIME_absolute_get ();
+  while (NULL != (pos = pending_timeout_head))
+  {
+    if (now.abs_value_us >= pos->timeout.abs_value_us)
+      pos->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT;
+    if (0 == pos->reason)
+      break;
+    GNUNET_CONTAINER_DLL_remove (pending_timeout_head,
+                                 pending_timeout_tail,
+                                 pos);
+    if (pending_timeout_last == pos)
+      pending_timeout_last = NULL;
+    queue_ready_task (pos);
+  }
+  
+  if (0 == ready_count)
+    return GNUNET_NO;
+
+  /* find out which task priority level we are going to 
+     process this time */
+  max_priority_added = GNUNET_SCHEDULER_PRIORITY_KEEP;
+  GNUNET_assert (NULL == ready_head[GNUNET_SCHEDULER_PRIORITY_KEEP]);
+  /* yes, p>0 is correct, 0 is "KEEP" which should
+   * always be an empty queue (see assertion)! */
+  for (p = GNUNET_SCHEDULER_PRIORITY_COUNT - 1; p > 0; p--)
+  {
+    pos = ready_head[p];
+    if (NULL != pos)
+      break;
+  }
+  GNUNET_assert (NULL != pos);        /* ready_count wrong? */
+
+  /* process all tasks at this priority level, then yield */
+  while (NULL != (pos = ready_head[p]))
+  {
+    GNUNET_CONTAINER_DLL_remove (ready_head[p],
+                                 ready_tail[p],
+                                 pos);
+    ready_count--;
+    current_priority = pos->priority;
+    current_lifeness = pos->lifeness;
+    active_task = pos;
+#if PROFILE_DELAYS
+    if (GNUNET_TIME_absolute_get_duration (pos->start_time).rel_value_us >
+        DELAY_THRESHOLD.rel_value_us)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Task %p took %s to be scheduled\n",
+           pos,
+           GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (pos->start_time),
+                                                   GNUNET_YES));
+    }
+#endif
+    tc.reason = pos->reason;
+    GNUNET_NETWORK_fdset_zero (sh->rs);
+    GNUNET_NETWORK_fdset_zero (sh->ws);
+    tc.fds_len = pos->fds_len;
+    tc.fds = pos->fds;
+    tc.read_ready = (NULL == pos->read_set) ? sh->rs : pos->read_set;
+    if ( (-1 != pos->read_fd) &&
+         (0 != (pos->reason & GNUNET_SCHEDULER_REASON_READ_READY)) )
+      GNUNET_NETWORK_fdset_set_native (sh->rs,
+                                       pos->read_fd);
+    tc.write_ready = (NULL == pos->write_set) ? sh->ws : pos->write_set;
+    if ((-1 != pos->write_fd) &&
+        (0 != (pos->reason & GNUNET_SCHEDULER_REASON_WRITE_READY)))
+      GNUNET_NETWORK_fdset_set_native (sh->ws,
+                                       pos->write_fd);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Running task: %p\n",
+         pos);
+    pos->callback (pos->callback_cls);
+    active_task = NULL;
+    dump_backtrace (pos);
+    destroy_task (pos);
+    tasks_run++;
+  }
+  if (0 == ready_count)
+    return GNUNET_NO;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Initialize and run scheduler.  This function will return when all
+ * tasks have completed.  On systems with signals, receiving a SIGTERM
+ * (and other similar signals) will cause #GNUNET_SCHEDULER_shutdown
+ * to be run after the active task is complete.  As a result, SIGTERM
+ * causes all shutdown tasks to be scheduled with reason
+ * #GNUNET_SCHEDULER_REASON_SHUTDOWN.  (However, tasks added
+ * afterwards will execute normally!).  Note that any particular
+ * signal will only shut down one scheduler; applications should
+ * always only create a single scheduler.
+ *
+ * @param driver drive to use for the event loop
+ * @param task task to run first (and immediately)
+ * @param task_cls closure of @a task
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+int
+GNUNET_SCHEDULER_run_with_driver (const struct GNUNET_SCHEDULER_Driver *driver,
+                                  GNUNET_SCHEDULER_TaskCallback task,
+                                  void *task_cls)
+{
+  int ret;
+  struct GNUNET_SIGNAL_Context *shc_int;
+  struct GNUNET_SIGNAL_Context *shc_term;
+#if (SIGTERM != GNUNET_TERM_SIG)
+  struct GNUNET_SIGNAL_Context *shc_gterm;
+#endif
+#ifndef MINGW
+  struct GNUNET_SIGNAL_Context *shc_quit;
+  struct GNUNET_SIGNAL_Context *shc_hup;
+  struct GNUNET_SIGNAL_Context *shc_pipe;
+#endif
+  struct GNUNET_SCHEDULER_Task tsk;
+  const struct GNUNET_DISK_FileHandle *pr;
+  struct GNUNET_SCHEDULER_Handle sh;
+
+  /* general set-up */
+  GNUNET_assert (NULL == active_task);
+  GNUNET_assert (NULL == shutdown_pipe_handle);
+  shutdown_pipe_handle = GNUNET_DISK_pipe (GNUNET_NO,
+                                           GNUNET_NO,
+                                           GNUNET_NO,
+                                           GNUNET_NO);
+  GNUNET_assert (NULL != shutdown_pipe_handle);
+  pr = GNUNET_DISK_pipe_handle (shutdown_pipe_handle,
+                                GNUNET_DISK_PIPE_END_READ);
+  GNUNET_assert (NULL != pr);
+  my_pid = getpid ();
+
+  /* install signal handlers */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Registering signal handlers\n");
+  shc_int = GNUNET_SIGNAL_handler_install (SIGINT,
+                                           &sighandler_shutdown);
+  shc_term = GNUNET_SIGNAL_handler_install (SIGTERM,
+                                            &sighandler_shutdown);
+#if (SIGTERM != GNUNET_TERM_SIG)
+  shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG,
+                                             &sighandler_shutdown);
+#endif
+#ifndef MINGW
+  shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE,
+                                            &sighandler_pipe);
+  shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT,
+                                            &sighandler_shutdown);
+  shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP,
+                                           &sighandler_shutdown);
+#endif
+
+  /* Setup initial tasks */
+  current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT;
+  current_lifeness = GNUNET_YES;
+  memset (&tsk,
+          0,
+          sizeof (tsk));
+  active_task = &tsk;
+  tsk.sh = &sh;
+  GNUNET_SCHEDULER_add_with_reason_and_priority (task,
+                                                 task_cls,
+                                                 GNUNET_SCHEDULER_REASON_STARTUP,
+                                                 GNUNET_SCHEDULER_PRIORITY_DEFAULT);
+  GNUNET_SCHEDULER_add_now_with_lifeness (GNUNET_NO,
+                                          &GNUNET_OS_install_parent_control_handler,
+                                          NULL);
+  active_task = NULL;
+  driver->set_wakeup (driver->cls,
+                      GNUNET_TIME_absolute_get ());
+
+  /* begin main event loop */
+  sh.rs = GNUNET_NETWORK_fdset_create ();
+  sh.ws = GNUNET_NETWORK_fdset_create ();
+  sh.driver = driver;
+  ret = driver->loop (driver->cls,
+                      &sh);
+  GNUNET_NETWORK_fdset_destroy (sh.rs);
+  GNUNET_NETWORK_fdset_destroy (sh.ws);
+
+  /* uninstall signal handlers */
+  GNUNET_SIGNAL_handler_uninstall (shc_int);
+  GNUNET_SIGNAL_handler_uninstall (shc_term);
+#if (SIGTERM != GNUNET_TERM_SIG)
+  GNUNET_SIGNAL_handler_uninstall (shc_gterm);
+#endif
+#ifndef MINGW
+  GNUNET_SIGNAL_handler_uninstall (shc_pipe);
+  GNUNET_SIGNAL_handler_uninstall (shc_quit);
+  GNUNET_SIGNAL_handler_uninstall (shc_hup);
+#endif
+  GNUNET_DISK_pipe_close (shutdown_pipe_handle);
+  shutdown_pipe_handle = NULL;
+  return ret;
+}
+
+
+/**
+ * Obtain the driver for using select() as the event loop.
+ *
+ * @return NULL on error
+ */
+const struct GNUNET_SCHEDULER_Driver *
+GNUNET_SCHEDULER_driver_select ()
+{
+  GNUNET_break (0); // not implemented
+  return NULL;
+}
+
+
 /* end of scheduler.c */
diff --git a/src/util/server_nc.c b/src/util/server_nc.c
deleted file mode 100644
index a95cd7f6d..000000000
--- a/src/util/server_nc.c
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2010 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/server_nc.c
- * @brief convenience functions for transmission of
- *        a notification stream
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util-server-nc", __VA_ARGS__)
-
-
-/**
- * Entry in list of messages pending to be transmitted.
- */
-struct PendingMessageList
-{
-
-  /**
-   * This is a doubly-linked list.
-   */
-  struct PendingMessageList *next;
-
-  /**
-   * This is a doubly-linked list.
-   */
-  struct PendingMessageList *prev;
-
-  /**
-   * Message to transmit (allocated at the end of this
-   * struct, do not free)
-   */
-  const struct GNUNET_MessageHeader *msg;
-
-  /**
-   * Can this message be dropped?
-   */
-  int can_drop;
-
-};
-
-
-/**
- * Lists of clients we manage for notifications.
- */
-struct ClientList
-{
-
-  /**
-   * This is a doubly linked list.
-   */
-  struct ClientList *next;
-
-  /**
-   * This is a doubly linked list.
-   */
-  struct ClientList *prev;
-
-  /**
-   * Overall context this client belongs to.
-   */
-  struct GNUNET_SERVER_NotificationContext *nc;
-
-  /**
-   * Handle to the client.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Handle for pending transmission request to the client (or NULL).
-   */
-  struct GNUNET_SERVER_TransmitHandle *th;
-
-  /**
-   * Head of linked list of requests queued for transmission.
-   */
-  struct PendingMessageList *pending_head;
-
-  /**
-   * Tail of linked list of requests queued for transmission.
-   */
-  struct PendingMessageList *pending_tail;
-
-  /**
-   * Number of messages currently in the list.
-   */
-  unsigned int num_pending;
-
-};
-
-
-/**
- * The notification context is the key datastructure for a convenience
- * API used for transmission of notifications to the client until the
- * client disconnects (or the notification context is destroyed, in
- * which case we disconnect these clients).  Essentially, all
- * (notification) messages are queued up until the client is able to
- * read them.
- */
-struct GNUNET_SERVER_NotificationContext
-{
-
-  /**
-   * Server we do notifications for.
-   */
-  struct GNUNET_SERVER_Handle *server;
-
-  /**
-   * Head of list of clients receiving notifications.
-   */
-  struct ClientList *clients_head;
-
-  /**
-   * Tail of list of clients receiving notifications.
-   */
-  struct ClientList *clients_tail;
-
-  /**
-   * Maximum number of optional messages to queue per client.
-   */
-  unsigned int queue_length;
-
-};
-
-
-/**
- * Client has disconnected, clean up.
- *
- * @param cls our `struct GNUNET_SERVER_NotificationContext *`
- * @param client handle of client that disconnected
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_SERVER_NotificationContext *nc = cls;
-  struct ClientList *pos;
-  struct PendingMessageList *pml;
-
-  if (NULL == client)
-  {
-    nc->server = NULL;
-    return;
-  }
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    if (pos->client == client)
-      break;
-  if (NULL == pos)
-    return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Client disconnected, cleaning up %u messages in NC queue\n",
-       pos->num_pending);
-  GNUNET_CONTAINER_DLL_remove (nc->clients_head,
-                               nc->clients_tail,
-                               pos);
-  while (NULL != (pml = pos->pending_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (pos->pending_head,
-                                 pos->pending_tail,
-                                 pml);
-    GNUNET_free (pml);
-    pos->num_pending--;
-  }
-  if (NULL != pos->th)
-  {
-    GNUNET_SERVER_notify_transmit_ready_cancel (pos->th);
-    pos->th = NULL;
-  }
-  GNUNET_SERVER_client_drop (client);
-  GNUNET_assert (0 == pos->num_pending);
-  GNUNET_free (pos);
-}
-
-
-/**
- * Create a new notification context.
- *
- * @param server server for which this function creates the context
- * @param queue_length maximum number of messages to keep in
- *        the notification queue; optional messages are dropped
- *        if the queue gets longer than this number of messages
- * @return handle to the notification context
- */
-struct GNUNET_SERVER_NotificationContext *
-GNUNET_SERVER_notification_context_create (struct GNUNET_SERVER_Handle *server,
-                                           unsigned int queue_length)
-{
-  struct GNUNET_SERVER_NotificationContext *ret;
-
-  ret = GNUNET_new (struct GNUNET_SERVER_NotificationContext);
-  ret->server = server;
-  ret->queue_length = queue_length;
-  GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect,
-                                   ret);
-  return ret;
-}
-
-
-/**
- * Destroy the context, force disconnect for all clients.
- *
- * @param nc context to destroy.
- */
-void
-GNUNET_SERVER_notification_context_destroy (struct GNUNET_SERVER_NotificationContext *nc)
-{
-  struct ClientList *pos;
-  struct PendingMessageList *pml;
-
-  while (NULL != (pos = nc->clients_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (nc->clients_head,
-                                 nc->clients_tail,
-                                 pos);
-    if (NULL != pos->th)
-    {
-      GNUNET_SERVER_notify_transmit_ready_cancel (pos->th);
-      pos->th = NULL;
-    }
-    GNUNET_SERVER_client_drop (pos->client);
-    while (NULL != (pml = pos->pending_head))
-    {
-      GNUNET_CONTAINER_DLL_remove (pos->pending_head,
-                                   pos->pending_tail,
-                                   pml);
-      GNUNET_free (pml);
-      pos->num_pending--;
-    }
-    GNUNET_assert (0 == pos->num_pending);
-    GNUNET_free (pos);
-  }
-  if (NULL != nc->server)
-    GNUNET_SERVER_disconnect_notify_cancel (nc->server,
-                                            &handle_client_disconnect,
-                                            nc);
-  GNUNET_free (nc);
-}
-
-
-/**
- * Add a client to the notification context.
- *
- * @param nc context to modify
- * @param client client to add
- */
-void
-GNUNET_SERVER_notification_context_add (struct GNUNET_SERVER_NotificationContext *nc,
-                                        struct GNUNET_SERVER_Client *client)
-{
-  struct ClientList *cl;
-
-  for (cl = nc->clients_head; NULL != cl; cl = cl->next)
-    if (cl->client == client)
-      return; /* already present */
-  cl = GNUNET_new (struct ClientList);
-  GNUNET_CONTAINER_DLL_insert (nc->clients_head,
-                               nc->clients_tail,
-                               cl);
-  cl->nc = nc;
-  cl->client = client;
-  GNUNET_SERVER_client_keep (client);
-}
-
-
-/**
- * Function called to notify a client about the socket begin ready to
- * queue more data.  @a buf will be NULL and @a size zero if the socket
- * was closed for writing in the meantime.
- *
- * @param cls the `struct ClientList *`
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
- */
-static size_t
-transmit_message (void *cls,
-                  size_t size,
-                  void *buf)
-{
-  struct ClientList *cl = cls;
-  char *cbuf = buf;
-  struct PendingMessageList *pml;
-  uint16_t msize;
-  size_t ret;
-
-  cl->th = NULL;
-  if (NULL == buf)
-  {
-    /* 'cl' should be freed via disconnect notification shortly */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Failed to transmit message from NC queue to client\n");
-    return 0;
-  }
-  ret = 0;
-  while (NULL != (pml = cl->pending_head))
-  {
-    msize = ntohs (pml->msg->size);
-    if (size < msize)
-      break;
-    GNUNET_CONTAINER_DLL_remove (cl->pending_head,
-                                 cl->pending_tail,
-                                 pml);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Copying message of type %u and size %u from pending queue to transmission buffer\n",
-         ntohs (pml->msg->type),
-         msize);
-    GNUNET_memcpy (&cbuf[ret], pml->msg, msize);
-    ret += msize;
-    size -= msize;
-    GNUNET_free (pml);
-    cl->num_pending--;
-  }
-  if (NULL != pml)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Have %u messages left in NC queue, will try transmission again\n",
-         cl->num_pending);
-    cl->th =
-        GNUNET_SERVER_notify_transmit_ready (cl->client,
-                                             ntohs (pml->msg->size),
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             &transmit_message, cl);
-  }
-  else
-  {
-    GNUNET_assert (0 == cl->num_pending);
-  }
-  return ret;
-}
-
-
-/**
- * Send a message to a particular client.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-static void
-do_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-            struct ClientList *client,
-            const struct GNUNET_MessageHeader *msg,
-            int can_drop)
-{
-  struct PendingMessageList *pml;
-  uint16_t size;
-
-  if ( (client->num_pending > nc->queue_length) &&
-       (GNUNET_YES == can_drop) )
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Dropping message of type %u and size %u due to full queue (%u entries)\n",
-         ntohs (msg->type), ntohs (msg->size), (unsigned int) nc->queue_length);
-    return;                     /* drop! */
-  }
-  if (client->num_pending > nc->queue_length)
-  {
-    /* FIXME: consider checking for other messages in the
-     * queue that are 'droppable' */
-  }
-  client->num_pending++;
-  size = ntohs (msg->size);
-  pml = GNUNET_malloc (sizeof (struct PendingMessageList) + size);
-  pml->msg = (const struct GNUNET_MessageHeader *) &pml[1];
-  pml->can_drop = can_drop;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding message of type %u and size %u to pending queue (which has %u entries)\n",
-       ntohs (msg->type),
-       ntohs (msg->size),
-       (unsigned int) nc->queue_length);
-  GNUNET_memcpy (&pml[1], msg, size);
-  /* append */
-  GNUNET_CONTAINER_DLL_insert_tail (client->pending_head,
-                                    client->pending_tail,
-                                    pml);
-  if (NULL == client->th)
-    client->th =
-        GNUNET_SERVER_notify_transmit_ready (client->client,
-                                             ntohs (client->pending_head->
-                                                    msg->size),
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             &transmit_message, client);
-}
-
-
-/**
- * Send a message to a particular client; must have
- * already been added to the notification context.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-                                            struct GNUNET_SERVER_Client *client,
-                                            const struct GNUNET_MessageHeader *msg,
-                                            int can_drop)
-{
-  struct ClientList *pos;
-
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    if (pos->client == client)
-      break;
-  GNUNET_assert (NULL != pos);
-  do_unicast (nc, pos, msg, can_drop);
-}
-
-
-/**
- * Send a message to all clients of this context.
- *
- * @param nc context to modify
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_broadcast (struct
-                                              GNUNET_SERVER_NotificationContext *nc,
-                                              const struct GNUNET_MessageHeader *msg,
-                                              int can_drop)
-{
-  struct ClientList *pos;
-
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    do_unicast (nc, pos, msg, can_drop);
-}
-
-
-/**
- * Return active number of subscribers in this context.
- *
- * @param nc context to query
- * @return number of current subscribers
- */
-unsigned int
-GNUNET_SERVER_notification_context_get_size (struct GNUNET_SERVER_NotificationContext *nc)
-{
-  unsigned int num;
-  struct ClientList *pos;
-
-  num = 0;
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    num++;
-  return num;
-}
-
-/* end of server_nc.c */
diff --git a/src/util/server_tc.c b/src/util/server_tc.c
deleted file mode 100644
index 40a8ba015..000000000
--- a/src/util/server_tc.c
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/server_tc.c
- * @brief convenience functions for transmission of
- *        complex responses as a server
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-
-
-/**
- * How much buffer space do we want to have at least
- * before transmitting another increment?
- */
-#define MIN_BLOCK_SIZE 128
-
-
-
-struct GNUNET_SERVER_TransmitContext
-{
-  /**
-   * Which client are we transmitting to?
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Transmission buffer. (current offset for writing).
-   */
-  char *buf;
-
-  /**
-   * Number of bytes in buf.
-   */
-  size_t total;
-
-  /**
-   * Offset for writing in buf.
-   */
-  size_t off;
-
-  /**
-   * Timeout for this request.
-   */
-  struct GNUNET_TIME_Absolute timeout;
-};
-
-
-/**
- * Helper function for incremental transmission of the response.
- */
-static size_t
-transmit_response (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_SERVER_TransmitContext *tc = cls;
-  size_t msize;
-
-  if (NULL == buf)
-  {
-    GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-    return 0;
-  }
-  if (tc->total - tc->off > size)
-    msize = size;
-  else
-    msize = tc->total - tc->off;
-  GNUNET_memcpy (buf, &tc->buf[tc->off], msize);
-  tc->off += msize;
-  if (tc->total == tc->off)
-  {
-    GNUNET_SERVER_receive_done (tc->client, GNUNET_OK);
-    GNUNET_SERVER_client_drop (tc->client);
-    GNUNET_free_non_null (tc->buf);
-    GNUNET_free (tc);
-  }
-  else
-  {
-    if (NULL ==
-        GNUNET_SERVER_notify_transmit_ready (tc->client,
-                                             GNUNET_MIN (MIN_BLOCK_SIZE,
-                                                         tc->total - tc->off),
-                                             GNUNET_TIME_absolute_get_remaining
-                                             (tc->timeout), &transmit_response,
-                                             tc))
-    {
-      GNUNET_break (0);
-      GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-    }
-  }
-  return msize;
-}
-
-
-/**
- * Create a new transmission context for the
- * given client.
- *
- * @param client client to create the context for.
- * @return NULL on error
- */
-struct GNUNET_SERVER_TransmitContext *
-GNUNET_SERVER_transmit_context_create (struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_SERVER_TransmitContext *tc;
-
-  GNUNET_assert (NULL != client);
-  tc = GNUNET_new (struct GNUNET_SERVER_TransmitContext);
-  GNUNET_SERVER_client_keep (client);
-  tc->client = client;
-  return tc;
-}
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param data what to append to the result message
- * @param length length of data
- * @param type type of the message
- */
-void
-GNUNET_SERVER_transmit_context_append_data (struct GNUNET_SERVER_TransmitContext
-                                            *tc, const void *data,
-                                            size_t length, uint16_t type)
-{
-  struct GNUNET_MessageHeader *msg;
-  size_t size;
-
-  GNUNET_assert (length < GNUNET_SERVER_MAX_MESSAGE_SIZE);
-  size = length + sizeof (struct GNUNET_MessageHeader);
-  GNUNET_assert (size > length);
-  tc->buf = GNUNET_realloc (tc->buf, tc->total + size);
-  msg = (struct GNUNET_MessageHeader *) &tc->buf[tc->total];
-  tc->total += size;
-  msg->size = htons (size);
-  msg->type = htons (type);
-  GNUNET_memcpy (&msg[1], data, length);
-}
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param msg message to append
- */
-void
-GNUNET_SERVER_transmit_context_append_message (struct
-                                               GNUNET_SERVER_TransmitContext
-                                               *tc,
-                                               const struct GNUNET_MessageHeader
-                                               *msg)
-{
-  struct GNUNET_MessageHeader *m;
-  uint16_t size;
-
-  size = ntohs (msg->size);
-  tc->buf = GNUNET_realloc (tc->buf, tc->total + size);
-  m = (struct GNUNET_MessageHeader *) &tc->buf[tc->total];
-  tc->total += size;
-  GNUNET_memcpy (m, msg, size);
-}
-
-
-/**
- * Execute a transmission context.  If there is
- * an error in the transmission, the #GNUNET_SERVER_receive_done()
- * method will be called with an error code (#GNUNET_SYSERR),
- * otherwise with #GNUNET_OK.
- *
- * @param tc transmission context to use
- * @param timeout when to time out and abort the transmission
- */
-void
-GNUNET_SERVER_transmit_context_run (struct GNUNET_SERVER_TransmitContext *tc,
-                                    struct GNUNET_TIME_Relative timeout)
-{
-  tc->timeout = GNUNET_TIME_relative_to_absolute (timeout);
-  if (NULL ==
-      GNUNET_SERVER_notify_transmit_ready (tc->client,
-                                           GNUNET_MIN (MIN_BLOCK_SIZE,
-                                                       tc->total), timeout,
-                                           &transmit_response, tc))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-  }
-}
-
-
-/**
- * Destroy a transmission context. This function must not be called
- * after 'GNUNET_SERVER_transmit_context_run'.
- *
- * @param tc transmission context to destroy
- * @param success code to give to 'GNUNET_SERVER_receive_done' for
- *        the client:  GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                GNUNET_NO to close the connection (normal behavior)
- *                GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_transmit_context_destroy (struct GNUNET_SERVER_TransmitContext
-                                        *tc, int success)
-{
-  GNUNET_SERVER_receive_done (tc->client, success);
-  GNUNET_SERVER_client_drop (tc->client);
-  GNUNET_free_non_null (tc->buf);
-  GNUNET_free (tc);
-}
-
-
-/* end of server_tc.c */
diff --git a/src/util/service.c b/src/util/service.c
index 3beb4a085..f63737e56 100644
--- a/src/util/service.c
+++ b/src/util/service.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009, 2012 GNUnet e.V.
+     Copyright (C) 2016 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,9 +19,10 @@
 */
 
 /**
- * @file util/service.c
- * @brief functions related to starting services
+ * @file util/service_new.c
+ * @brief functions related to starting services (redesign)
  * @author Christian Grothoff
+ * @author Florian Dold
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
@@ -36,84 +37,51 @@
 #endif
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-service", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-service", syscall)
 
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
+#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-service", syscall, filename)
 
 
-/* ******************* access control ******************** */
-
 /**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param add the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ * Information the service tracks per listen operation.
  */
-static int
-check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
-                   const struct in_addr *add)
+struct ServiceListenContext
 {
-  unsigned int i;
 
-  if (NULL == list)
-    return GNUNET_NO;
-  i = 0;
-  while ((list[i].network.s_addr != 0) || (list[i].netmask.s_addr != 0))
-  {
-    if ((add->s_addr & list[i].netmask.s_addr) ==
-        (list[i].network.s_addr & list[i].netmask.s_addr))
-      return GNUNET_YES;
-    i++;
-  }
-  return GNUNET_NO;
-}
+  /**
+   * Kept in a DLL.
+   */
+  struct ServiceListenContext *next;
 
+  /**
+   * Kept in a DLL.
+   */
+  struct ServiceListenContext *prev;
 
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param ip the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
-                   const struct in6_addr *ip)
-{
-  unsigned int i;
-  unsigned int j;
-  struct in6_addr zero;
+  /**
+   * Service this listen context belongs to.
+   */
+  struct GNUNET_SERVICE_Handle *sh;
 
-  if (NULL == list)
-    return GNUNET_NO;
-  memset (&zero, 0, sizeof (struct in6_addr));
-  i = 0;
-NEXT:
-  while (0 != memcmp (&zero, &list[i].network, sizeof (struct in6_addr)))
-  {
-    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
-      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
-          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
-      {
-        i++;
-        goto NEXT;
-      }
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
+  /**
+   * Socket we are listening on.
+   */
+  struct GNUNET_NETWORK_Handle *listen_socket;
 
+  /**
+   * Task scheduled to do the listening.
+   */
+  struct GNUNET_SCHEDULER_Task *listen_task;
 
-/* ****************** service struct ****************** */
+};
 
 
 /**
- * Context for "service_task".
+ * Handle to a service.
  */
-struct GNUNET_SERVICE_Context
+struct GNUNET_SERVICE_Handle
 {
   /**
    * Our configuration.
@@ -121,25 +89,54 @@ struct GNUNET_SERVICE_Context
   const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
-   * Handle for the server.
+   * Name of our service.
    */
-  struct GNUNET_SERVER_Handle *server;
+  const char *service_name;
 
   /**
-   * NULL-terminated array of addresses to bind to, NULL if we got pre-bound
-   * listen sockets.
+   * Main service-specific task to run.
    */
-  struct sockaddr **addrs;
+  GNUNET_SERVICE_InitCallback service_init_cb;
 
   /**
-   * Name of our service.
+   * Function to call when clients connect.
    */
-  const char *service_name;
+  GNUNET_SERVICE_ConnectHandler connect_cb;
 
   /**
-   * Main service-specific task to run.
+   * Function to call when clients disconnect / are disconnected.
    */
-  GNUNET_SERVICE_Main task;
+  GNUNET_SERVICE_DisconnectHandler disconnect_cb;
+
+  /**
+   * Closure for @e service_init_cb, @e connect_cb, @e disconnect_cb.
+   */
+  void *cb_cls;
+
+  /**
+   * DLL of listen sockets used to accept new connections.
+   */
+  struct ServiceListenContext *slc_head;
+
+  /**
+   * DLL of listen sockets used to accept new connections.
+   */
+  struct ServiceListenContext *slc_tail;
+
+  /**
+   * Our clients, kept in a DLL.
+   */
+  struct GNUNET_SERVICE_Client *clients_head;
+
+  /**
+   * Our clients, kept in a DLL.
+   */
+  struct GNUNET_SERVICE_Client *clients_tail;
+
+  /**
+   * Message handlers to use for all clients.
+   */
+  struct GNUNET_MQ_MessageHandler *handlers;
 
   /**
    * Closure for @e task.
@@ -169,30 +166,38 @@ struct GNUNET_SERVICE_Context
   struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
 
   /**
-   * My (default) message handlers.  Adjusted copy
-   * of "defhandlers".
+   * Do we require a matching UID for UNIX domain socket connections?
+   * #GNUNET_NO means that the UID does not have to match (however,
+   * @e match_gid may still impose other access control checks).
    */
-  struct GNUNET_SERVER_MessageHandler *my_handlers;
+  int match_uid;
 
   /**
-   * Array of the lengths of the entries in addrs.
+   * Do we require a matching GID for UNIX domain socket connections?
+   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
+   * checking that the client's UID is in our group OR that the
+   * client's GID is our GID.  If both "match_gid" and @e match_uid are
+   * #GNUNET_NO, all users on the local system have access.
    */
-  socklen_t *addrlens;
+  int match_gid;
 
   /**
-   * NULL-terminated array of listen sockets we should take over.
+   * Set to #GNUNET_YES if we got a shutdown signal and terminate
+   * the service if #have_non_monitor_clients() returns #GNUNET_YES.
    */
-  struct GNUNET_NETWORK_Handle **lsocks;
+  int got_shutdown;
 
   /**
-   * Task ID of the shutdown task.
+   * Our options.
    */
-  struct GNUNET_SCHEDULER_Task *shutdown_task;
+  enum GNUNET_SERVICE_Options options;
 
   /**
-   * Idle timeout for server.
+   * If we are daemonizing, this FD is set to the
+   * pipe to the parent.  Send '.' if we started
+   * ok, '!' if not.  -1 if we are not daemonizing.
    */
-  struct GNUNET_TIME_Relative timeout;
+  int ready_confirm_fd;
 
   /**
    * Overall success/failure of the service start.
@@ -200,182 +205,203 @@ struct GNUNET_SERVICE_Context
   int ret;
 
   /**
-   * If we are daemonizing, this FD is set to the
-   * pipe to the parent.  Send '.' if we started
-   * ok, '!' if not.  -1 if we are not daemonizing.
+   * If #GNUNET_YES, consider unknown message types an error where the
+   * client is disconnected.
    */
-  int ready_confirm_fd;
+  int require_found;
+};
+
+
+/**
+ * Handle to a client that is connected to a service.
+ */
+struct GNUNET_SERVICE_Client
+{
 
   /**
-   * Do we close connections if we receive messages
-   * for which we have no handler?
+   * Kept in a DLL.
    */
-  int require_found;
+  struct GNUNET_SERVICE_Client *next;
 
   /**
-   * Do we require a matching UID for UNIX domain socket connections?
-   * #GNUNET_NO means that the UID does not have to match (however,
-   * @e match_gid may still impose other access control checks).
+   * Kept in a DLL.
    */
-  int match_uid;
+  struct GNUNET_SERVICE_Client *prev;
 
   /**
-   * Do we require a matching GID for UNIX domain socket connections?
-   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
-   * checking that the client's UID is in our group OR that the
-   * client's GID is our GID.  If both "match_gid" and @e match_uid are
-   * #GNUNET_NO, all users on the local system have access.
+   * Service that this client belongs to.
    */
-  int match_gid;
+  struct GNUNET_SERVICE_Handle *sh;
 
   /**
-   * Our options.
+   * Socket of this client.
    */
-  enum GNUNET_SERVICE_Options options;
+  struct GNUNET_NETWORK_Handle *sock;
 
-};
+  /**
+   * Message queue for the client.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Tokenizer we use for processing incoming data.
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
+   * Task that warns about missing calls to
+   * #GNUNET_SERVICE_client_continue().
+   */
+  struct GNUNET_SCHEDULER_Task *warn_task;
 
+  /**
+   * Task run to finish dropping the client after the stack has
+   * properly unwound.
+   */
+  struct GNUNET_SCHEDULER_Task *drop_task;
+
+  /**
+   * Task that receives data from the client to
+   * pass it to the handlers.
+   */
+  struct GNUNET_SCHEDULER_Task *recv_task;
+
+  /**
+   * Task that transmit data to the client.
+   */
+  struct GNUNET_SCHEDULER_Task *send_task;
+
+  /**
+   * Pointer to the message to be transmitted by @e send_task.
+   */
+  const struct GNUNET_MessageHeader *msg;
+
+  /**
+   * User context value, value returned from
+   * the connect callback.
+   */
+  void *user_context;
+
+  /**
+   * Time when we last gave a message from this client
+   * to the application.
+   */
+  struct GNUNET_TIME_Absolute warn_start;
+
+  /**
+   * Current position in @e msg at which we are transmitting.
+   */
+  size_t msg_pos;
+
+  /**
+   * Persist the file handle for this client no matter what happens,
+   * force the OS to close once the process actually dies.  Should only
+   * be used in special cases!
+   */
+  int persist;
+
+  /**
+   * Is this client a 'monitor' client that should not be counted
+   * when deciding on destroying the server during soft shutdown?
+   * (see also #GNUNET_SERVICE_start)
+   */
+  int is_monitor;
+
+  /**
+   * Are we waiting for the application to call #GNUNET_SERVICE_client_continue()?
+   */
+  int needs_continue;
+
+  /**
+   * Type of last message processed (for warn_no_receive_done).
+   */
+  uint16_t warn_type;
+};
 
-/* ****************** message handlers ****************** */
 
 /**
- * Send a 'TEST' message back to the client.
+ * Check if any of the clients we have left are unrelated to
+ * monitoring.
  *
- * @param cls the 'struct GNUNET_SERVER_Client' to send TEST to
- * @param size number of bytes available in 'buf'
- * @param buf where to copy the message
- * @return number of bytes written to 'buf'
+ * @param sh service to check clients for
+ * @return #GNUNET_YES if we have non-monitoring clients left
  */
-static size_t
-write_test (void *cls, size_t size, void *buf)
+static int
+have_non_monitor_clients (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVER_Client *client = cls;
-  struct GNUNET_MessageHeader *msg;
+  struct GNUNET_SERVICE_Client *client;
 
-  if (size < sizeof (struct GNUNET_MessageHeader))
+  for (client = sh->clients_head;NULL != client; client = client->next)
   {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return 0;                   /* client disconnected */
+    if (client->is_monitor)
+      continue;
+    return GNUNET_YES;
   }
-  msg = (struct GNUNET_MessageHeader *) buf;
-  msg->type = htons (GNUNET_MESSAGE_TYPE_TEST);
-  msg->size = htons (sizeof (struct GNUNET_MessageHeader));
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-  return sizeof (struct GNUNET_MessageHeader);
+  return GNUNET_NO;
 }
 
 
 /**
- * Handler for TEST message.
+ * Shutdown task triggered when a service should be terminated.
+ * This considers active clients and the service options to see
+ * how this specific service is to be terminated, and depending
+ * on this proceeds with the shutdown logic.
  *
- * @param cls closure (refers to service)
- * @param client identification of the client
- * @param message the actual message
+ * @param cls our `struct GNUNET_SERVICE_Handle`
  */
 static void
-handle_test (void *cls, struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
+service_shutdown (void *cls)
 {
-  /* simply bounce message back to acknowledge */
-  if (NULL ==
-      GNUNET_SERVER_notify_transmit_ready (client,
-                                           sizeof (struct GNUNET_MessageHeader),
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           &write_test, client))
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-}
-
-
-/**
- * Default handlers for all services.  Will be copied and the
- * "callback_cls" fields will be replaced with the specific service
- * struct.
- */
-static const struct GNUNET_SERVER_MessageHandler defhandlers[] = {
-  {&handle_test, NULL, GNUNET_MESSAGE_TYPE_TEST,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-/* ****************** service core routines ************** */
+  struct GNUNET_SERVICE_Handle *sh = cls;
 
-
-/**
- * Check if access to the service is allowed from the given address.
- *
- * @param cls closure
- * @param uc credentials, if available, otherwise NULL
- * @param addr address
- * @param addrlen length of address
- * @return #GNUNET_YES to allow, #GNUNET_NO to deny, #GNUNET_SYSERR
- *   for unknown address family (will be denied).
- */
-static int
-check_access (void *cls, const struct GNUNET_CONNECTION_Credentials *uc,
-              const struct sockaddr *addr, socklen_t addrlen)
-{
-  struct GNUNET_SERVICE_Context *sctx = cls;
-  const struct sockaddr_in *i4;
-  const struct sockaddr_in6 *i6;
-  int ret;
-
-  switch (addr->sa_family)
+  switch (sh->options)
   {
-  case AF_INET:
-    GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-    i4 = (const struct sockaddr_in *) addr;
-    ret = ((NULL == sctx->v4_allowed) ||
-           (check_ipv4_listed (sctx->v4_allowed, &i4->sin_addr))) &&
-        ((NULL == sctx->v4_denied) ||
-         (!check_ipv4_listed (sctx->v4_denied, &i4->sin_addr)));
+  case GNUNET_SERVICE_OPTION_NONE:
+    GNUNET_SERVICE_shutdown (sh);
     break;
-  case AF_INET6:
-    GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
-    i6 = (const struct sockaddr_in6 *) addr;
-    ret = ((NULL == sctx->v6_allowed) ||
-           (check_ipv6_listed (sctx->v6_allowed, &i6->sin6_addr))) &&
-        ((NULL == sctx->v6_denied) ||
-         (!check_ipv6_listed (sctx->v6_denied, &i6->sin6_addr)));
+  case GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN:
+    /* This task should never be run if we are using
+       the manual shutdown. */
+    GNUNET_assert (0);
     break;
-#ifndef WINDOWS
-  case AF_UNIX:
-    ret = GNUNET_OK;            /* controlled using file-system ACL now */
+  case GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN:
+    sh->got_shutdown = GNUNET_YES;
+    GNUNET_SERVICE_suspend (sh);
+    if (GNUNET_NO == have_non_monitor_clients (sh))
+      GNUNET_SERVICE_shutdown (sh);
     break;
-#endif
-  default:
-    LOG (GNUNET_ERROR_TYPE_WARNING, _("Unknown address family %d\n"),
-         addr->sa_family);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_OK != ret)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Access from `%s' denied to service `%s'\n"),
-         GNUNET_a2s (addr, addrlen),
-         sctx->service_name);
   }
-  return ret;
 }
 
 
 /**
- * Get the name of the file where we will
- * write the PID of the service.
+ * First task run by any service.  Initializes our shutdown task,
+ * starts the listening operation on our listen sockets and launches
+ * the custom logic of the application service.
  *
- * @param sctx service context
- * @return name of the file for the process ID
+ * @param cls our `struct GNUNET_SERVICE_Handle`
  */
-static char *
-get_pid_file_name (struct GNUNET_SERVICE_Context *sctx)
+static void
+service_main (void *cls)
 {
-  char *pif;
+  struct GNUNET_SERVICE_Handle *sh = cls;
 
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
-                                               "PIDFILE", &pif))
-    return NULL;
-  return pif;
+  if (GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN != sh->options)
+    GNUNET_SCHEDULER_add_shutdown (&service_shutdown,
+                                   sh);
+  GNUNET_SERVICE_resume (sh);
+
+  if (-1 != sh->ready_confirm_fd)
+  {
+    GNUNET_break (1 == WRITE (sh->ready_confirm_fd, ".", 1));
+    GNUNET_break (0 == CLOSE (sh->ready_confirm_fd));
+    sh->ready_confirm_fd = -1;
+  }
+
+  if (NULL != sh->service_init_cb)
+    sh->service_init_cb (sh->cb_cls,
+                         sh->cfg,
+                         sh);
 }
 
 
@@ -383,32 +409,37 @@ get_pid_file_name (struct GNUNET_SERVICE_Context *sctx)
  * Parse an IPv4 access control list.
  *
  * @param ret location where to write the ACL (set)
- * @param sctx service context to use to get the configuration
+ * @param sh service context to use to get the configuration
  * @param option name of the ACL option to parse
  * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
  *         no ACL configured)
  */
 static int
 process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Context *sctx,
+              struct GNUNET_SERVICE_Handle *sh,
               const char *option)
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
+                                         sh->service_name,
+                                         option))
   {
     *ret = NULL;
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->service_name,
-                                                       option, &opt));
+                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
+                                                       sh->service_name,
+                                                       option,
+                                                       &opt));
   if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->service_name, option);
+         opt,
+         sh->service_name,
+         option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -421,32 +452,37 @@ process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
  * Parse an IPv6 access control list.
  *
  * @param ret location where to write the ACL (set)
- * @param sctx service context to use to get the configuration
+ * @param sh service context to use to get the configuration
  * @param option name of the ACL option to parse
  * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
  *         no ACL configured)
  */
 static int
 process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Context *sctx,
+              struct GNUNET_SERVICE_Handle *sh,
               const char *option)
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
+                                         sh->service_name,
+                                         option))
   {
     *ret = NULL;
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->service_name,
-                                                       option, &opt));
+                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
+                                                       sh->service_name,
+                                                       option,
+                                                       &opt));
   if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->service_name, option);
+         opt,
+         sh->service_name,
+         option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -476,12 +512,14 @@ add_unixpath (struct sockaddr **saddrs,
 
   un = GNUNET_new (struct sockaddr_un);
   un->sun_family = AF_UNIX;
-  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
+  strncpy (un->sun_path,
+           unixpath,
+           sizeof (un->sun_path) - 1);
 #ifdef LINUX
   if (GNUNET_YES == abstract)
     un->sun_path[0] = '\0';
 #endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
+#if HAVE_SOCKADDR_UN_SUN_LEN
   un->sun_len = (u_char) sizeof (struct sockaddr_un);
 #endif
   *saddrs = (struct sockaddr *) un;
@@ -514,11 +552,11 @@ add_unixpath (struct sockaddr **saddrs,
  *              zero (in this case, `*addrs` and `*addr_lens` will be
  *              set to NULL).
  */
-int
-GNUNET_SERVICE_get_server_addresses (const char *service_name,
-                                     const struct GNUNET_CONFIGURATION_Handle *cfg,
-                                     struct sockaddr ***addrs,
-                                     socklen_t ** addr_lens)
+static int
+get_server_addresses (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      struct sockaddr ***addrs,
+                      socklen_t **addr_lens)
 {
   int disablev6;
   struct GNUNET_NETWORK_Handle *desc;
@@ -539,11 +577,15 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   *addrs = NULL;
   *addr_lens = NULL;
   desc = NULL;
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "DISABLEV6"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "DISABLEV6"))
   {
     if (GNUNET_SYSERR ==
         (disablev6 =
-         GNUNET_CONFIGURATION_get_value_yesno (cfg, service_name, "DISABLEV6")))
+         GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                               service_name,
+                                               "DISABLEV6")))
       return GNUNET_SYSERR;
   }
   else
@@ -552,33 +594,44 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   if (! disablev6)
   {
     /* probe IPv6 support */
-    desc = GNUNET_NETWORK_socket_create (PF_INET6, SOCK_STREAM, 0);
+    desc = GNUNET_NETWORK_socket_create (PF_INET6,
+                                         SOCK_STREAM,
+                                         0);
     if (NULL == desc)
     {
-      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
-          (EACCES == errno))
+      if ( (ENOBUFS == errno) ||
+           (ENOMEM == errno) ||
+           (ENFILE == errno) ||
+           (EACCES == errno) )
       {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                      "socket");
         return GNUNET_SYSERR;
       }
       LOG (GNUNET_ERROR_TYPE_INFO,
            _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
-           service_name, STRERROR (errno));
+           service_name,
+           STRERROR (errno));
       disablev6 = GNUNET_YES;
     }
     else
     {
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (desc));
       desc = NULL;
     }
   }
 
   port = 0;
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "PORT"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "PORT"))
   {
     if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_number (cfg, service_name,
-                                               "PORT", &port))
+        GNUNET_CONFIGURATION_get_value_number (cfg,
+                                               service_name,
+                                               "PORT",
+                                               &port))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Require valid port number for service `%s' in configuration!\n"),
@@ -593,11 +646,15 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     }
   }
 
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "BINDTO"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "BINDTO"))
   {
     GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_string (cfg, service_name,
-                                                         "BINDTO", &hostname));
+                  GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                         service_name,
+                                                         "BINDTO",
+                                                         &hostname));
   }
   else
     hostname = NULL;
@@ -606,10 +663,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   abstract = GNUNET_NO;
 #ifdef AF_UNIX
   if ((GNUNET_YES ==
-       GNUNET_CONFIGURATION_have_value (cfg, service_name, "UNIXPATH")) &&
+       GNUNET_CONFIGURATION_have_value (cfg,
+                                        service_name,
+                                        "UNIXPATH")) &&
       (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_filename (cfg, service_name, "UNIXPATH",
-                                              &unixpath)) &&
+       GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                                service_name,
+                                                "UNIXPATH",
+                                                &unixpath)) &&
       (0 < strlen (unixpath)))
   {
     /* probe UNIX support */
@@ -618,7 +679,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     if (strlen (unixpath) >= sizeof (s_un.sun_path))
     {
       LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("UNIXPATH `%s' too long, maximum length is %llu\n"), unixpath,
+           _("UNIXPATH `%s' too long, maximum length is %llu\n"),
+           unixpath,
            (unsigned long long) sizeof (s_un.sun_path));
       unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
       LOG (GNUNET_ERROR_TYPE_INFO,
@@ -632,22 +694,27 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     if (GNUNET_SYSERR == abstract)
       abstract = GNUNET_NO;
 #endif
-    if ((GNUNET_YES != abstract)
-        && (GNUNET_OK !=
-            GNUNET_DISK_directory_create_for_file (unixpath)))
+    if ( (GNUNET_YES != abstract) &&
+         (GNUNET_OK !=
+          GNUNET_DISK_directory_create_for_file (unixpath)) )
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                                 "mkdir",
                                 unixpath);
   }
   if (NULL != unixpath)
   {
-    desc = GNUNET_NETWORK_socket_create (AF_UNIX, SOCK_STREAM, 0);
+    desc = GNUNET_NETWORK_socket_create (AF_UNIX,
+                                         SOCK_STREAM,
+                                         0);
     if (NULL == desc)
     {
-      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+      if ((ENOBUFS == errno) ||
+          (ENOMEM == errno) ||
+          (ENFILE == errno) ||
           (EACCES == errno))
       {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                      "socket");
         GNUNET_free_non_null (hostname);
         GNUNET_free (unixpath);
         return GNUNET_SYSERR;
@@ -661,7 +728,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     }
     else
     {
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (desc));
       desc = NULL;
     }
   }
@@ -677,9 +745,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   }
   if (0 == port)
   {
-    saddrs = GNUNET_malloc (2 * sizeof (struct sockaddr *));
-    saddrlens = GNUNET_malloc (2 * sizeof (socklen_t));
-    add_unixpath (saddrs, saddrlens, unixpath, abstract);
+    saddrs = GNUNET_new_array (2,
+                               struct sockaddr *);
+    saddrlens = GNUNET_new_array (2,
+                                  socklen_t);
+    add_unixpath (saddrs,
+                  saddrlens,
+                  unixpath,
+                  abstract);
     GNUNET_free_non_null (unixpath);
     GNUNET_free_non_null (hostname);
     *addrs = saddrs;
@@ -693,11 +766,16 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
          "Resolving `%s' since that is where `%s' will bind to.\n",
          hostname,
          service_name);
-    memset (&hints, 0, sizeof (struct addrinfo));
+    memset (&hints,
+            0,
+            sizeof (struct addrinfo));
     if (disablev6)
       hints.ai_family = AF_INET;
     hints.ai_protocol = IPPROTO_TCP;
-    if ((0 != (ret = getaddrinfo (hostname, NULL, &hints, &res))) ||
+    if ((0 != (ret = getaddrinfo (hostname,
+                                  NULL,
+                                  &hints,
+                                  &res))) ||
         (NULL == res))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
@@ -713,7 +791,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     while (NULL != (pos = next))
     {
       next = pos->ai_next;
-      if ((disablev6) && (pos->ai_family == AF_INET6))
+      if ( (disablev6) &&
+           (pos->ai_family == AF_INET6) )
         continue;
       i++;
     }
@@ -731,32 +810,45 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     resi = i;
     if (NULL != unixpath)
       resi++;
-    saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-    saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+    saddrs = GNUNET_new_array (resi + 1,
+                               struct sockaddr *);
+    saddrlens = GNUNET_new_array (resi + 1,
+                                  socklen_t);
     i = 0;
     if (NULL != unixpath)
     {
-      add_unixpath (saddrs, saddrlens, unixpath, abstract);
+      add_unixpath (saddrs,
+                    saddrlens,
+                    unixpath,
+                    abstract);
       i++;
     }
     next = res;
     while (NULL != (pos = next))
     {
       next = pos->ai_next;
-      if ((disablev6) && (AF_INET6 == pos->ai_family))
+      if ( (disablev6) &&
+           (AF_INET6 == pos->ai_family) )
         continue;
-      if ((IPPROTO_TCP != pos->ai_protocol) && (0 != pos->ai_protocol))
+      if ( (IPPROTO_TCP != pos->ai_protocol) &&
+           (0 != pos->ai_protocol) )
         continue;               /* not TCP */
-      if ((SOCK_STREAM != pos->ai_socktype) && (0 != pos->ai_socktype))
+      if ( (SOCK_STREAM != pos->ai_socktype) &&
+           (0 != pos->ai_socktype) )
         continue;               /* huh? */
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Service `%s' will bind to `%s'\n",
-           service_name, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Service `%s' will bind to `%s'\n",
+           service_name,
+           GNUNET_a2s (pos->ai_addr,
+                       pos->ai_addrlen));
       if (AF_INET == pos->ai_family)
       {
         GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
         saddrlens[i] = pos->ai_addrlen;
         saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        GNUNET_memcpy (saddrs[i],
+                       pos->ai_addr,
+                       saddrlens[i]);
         ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
       }
       else
@@ -765,7 +857,9 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
         GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
         saddrlens[i] = pos->ai_addrlen;
         saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        GNUNET_memcpy (saddrs[i],
+                       pos->ai_addr,
+                       saddrlens[i]);
         ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
       }
       i++;
@@ -784,11 +878,16 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
       if (NULL != unixpath)
         resi++;
       i = 0;
-      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      saddrs = GNUNET_new_array (resi + 1,
+                                 struct sockaddr *);
+      saddrlens = GNUNET_new_array (resi + 1,
+                                    socklen_t);
       if (NULL != unixpath)
       {
-        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        add_unixpath (saddrs,
+                      saddrlens,
+                      unixpath,
+                      abstract);
         i++;
       }
       saddrlens[i] = sizeof (struct sockaddr_in);
@@ -805,12 +904,17 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
       resi = 2;
       if (NULL != unixpath)
         resi++;
-      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      saddrs = GNUNET_new_array (resi + 1,
+                                 struct sockaddr *);
+      saddrlens = GNUNET_new_array (resi + 1,
+                                    socklen_t);
       i = 0;
       if (NULL != unixpath)
       {
-        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        add_unixpath (saddrs,
+                      saddrlens,
+                      unixpath,
+                      abstract);
         i++;
       }
       saddrlens[i] = sizeof (struct sockaddr_in6);
@@ -841,13 +945,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
 /**
  * Read listen sockets from the parent process (ARM).
  *
- * @param sctx service context to initialize
- * @return #GNUNET_YES if ok, #GNUNET_NO if not ok (must bind yourself),
- * and #GNUNET_SYSERR on error.
+ * @param sh service context to initialize
+ * @return NULL-terminated array of sockets on success,
+ *         NULL if not ok (must bind yourself)
  */
-static int
-receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
+static struct GNUNET_NETWORK_Handle **
+receive_sockets_from_parent (struct GNUNET_SERVICE_Handle *sh)
 {
+  static struct GNUNET_NETWORK_Handle **lsocks;
   const char *env_buf;
   int fail;
   uint64_t count;
@@ -855,15 +960,19 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
   HANDLE lsocks_pipe;
 
   env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
-  if ((NULL == env_buf) || (strlen (env_buf) <= 0))
-    return GNUNET_NO;
+  if ( (NULL == env_buf) ||
+       (strlen (env_buf) <= 0) )
+    return NULL;
   /* Using W32 API directly here, because this pipe will
    * never be used outside of this function, and it's just too much of a bother
    * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
    */
-  lsocks_pipe = (HANDLE) strtoul (env_buf, NULL, 10);
-  if ( (0 == lsocks_pipe) || (INVALID_HANDLE_VALUE == lsocks_pipe))
-    return GNUNET_NO;
+  lsocks_pipe = (HANDLE) strtoul (env_buf,
+                                  NULL,
+                                  10);
+  if ( (0 == lsocks_pipe) ||
+       (INVALID_HANDLE_VALUE == lsocks_pipe))
+    return NULL;
   fail = 1;
   do
   {
@@ -871,11 +980,17 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
     int fail2;
     DWORD rd;
 
-    ret = ReadFile (lsocks_pipe, &count, sizeof (count), &rd, NULL);
-    if ((0 == ret) || (sizeof (count) != rd) || (0 == count))
+    ret = ReadFile (lsocks_pipe,
+                    &count,
+                    sizeof (count),
+                    &rd,
+                    NULL);
+    if ( (0 == ret) ||
+         (sizeof (count) != rd) ||
+         (0 == count) )
       break;
-    sctx->lsocks =
-        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (count + 1));
+    lsocks = GNUNET_new_array (count + 1,
+                               struct GNUNET_NETWORK_Handle *);
 
     fail2 = 1;
     for (i = 0; i < count; i++)
@@ -884,51 +999,165 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
       uint64_t size;
       SOCKET s;
 
-      ret = ReadFile (lsocks_pipe, &size, sizeof (size), &rd, NULL);
-      if ( (0 == ret) || (sizeof (size) != rd) || (sizeof (pi) != size) )
+      ret = ReadFile (lsocks_pipe,
+                      &size,
+                      sizeof (size),
+                      &rd,
+                      NULL);
+      if ( (0 == ret) ||
+           (sizeof (size) != rd) ||
+           (sizeof (pi) != size) )
         break;
-      ret = ReadFile (lsocks_pipe, &pi, sizeof (pi), &rd, NULL);
-      if ( (0 == ret) || (sizeof (pi) != rd))
+      ret = ReadFile (lsocks_pipe,
+                      &pi,
+                      sizeof (pi),
+                      &rd,
+                      NULL);
+      if ( (0 == ret) ||
+           (sizeof (pi) != rd))
         break;
-      s = WSASocketA (pi.iAddressFamily, pi.iSocketType, pi.iProtocol, &pi, 0, WSA_FLAG_OVERLAPPED);
-      sctx->lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
-      if (NULL == sctx->lsocks[i])
+      s = WSASocketA (pi.iAddressFamily,
+                      pi.iSocketType,
+                      pi.iProtocol,
+                      &pi,
+                      0,
+                      WSA_FLAG_OVERLAPPED);
+      lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
+      if (NULL == lsocks[i])
         break;
       else if (i == count - 1)
         fail2 = 0;
     }
     if (fail2)
       break;
-    sctx->lsocks[count] = NULL;
+    lsocks[count] = NULL;
     fail = 0;
   }
   while (fail);
-
   CloseHandle (lsocks_pipe);
 
   if (fail)
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
          _("Could not access a pre-bound socket, will try to bind myself\n"));
-    for (i = 0; (i < count) && (NULL != sctx->lsocks[i]); i++)
-      GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[i]));
-    GNUNET_free_non_null (sctx->lsocks);
-    sctx->lsocks = NULL;
-    return GNUNET_NO;
+    for (i = 0; (i < count) && (NULL != lsocks[i]); i++)
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (lsocks[i]));
+    GNUNET_free (lsocks);
+    return NULL;
   }
-  return GNUNET_YES;
+  return lsocks;
 }
 #endif
 
 
 /**
- * Setup addr, addrlen, idle_timeout
- * based on configuration!
+ * Create and initialize a listen socket for the server.
+ *
+ * @param server_addr address to listen on
+ * @param socklen length of @a server_addr
+ * @return NULL on error, otherwise the listen socket
+ */
+static struct GNUNET_NETWORK_Handle *
+open_listen_socket (const struct sockaddr *server_addr,
+                    socklen_t socklen)
+{
+  struct GNUNET_NETWORK_Handle *sock;
+  uint16_t port;
+  int eno;
+
+  switch (server_addr->sa_family)
+  {
+  case AF_INET:
+    port = ntohs (((const struct sockaddr_in *) server_addr)->sin_port);
+    break;
+  case AF_INET6:
+    port = ntohs (((const struct sockaddr_in6 *) server_addr)->sin6_port);
+    break;
+  case AF_UNIX:
+    port = 0;
+    break;
+  default:
+    GNUNET_break (0);
+    port = 0;
+    break;
+  }
+  sock = GNUNET_NETWORK_socket_create (server_addr->sa_family,
+                                       SOCK_STREAM,
+                                       0);
+  if (NULL == sock)
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "socket");
+    errno = 0;
+    return NULL;
+  }
+  /* bind the socket */
+  if (GNUNET_OK != GNUNET_NETWORK_socket_bind (sock,
+                                               server_addr,
+                                               socklen))
+  {
+    eno = errno;
+    if (EADDRINUSE != errno)
+    {
+      /* we don't log 'EADDRINUSE' here since an IPv4 bind may
+       * fail if we already took the port on IPv6; if both IPv4 and
+       * IPv6 binds fail, then our caller will log using the
+       * errno preserved in 'eno' */
+      LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                    "bind");
+      if (0 != port)
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             _("`%s' failed for port %d (%s).\n"),
+             "bind",
+             port,
+             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
+      eno = 0;
+    }
+    else
+    {
+      if (0 != port)
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             _("`%s' failed for port %d (%s): address already in use\n"),
+             "bind", port,
+             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
+      else if (AF_UNIX == server_addr->sa_family)
+      {
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             _("`%s' failed for `%s': address already in use\n"),
+             "bind",
+             GNUNET_a2s (server_addr, socklen));
+      }
+    }
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (sock));
+    errno = eno;
+    return NULL;
+  }
+  if (GNUNET_OK != GNUNET_NETWORK_socket_listen (sock,
+                                                 5))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "listen");
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (sock));
+    errno = 0;
+    return NULL;
+  }
+  if (0 != port)
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Server starts to listen on port %u.\n",
+         port);
+  return sock;
+}
+
+
+/**
+ * Setup service handle
  *
  * Configuration may specify:
  * - PORT (where to bind to for TCP)
  * - UNIXPATH (where to bind to for UNIX domain sockets)
- * - TIMEOUT (after how many ms does an inactive service timeout);
  * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
  * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
  * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
@@ -936,108 +1165,170 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
  * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
  * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
  *
- * @param sctx service context to initialize
+ * @param sh service context to initialize
  * @return #GNUNET_OK if configuration succeeded
  */
 static int
-setup_service (struct GNUNET_SERVICE_Context *sctx)
+setup_service (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_TIME_Relative idleout;
   int tolerant;
-
+  struct GNUNET_NETWORK_Handle **lsocks;
 #ifndef MINGW
   const char *nfds;
   unsigned int cnt;
   int flags;
 #endif
 
-  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, "TIMEOUT"))
-  {
-    if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->service_name,
-                                             "TIMEOUT", &idleout))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TIMEOUT", sctx->service_name);
-      return GNUNET_SYSERR;
-    }
-    sctx->timeout = idleout;
-  }
-  else
-    sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-
   if (GNUNET_CONFIGURATION_have_value
-      (sctx->cfg, sctx->service_name, "TOLERANT"))
+      (sh->cfg,
+       sh->service_name,
+       "TOLERANT"))
   {
     if (GNUNET_SYSERR ==
         (tolerant =
-         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+         GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                               sh->service_name,
                                                "TOLERANT")))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TOLERANT", sctx->service_name);
+           "TOLERANT",
+           sh->service_name);
       return GNUNET_SYSERR;
     }
   }
   else
     tolerant = GNUNET_NO;
 
+  lsocks = NULL;
 #ifndef MINGW
   errno = 0;
-  if ((NULL != (nfds = getenv ("LISTEN_FDS"))) &&
-      (1 == SSCANF (nfds, "%u", &cnt)) && (cnt > 0) && (cnt < FD_SETSIZE) &&
-      (cnt + 4 < FD_SETSIZE))
+  if ( (NULL != (nfds = getenv ("LISTEN_FDS"))) &&
+       (1 == SSCANF (nfds,
+                     "%u",
+                     &cnt)) &&
+       (cnt > 0) &&
+       (cnt < FD_SETSIZE) &&
+       (cnt + 4 < FD_SETSIZE) )
   {
-    sctx->lsocks =
-        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (cnt + 1));
+    lsocks = GNUNET_new_array (cnt + 1,
+                               struct GNUNET_NETWORK_Handle *);
     while (0 < cnt--)
     {
-      flags = fcntl (3 + cnt, F_GETFD);
-      if ((flags < 0) || (0 != (flags & FD_CLOEXEC)) ||
-          (NULL ==
-           (sctx->lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
+      flags = fcntl (3 + cnt,
+                     F_GETFD);
+      if ( (flags < 0) ||
+           (0 != (flags & FD_CLOEXEC)) ||
+           (NULL ==
+            (lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
       {
         LOG (GNUNET_ERROR_TYPE_ERROR,
-             _
-             ("Could not access pre-bound socket %u, will try to bind myself\n"),
+             _("Could not access pre-bound socket %u, will try to bind myself\n"),
              (unsigned int) 3 + cnt);
         cnt++;
-        while (sctx->lsocks[cnt] != NULL)
-          GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[cnt++]));
-        GNUNET_free (sctx->lsocks);
-        sctx->lsocks = NULL;
+        while (NULL != lsocks[cnt])
+          GNUNET_break (GNUNET_OK ==
+                        GNUNET_NETWORK_socket_close (lsocks[cnt++]));
+        GNUNET_free (lsocks);
+        lsocks = NULL;
         break;
       }
     }
     unsetenv ("LISTEN_FDS");
   }
 #else
-  if (getenv ("GNUNET_OS_READ_LSOCKS") != NULL)
+  if (NULL != getenv ("GNUNET_OS_READ_LSOCKS"))
   {
-    receive_sockets_from_parent (sctx);
+    lsocks = receive_sockets_from_parent (sh);
     putenv ("GNUNET_OS_READ_LSOCKS=");
   }
 #endif
 
-  if ((NULL == sctx->lsocks) &&
-      (GNUNET_SYSERR ==
-       GNUNET_SERVICE_get_server_addresses (sctx->service_name, sctx->cfg,
-                                            &sctx->addrs, &sctx->addrlens)))
-    return GNUNET_SYSERR;
-  sctx->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
-  sctx->match_uid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+  if (NULL != lsocks)
+  {
+    /* listen only on inherited sockets if we have any */
+    struct GNUNET_NETWORK_Handle **ls;
+
+    for (ls = lsocks; NULL != *ls; ls++)
+    {
+      struct ServiceListenContext *slc;
+
+      slc = GNUNET_new (struct ServiceListenContext);
+      slc->sh = sh;
+      slc->listen_socket = *ls;
+      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
+                                   sh->slc_tail,
+                                   slc);
+    }
+    GNUNET_free (lsocks);
+  }
+  else
+  {
+    struct sockaddr **addrs;
+    socklen_t *addrlens;
+    int num;
+
+    num = get_server_addresses (sh->service_name,
+                                sh->cfg,
+                                &addrs,
+                                &addrlens);
+    if (GNUNET_SYSERR == num)
+      return GNUNET_SYSERR;
+
+    for (int i = 0; i < num; i++)
+    {
+      struct ServiceListenContext *slc;
+
+      slc = GNUNET_new (struct ServiceListenContext);
+      slc->sh = sh;
+      slc->listen_socket = open_listen_socket (addrs[i],
+                                               addrlens[i]);
+      if (NULL == slc->listen_socket)
+      {
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                             "bind");
+        GNUNET_free (addrs[i++]);
+        GNUNET_free (slc);
+        continue;
+      }
+      GNUNET_free (addrs[i++]);
+      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
+                                   sh->slc_tail,
+                                   slc);
+    }
+    GNUNET_free_non_null (addrlens);
+    GNUNET_free_non_null (addrs);
+    if ( (0 != num) &&
+         (NULL == sh->slc_head) )
+    {
+      /* All attempts to bind failed, hard failure */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Could not bind to any of the ports I was supposed to, refusing to run!\n"));
+      return GNUNET_SYSERR;
+    }
+  }
+
+  sh->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
+  sh->match_uid
+    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                            sh->service_name,
                                             "UNIX_MATCH_UID");
-  sctx->match_gid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+  sh->match_gid
+    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                            sh->service_name,
                                             "UNIX_MATCH_GID");
-  process_acl4 (&sctx->v4_denied, sctx, "REJECT_FROM");
-  process_acl4 (&sctx->v4_allowed, sctx, "ACCEPT_FROM");
-  process_acl6 (&sctx->v6_denied, sctx, "REJECT_FROM6");
-  process_acl6 (&sctx->v6_allowed, sctx, "ACCEPT_FROM6");
-
+  process_acl4 (&sh->v4_denied,
+                sh,
+                "REJECT_FROM");
+  process_acl4 (&sh->v4_allowed,
+                sh,
+                "ACCEPT_FROM");
+  process_acl6 (&sh->v6_denied,
+                sh,
+                "REJECT_FROM6");
+  process_acl6 (&sh->v6_allowed,
+                sh,
+                "ACCEPT_FROM6");
   return GNUNET_OK;
 }
 
@@ -1046,185 +1337,129 @@ setup_service (struct GNUNET_SERVICE_Context *sctx)
  * Get the name of the user that'll be used
  * to provide the service.
  *
- * @param sctx service context
+ * @param sh service context
  * @return value of the 'USERNAME' option
  */
 static char *
-get_user_name (struct GNUNET_SERVICE_Context *sctx)
+get_user_name (struct GNUNET_SERVICE_Handle *sh)
 {
   char *un;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
-                                               "USERNAME", &un))
+      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
+                                               sh->service_name,
+                                               "USERNAME",
+                                               &un))
     return NULL;
   return un;
 }
 
 
 /**
- * Write PID file.
+ * Set user ID.
  *
- * @param sctx service context
- * @param pid PID to write (should be equal to 'getpid()'
- * @return  #GNUNET_OK on success (including no work to be done)
+ * @param sh service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
-write_pid_file (struct GNUNET_SERVICE_Context *sctx, pid_t pid)
+set_user_id (struct GNUNET_SERVICE_Handle *sh)
 {
-  FILE *pidfd;
-  char *pif;
   char *user;
-  char *rdir;
-  int len;
-
-  if (NULL == (pif = get_pid_file_name (sctx)))
-    return GNUNET_OK;           /* no file desired */
-  user = get_user_name (sctx);
-  rdir = GNUNET_strdup (pif);
-  len = strlen (rdir);
-  while ((len > 0) && (rdir[len] != DIR_SEPARATOR))
-    len--;
-  rdir[len] = '\0';
-  if (0 != ACCESS (rdir, F_OK))
-  {
-    /* we get to create a directory -- and claim it
-     * as ours! */
-    (void) GNUNET_DISK_directory_create (rdir);
-    if ((NULL != user) && (0 < strlen (user)))
-      GNUNET_DISK_file_change_owner (rdir, user);
-  }
-  if (0 != ACCESS (rdir, W_OK | X_OK))
-  {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "access", rdir);
-    GNUNET_free (rdir);
-    GNUNET_free_non_null (user);
-    GNUNET_free (pif);
+
+  if (NULL == (user = get_user_name (sh)))
+    return GNUNET_OK;           /* keep */
+#ifndef MINGW
+  struct passwd *pws;
+
+  errno = 0;
+  pws = getpwnam (user);
+  if (NULL == pws)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Cannot obtain information about user `%s': %s\n"),
+         user,
+         errno == 0 ? _("No such user") : STRERROR (errno));
+    GNUNET_free (user);
     return GNUNET_SYSERR;
   }
-  GNUNET_free (rdir);
-  pidfd = FOPEN (pif, "w");
-  if (NULL == pidfd)
+  if ( (0 != setgid (pws->pw_gid)) ||
+       (0 != setegid (pws->pw_gid)) ||
+#if HAVE_INITGROUPS
+       (0 != initgroups (user,
+                         pws->pw_gid)) ||
+#endif
+       (0 != setuid (pws->pw_uid)) ||
+       (0 != seteuid (pws->pw_uid)))
   {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "fopen", pif);
-    GNUNET_free (pif);
-    GNUNET_free_non_null (user);
-    return GNUNET_SYSERR;
+    if ((0 != setregid (pws->pw_gid,
+                        pws->pw_gid)) ||
+        (0 != setreuid (pws->pw_uid,
+                        pws->pw_uid)))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Cannot change user/group to `%s': %s\n"),
+           user,
+           STRERROR (errno));
+      GNUNET_free (user);
+      return GNUNET_SYSERR;
+    }
   }
-  if (0 > FPRINTF (pidfd, "%u", pid))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "fprintf", pif);
-  GNUNET_break (0 == FCLOSE (pidfd));
-  if ((NULL != user) && (0 < strlen (user)))
-    GNUNET_DISK_file_change_owner (pif, user);
-  GNUNET_free_non_null (user);
-  GNUNET_free (pif);
+#endif
+  GNUNET_free (user);
   return GNUNET_OK;
 }
 
 
 /**
- * Task run during shutdown.  Stops the server/service.
+ * Get the name of the file where we will
+ * write the PID of the service.
  *
- * @param cls the `struct GNUNET_SERVICE_Context`
+ * @param sh service context
+ * @return name of the file for the process ID
  */
-static void
-shutdown_task (void *cls)
+static char *
+get_pid_file_name (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVICE_Context *service = cls;
-  struct GNUNET_SERVER_Handle *server = service->server;
+  char *pif;
 
-  service->shutdown_task = NULL;
-  if (0 != (service->options & GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN))
-    GNUNET_SERVER_stop_listening (server);
-  else
-    GNUNET_SERVER_destroy (server);
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
+                                               sh->service_name,
+                                               "PIDFILE",
+                                               &pif))
+    return NULL;
+  return pif;
 }
 
 
 /**
- * Initial task for the service.
+ * Delete the PID file that was created by our parent.
  *
- * @param cls service context
+ * @param sh service context
  */
 static void
-service_task (void *cls)
+pid_file_delete (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVICE_Context *sctx = cls;
-  unsigned int i;
+  char *pif = get_pid_file_name (sh);
 
-  (void) GNUNET_SPEEDUP_start_ (sctx->cfg);
-  GNUNET_RESOLVER_connect (sctx->cfg);
-  if (NULL != sctx->lsocks)
-    sctx->server
-      = GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
-                                           sctx->timeout, sctx->require_found);
-  else
-    sctx->server
-      = GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
-                              sctx->timeout, sctx->require_found);
-  if (NULL == sctx->server)
-  {
-    if (NULL != sctx->addrs)
-      for (i = 0; NULL != sctx->addrs[i]; i++)
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             _("Failed to start `%s' at `%s'\n"),
-             sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
-    sctx->ret = GNUNET_SYSERR;
-    return;
-  }
-#ifndef WINDOWS
-  if (NULL != sctx->addrs)
-    for (i = 0; NULL != sctx->addrs[i]; i++)
-      if ((AF_UNIX == sctx->addrs[i]->sa_family)
-          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
-        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
-                                     sctx->match_uid,
-                                     sctx->match_gid);
-#endif
-
-
-  if (0 == (sctx->options & GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN))
-  {
-    /* install a task that will kill the server
-     * process if the scheduler ever gets a shutdown signal */
-    sctx->shutdown_task = GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                                         sctx);
-  }
-  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
-  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
-  i = 0;
-  while (NULL != sctx->my_handlers[i].callback)
-    sctx->my_handlers[i++].callback_cls = sctx;
-  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
-  if (-1 != sctx->ready_confirm_fd)
-  {
-    GNUNET_break (1 == WRITE (sctx->ready_confirm_fd, ".", 1));
-    GNUNET_break (0 == CLOSE (sctx->ready_confirm_fd));
-    sctx->ready_confirm_fd = -1;
-    write_pid_file (sctx, getpid ());
-  }
-  if (NULL != sctx->addrs)
-  {
-    i = 0;
-    while (NULL != sctx->addrs[i])
-    {
-      LOG (GNUNET_ERROR_TYPE_INFO, _("Service `%s' runs at %s\n"),
-           sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
-      i++;
-    }
-  }
-  sctx->task (sctx->task_cls, sctx->server, sctx->cfg);
+  if (NULL == pif)
+    return;                     /* no PID file */
+  if (0 != UNLINK (pif))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
+                       "unlink",
+                       pif);
+  GNUNET_free (pif);
 }
 
 
 /**
  * Detach from terminal.
  *
- * @param sctx service context
+ * @param sh service context
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
-detach_terminal (struct GNUNET_SERVICE_Context *sctx)
+detach_terminal (struct GNUNET_SERVICE_Handle *sh)
 {
 #ifndef MINGW
   pid_t pid;
@@ -1233,13 +1468,15 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
   if (0 != PIPE (filedes))
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "pipe");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "pipe");
     return GNUNET_SYSERR;
   }
   pid = fork ();
   if (pid < 0)
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "fork");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "fork");
     return GNUNET_SYSERR;
   }
   if (0 != pid)
@@ -1249,15 +1486,19 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
     GNUNET_break (0 == CLOSE (filedes[1]));
     c = 'X';
-    if (1 != READ (filedes[0], &c, sizeof (char)))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "read");
+    if (1 != READ (filedes[0],
+                   &c,
+                   sizeof (char)))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
+                    "read");
     fflush (stdout);
     switch (c)
     {
     case '.':
       exit (0);
     case 'I':
-      LOG (GNUNET_ERROR_TYPE_INFO, _("Service process failed to initialize\n"));
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process failed to initialize\n"));
       break;
     case 'S':
       LOG (GNUNET_ERROR_TYPE_INFO,
@@ -1273,13 +1514,16 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
   GNUNET_break (0 == CLOSE (0));
   GNUNET_break (0 == CLOSE (1));
   GNUNET_break (0 == CLOSE (filedes[0]));
-  nullfd = OPEN ("/dev/null", O_RDWR | O_APPEND);
+  nullfd = OPEN ("/dev/null",
+                 O_RDWR | O_APPEND);
   if (nullfd < 0)
     return GNUNET_SYSERR;
   /* set stdin/stdout to /dev/null */
-  if ((dup2 (nullfd, 0) < 0) || (dup2 (nullfd, 1) < 0))
+  if ( (dup2 (nullfd, 0) < 0) ||
+       (dup2 (nullfd, 1) < 0) )
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "dup2");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "dup2");
     (void) CLOSE (nullfd);
     return GNUNET_SYSERR;
   }
@@ -1287,8 +1531,9 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
   /* Detach from controlling terminal */
   pid = setsid ();
   if (-1 == pid)
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "setsid");
-  sctx->ready_confirm_fd = filedes[1];
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "setsid");
+  sh->ready_confirm_fd = filedes[1];
 #else
   /* FIXME: we probably need to do something else
    * elsewhere in order to fork the process itself... */
@@ -1299,144 +1544,228 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
 
 /**
- * Set user ID.
+ * Tear down the service, closing the listen sockets and
+ * freeing the ACLs.
  *
- * @param sctx service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ * @param sh handle to the service to tear down.
  */
-static int
-set_user_id (struct GNUNET_SERVICE_Context *sctx)
+static void
+teardown_service (struct GNUNET_SERVICE_Handle *sh)
 {
-  char *user;
-
-  if (NULL == (user = get_user_name (sctx)))
-    return GNUNET_OK;           /* keep */
-#ifndef MINGW
-  struct passwd *pws;
+  struct ServiceListenContext *slc;
 
-  errno = 0;
-  pws = getpwnam (user);
-  if (NULL == pws)
+  GNUNET_free_non_null (sh->v4_denied);
+  GNUNET_free_non_null (sh->v6_denied);
+  GNUNET_free_non_null (sh->v4_allowed);
+  GNUNET_free_non_null (sh->v6_allowed);
+  while (NULL != (slc = sh->slc_head))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Cannot obtain information about user `%s': %s\n"), user,
-         errno == 0 ? _("No such user") : STRERROR (errno));
-    GNUNET_free (user);
-    return GNUNET_SYSERR;
+    GNUNET_CONTAINER_DLL_remove (sh->slc_head,
+                                 sh->slc_tail,
+                                 slc);
+    if (NULL != slc->listen_task)
+      GNUNET_SCHEDULER_cancel (slc->listen_task);
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (slc->listen_socket));
+    GNUNET_free (slc);
   }
-  if ((0 != setgid (pws->pw_gid)) || (0 != setegid (pws->pw_gid)) ||
-#if HAVE_INITGROUPS
-      (0 != initgroups (user, pws->pw_gid)) ||
-#endif
-      (0 != setuid (pws->pw_uid)) || (0 != seteuid (pws->pw_uid)))
+}
+
+
+/**
+ * Low-level function to start a service if the scheduler
+ * is already running.  Should only be used directly in
+ * special cases.
+ *
+ * The function will launch the service with the name @a service_name
+ * using the @a service_options to configure its shutdown
+ * behavior. When clients connect or disconnect, the respective
+ * @a connect_cb or @a disconnect_cb functions will be called. For
+ * messages received from the clients, the respective @a handlers will
+ * be invoked; for the closure of the handlers we use the return value
+ * from the @a connect_cb invocation of the respective client.
+ *
+ * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
+ * message to receive further messages from this client.  If
+ * #GNUNET_SERVICE_client_continue() is not called within a short
+ * time, a warning will be logged. If delays are expected, services
+ * should call #GNUNET_SERVICE_client_disable_continue_warning() to
+ * disable the warning.
+ *
+ * Clients sending invalid messages (based on @a handlers) will be
+ * dropped. Additionally, clients can be dropped at any time using
+ * #GNUNET_SERVICE_client_drop().
+ *
+ * The service must be stopped using #GNUNET_SERVICE_stop().
+ *
+ * @param service_name name of the service to run
+ * @param cfg configuration to use
+ * @param connect_cb function to call whenever a client connects
+ * @param disconnect_cb function to call whenever a client disconnects
+ * @param cls closure argument for @a connect_cb and @a disconnect_cb
+ * @param handlers NULL-terminated array of message handlers for the service,
+ *                 the closure will be set to the value returned by
+ *                 the @a connect_cb for the respective connection
+ * @return NULL on error
+ */
+struct GNUNET_SERVICE_Handle *
+GNUNET_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      GNUNET_SERVICE_ConnectHandler connect_cb,
+                      GNUNET_SERVICE_DisconnectHandler disconnect_cb,
+                      void *cls,
+                      const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  struct GNUNET_SERVICE_Handle *sh;
+
+  sh = GNUNET_new (struct GNUNET_SERVICE_Handle);
+  sh->service_name = service_name;
+  sh->cfg = cfg;
+  sh->connect_cb = connect_cb;
+  sh->disconnect_cb = disconnect_cb;
+  sh->cb_cls = cls;
+  sh->handlers = GNUNET_MQ_copy_handlers (handlers);
+  if (GNUNET_OK != setup_service (sh))
   {
-    if ((0 != setregid (pws->pw_gid, pws->pw_gid)) ||
-        (0 != setreuid (pws->pw_uid, pws->pw_uid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR, _("Cannot change user/group to `%s': %s\n"),
-           user, STRERROR (errno));
-      GNUNET_free (user);
-      return GNUNET_SYSERR;
-    }
+    GNUNET_free_non_null (sh->handlers);
+    GNUNET_free (sh);
+    return NULL;
   }
-#endif
-  GNUNET_free (user);
-  return GNUNET_OK;
+  GNUNET_SERVICE_resume (sh);
+  return sh;
 }
 
 
 /**
- * Delete the PID file that was created by our parent.
+ * Stops a service that was started with #GNUNET_SERVICE_start().
  *
- * @param sctx service context
+ * @param srv service to stop
  */
-static void
-pid_file_delete (struct GNUNET_SERVICE_Context *sctx)
+void
+GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Handle *srv)
 {
-  char *pif = get_pid_file_name (sctx);
-
-  if (NULL == pif)
-    return;                     /* no PID file */
-  if (0 != UNLINK (pif))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", pif);
-  GNUNET_free (pif);
+  struct GNUNET_SERVICE_Client *client;
+
+  GNUNET_SERVICE_suspend (srv);
+  while (NULL != (client = srv->clients_head))
+    GNUNET_SERVICE_client_drop (client);
+  teardown_service (srv);
+  GNUNET_free_non_null (srv->handlers);
+  GNUNET_free (srv);
 }
 
 
 /**
- * Run a standard GNUnet service startup sequence (initialize loggers
- * and configuration, parse options).
+ * Creates the "main" function for a GNUnet service.  You
+ * should almost always use the #GNUNET_SERVICE_MAIN macro
+ * instead of calling this function directly (except
+ * for ARM, which should call this function directly).
+ *
+ * The function will launch the service with the name @a service_name
+ * using the @a service_options to configure its shutdown
+ * behavior. Once the service is ready, the @a init_cb will be called
+ * for service-specific initialization.  @a init_cb will be given the
+ * service handler which can be used to control the service's
+ * availability.  When clients connect or disconnect, the respective
+ * @a connect_cb or @a disconnect_cb functions will be called. For
+ * messages received from the clients, the respective @a handlers will
+ * be invoked; for the closure of the handlers we use the return value
+ * from the @a connect_cb invocation of the respective client.
  *
- * @param argc number of command line arguments
- * @param argv command line arguments
- * @param service_name our service name
- * @param options service options
- * @param task main task of the service
- * @param task_cls closure for @a task
- * @return #GNUNET_SYSERR on error, #GNUNET_OK
- *         if we shutdown nicely
+ * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
+ * message to receive further messages from this client.  If
+ * #GNUNET_SERVICE_client_continue() is not called within a short
+ * time, a warning will be logged. If delays are expected, services
+ * should call #GNUNET_SERVICE_client_disable_continue_warning() to
+ * disable the warning.
+ *
+ * Clients sending invalid messages (based on @a handlers) will be
+ * dropped. Additionally, clients can be dropped at any time using
+ * #GNUNET_SERVICE_client_drop().
+ *
+ * @param argc number of command-line arguments in @a argv
+ * @param argv array of command-line arguments
+ * @param service_name name of the service to run
+ * @param options options controlling shutdown of the service
+ * @param service_init_cb function to call once the service is ready
+ * @param connect_cb function to call whenever a client connects
+ * @param disconnect_cb function to call whenever a client disconnects
+ * @param cls closure argument for @a service_init_cb, @a connect_cb and @a disconnect_cb
+ * @param handlers NULL-terminated array of message handlers for the service,
+ *                 the closure will be set to the value returned by
+ *                 the @a connect_cb for the respective connection
+ * @return 0 on success, non-zero on error
  */
 int
-GNUNET_SERVICE_run (int argc, char *const *argv,
-                    const char *service_name,
-                    enum GNUNET_SERVICE_Options options,
-                    GNUNET_SERVICE_Main task,
-                    void *task_cls)
+GNUNET_SERVICE_run_ (int argc,
+                     char *const *argv,
+                     const char *service_name,
+                     enum GNUNET_SERVICE_Options options,
+                     GNUNET_SERVICE_InitCallback service_init_cb,
+                     GNUNET_SERVICE_ConnectHandler connect_cb,
+                     GNUNET_SERVICE_DisconnectHandler disconnect_cb,
+                     void *cls,
+                     const struct GNUNET_MQ_MessageHandler *handlers)
 {
-#define HANDLE_ERROR do { GNUNET_break (0); goto shutdown; } while (0)
-
-  int err;
-  int ret;
-  char *cfg_fn;
-  char *opt_cfg_fn;
+  struct GNUNET_SERVICE_Handle sh;
+  char *cfg_filename;
+  char *opt_cfg_filename;
   char *loglev;
+  const char *xdg;
   char *logfile;
   int do_daemonize;
-  unsigned int i;
   unsigned long long skew_offset;
   unsigned long long skew_variance;
   long long clock_offset;
-  struct GNUNET_SERVICE_Context sctx;
   struct GNUNET_CONFIGURATION_Handle *cfg;
-  const char *xdg;
+  int ret;
+  int err;
 
   struct GNUNET_GETOPT_CommandLineOption service_options[] = {
-    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_fn),
-    {'d', "daemonize", NULL,
-     gettext_noop ("do daemonize (detach from terminal)"), 0,
-     GNUNET_GETOPT_set_one, &do_daemonize},
-    GNUNET_GETOPT_OPTION_HELP (NULL),
-    GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
-    GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
-    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
+    GNUNET_GETOPT_option_cfgfile (&opt_cfg_filename),
+    GNUNET_GETOPT_option_flag ('d',
+                                  "daemonize",
+                                  gettext_noop ("do daemonize (detach from terminal)"),
+                                  &do_daemonize),
+    GNUNET_GETOPT_option_help (NULL),
+    GNUNET_GETOPT_option_loglevel (&loglev),
+    GNUNET_GETOPT_option_logfile (&logfile),
+    GNUNET_GETOPT_option_version (PACKAGE_VERSION " " VCS_VERSION),
     GNUNET_GETOPT_OPTION_END
   };
+
   err = 1;
-  do_daemonize = 0;
-  logfile = NULL;
-  loglev = NULL;
-  opt_cfg_fn = NULL;
+  memset (&sh,
+          0,
+          sizeof (sh));
   xdg = getenv ("XDG_CONFIG_HOME");
   if (NULL != xdg)
-    GNUNET_asprintf (&cfg_fn,
+    GNUNET_asprintf (&cfg_filename,
                      "%s%s%s",
                      xdg,
                      DIR_SEPARATOR_STR,
                      GNUNET_OS_project_data_get ()->config_file);
   else
-    cfg_fn = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
-  memset (&sctx, 0, sizeof (sctx));
-  sctx.options = options;
-  sctx.ready_confirm_fd = -1;
-  sctx.ret = GNUNET_OK;
-  sctx.timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  sctx.task = task;
-  sctx.task_cls = task_cls;
-  sctx.service_name = service_name;
-  sctx.cfg = cfg = GNUNET_CONFIGURATION_create ();
+    cfg_filename = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
+  sh.ready_confirm_fd = -1;
+  sh.options = options;
+  sh.cfg = cfg = GNUNET_CONFIGURATION_create ();
+  sh.service_init_cb = service_init_cb;
+  sh.connect_cb = connect_cb;
+  sh.disconnect_cb = disconnect_cb;
+  sh.cb_cls = cls;
+  sh.handlers = GNUNET_MQ_copy_handlers (handlers);
+  sh.service_name = service_name;
 
   /* setup subsystems */
-  ret = GNUNET_GETOPT_run (service_name, service_options, argc, argv);
+  loglev = NULL;
+  logfile = NULL;
+  opt_cfg_filename = NULL;
+  do_daemonize = 0;
+  ret = GNUNET_GETOPT_run (service_name,
+                           service_options,
+                           argc,
+                           argv);
   if (GNUNET_SYSERR == ret)
     goto shutdown;
   if (GNUNET_NO == ret)
@@ -1444,254 +1773,844 @@ GNUNET_SERVICE_run (int argc, char *const *argv,
     err = 0;
     goto shutdown;
   }
-  if (GNUNET_OK != GNUNET_log_setup (service_name, loglev, logfile))
-    HANDLE_ERROR;
-  if (NULL == opt_cfg_fn)
-    opt_cfg_fn = GNUNET_strdup (cfg_fn);
-  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_fn))
+  if (GNUNET_OK != GNUNET_log_setup (service_name,
+                                     loglev,
+                                     logfile))
+  {
+    GNUNET_break (0);
+    goto shutdown;
+  }
+  if (NULL == opt_cfg_filename)
+    opt_cfg_filename = GNUNET_strdup (cfg_filename);
+  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_filename))
   {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, opt_cfg_fn))
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
+                                                    opt_cfg_filename))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Malformed configuration file `%s', exit ...\n"),
-                  opt_cfg_fn);
+                  opt_cfg_filename);
       goto shutdown;
     }
   }
   else
   {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, NULL))
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
+                                                    NULL))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Malformed configuration, exit ...\n"));
       goto shutdown;
     }
-    if (0 != strcmp (opt_cfg_fn, cfg_fn))
+    if (0 != strcmp (opt_cfg_filename,
+                     cfg_filename))
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Could not access configuration file `%s'\n"),
-                  opt_cfg_fn);
+                  opt_cfg_filename);
   }
-  if (GNUNET_OK != setup_service (&sctx))
+  if (GNUNET_OK != setup_service (&sh))
     goto shutdown;
-  if ((1 == do_daemonize) && (GNUNET_OK != detach_terminal (&sctx)))
-    HANDLE_ERROR;
-  if (GNUNET_OK != set_user_id (&sctx))
+  if ( (1 == do_daemonize) &&
+       (GNUNET_OK != detach_terminal (&sh)) )
+  {
+    GNUNET_break (0);
+    goto shutdown;
+  }
+  if (GNUNET_OK != set_user_id (&sh))
     goto shutdown;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Service `%s' runs with configuration from `%s'\n",
        service_name,
-       opt_cfg_fn);
+       opt_cfg_filename);
   if ((GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
-                                              "SKEW_OFFSET", &skew_offset)) &&
+       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
+                                              "TESTING",
+                                              "SKEW_OFFSET",
+                                              &skew_offset)) &&
       (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
-                                              "SKEW_VARIANCE", &skew_variance)))
+       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
+                                              "TESTING",
+                                              "SKEW_VARIANCE",
+                                              &skew_variance)))
   {
     clock_offset = skew_offset - skew_variance;
     GNUNET_TIME_set_offset (clock_offset);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Skewing clock by %dll ms\n", clock_offset);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Skewing clock by %dll ms\n",
+         clock_offset);
   }
+  GNUNET_RESOLVER_connect (sh.cfg);
+
   /* actually run service */
   err = 0;
-  GNUNET_SCHEDULER_run (&service_task, &sctx);
+  GNUNET_SCHEDULER_run (&service_main,
+                        &sh);
   /* shutdown */
-  if ((1 == do_daemonize) && (NULL != sctx.server))
-    pid_file_delete (&sctx);
-  GNUNET_free_non_null (sctx.my_handlers);
+  if (1 == do_daemonize)
+    pid_file_delete (&sh);
 
 shutdown:
-  if (-1 != sctx.ready_confirm_fd)
+  if (-1 != sh.ready_confirm_fd)
   {
-    if (1 != WRITE (sctx.ready_confirm_fd, err ? "I" : "S", 1))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "write");
-    GNUNET_break (0 == CLOSE (sctx.ready_confirm_fd));
+    if (1 != WRITE (sh.ready_confirm_fd,
+                    err ? "I" : "S",
+                    1))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
+                    "write");
+    GNUNET_break (0 == CLOSE (sh.ready_confirm_fd));
   }
 #if HAVE_MALLINFO
   {
     char *counter;
 
     if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sctx.cfg, service_name,
+          GNUNET_CONFIGURATION_have_value (sh.cfg,
+                                           service_name,
                                            "GAUGER_HEAP")) &&
          (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sctx.cfg, service_name,
+          GNUNET_CONFIGURATION_get_value_string (sh.cfg,
+                                                 service_name,
                                                  "GAUGER_HEAP",
                                                  &counter)) )
     {
       struct mallinfo mi;
 
       mi = mallinfo ();
-      GAUGER (service_name, counter, mi.usmblks, "blocks");
+      GAUGER (service_name,
+              counter,
+              mi.usmblks,
+              "blocks");
       GNUNET_free (counter);
     }
   }
 #endif
+  teardown_service (&sh);
+  GNUNET_free_non_null (sh.handlers);
   GNUNET_SPEEDUP_stop_ ();
   GNUNET_CONFIGURATION_destroy (cfg);
-  i = 0;
-  if (NULL != sctx.addrs)
-    while (NULL != sctx.addrs[i])
-      GNUNET_free (sctx.addrs[i++]);
-  GNUNET_free_non_null (sctx.addrs);
-  GNUNET_free_non_null (sctx.addrlens);
   GNUNET_free_non_null (logfile);
   GNUNET_free_non_null (loglev);
-  GNUNET_free (cfg_fn);
-  GNUNET_free_non_null (opt_cfg_fn);
-  GNUNET_free_non_null (sctx.v4_denied);
-  GNUNET_free_non_null (sctx.v6_denied);
-  GNUNET_free_non_null (sctx.v4_allowed);
-  GNUNET_free_non_null (sctx.v6_allowed);
-
-  return err ? GNUNET_SYSERR : sctx.ret;
+  GNUNET_free (cfg_filename);
+  GNUNET_free_non_null (opt_cfg_filename);
+
+  return err ? GNUNET_SYSERR : sh.ret;
 }
 
 
 /**
- * Run a service startup sequence within an existing
- * initialized system.
+ * Suspend accepting connections from the listen socket temporarily.
+ * Resume activity using #GNUNET_SERVICE_resume.
  *
- * @param service_name our service name
- * @param cfg configuration to use
- * @param options service options
- * @return NULL on error, service handle
+ * @param sh service to stop accepting connections.
  */
-struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      enum GNUNET_SERVICE_Options options)
+void
+GNUNET_SERVICE_suspend (struct GNUNET_SERVICE_Handle *sh)
 {
-  int i;
-  struct GNUNET_SERVICE_Context *sctx;
+  struct ServiceListenContext *slc;
 
-  sctx = GNUNET_new (struct GNUNET_SERVICE_Context);
-  sctx->ready_confirm_fd = -1;  /* no daemonizing */
-  sctx->ret = GNUNET_OK;
-  sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  sctx->service_name = service_name;
-  sctx->cfg = cfg;
-  sctx->options = options;
+  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
+  {
+    if (NULL != slc->listen_task)
+    {
+      GNUNET_SCHEDULER_cancel (slc->listen_task);
+      slc->listen_task = NULL;
+    }
+  }
+}
 
-  /* setup subsystems */
-  if (GNUNET_OK != setup_service (sctx))
+
+/**
+ * Task run when we are ready to transmit data to the
+ * client.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client *` to send to
+ */
+static void
+do_send (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  ssize_t ret;
+  size_t left;
+  const char *buf;
+
+  client->send_task = NULL;
+  buf = (const char *) client->msg;
+  left = ntohs (client->msg->size) - client->msg_pos;
+  ret = GNUNET_NETWORK_socket_send (client->sock,
+                                    &buf[client->msg_pos],
+                                    left);
+  GNUNET_assert (ret <= (ssize_t) left);
+  if (0 == ret)
   {
-    GNUNET_SERVICE_stop (sctx);
-    return NULL;
+    GNUNET_MQ_inject_error (client->mq,
+                            GNUNET_MQ_ERROR_WRITE);
+    return;
   }
-  if (NULL != sctx->lsocks)
-    sctx->server =
-        GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
-                                           sctx->timeout, sctx->require_found);
-  else
-    sctx->server =
-        GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
-                              sctx->timeout, sctx->require_found);
+  if (-1 == ret)
+  {
+    if ( (EAGAIN == errno) ||
+         (EINTR == errno) )
+    {
+      /* ignore */
+      ret = 0;
+    }
+    else
+    {
+      if (EPIPE != errno)
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+                             "send");
+      GNUNET_MQ_inject_error (client->mq,
+                              GNUNET_MQ_ERROR_WRITE);
+      return;
+    }
+  }
+  if (0 == client->msg_pos)
+  {
+    GNUNET_MQ_impl_send_in_flight (client->mq);
+  }
+  client->msg_pos += ret;
+  if (left > ret)
+  {
+    GNUNET_assert (NULL == client->drop_task);
+    client->send_task
+      = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                        client->sock,
+                                        &do_send,
+                                        client);
+    return;
+  }
+  GNUNET_MQ_impl_send_continue (client->mq);
+}
+
 
-  if (NULL == sctx->server)
+/**
+ * Signature of functions implementing the sending functionality of a
+ * message queue.
+ *
+ * @param mq the message queue
+ * @param msg the message to send
+ * @param impl_state our `struct GNUNET_SERVICE_Client *`
+ */
+static void
+service_mq_send (struct GNUNET_MQ_Handle *mq,
+                 const struct GNUNET_MessageHeader *msg,
+                 void *impl_state)
+{
+  struct GNUNET_SERVICE_Client *client = impl_state;
+
+  if (NULL != client->drop_task)
+    return; /* we're going down right now, do not try to send */
+  GNUNET_assert (NULL == client->send_task);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message of type %u and size %u to client\n",
+       ntohs (msg->type),
+       ntohs (msg->size));
+  client->msg = msg;
+  client->msg_pos = 0;
+  client->send_task
+    = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                      client->sock,
+                                      &do_send,
+                                      client);
+}
+
+
+/**
+ * Implementation function that cancels the currently sent message.
+ *
+ * @param mq message queue
+ * @param impl_state state specific to the implementation
+ */
+static void
+service_mq_cancel (struct GNUNET_MQ_Handle *mq,
+                   void *impl_state)
+{
+  struct GNUNET_SERVICE_Client *client = impl_state;
+
+  GNUNET_assert (0 == client->msg_pos);
+  client->msg = NULL;
+  GNUNET_SCHEDULER_cancel (client->send_task);
+  client->send_task = NULL;
+}
+
+
+/**
+ * Generic error handler, called with the appropriate
+ * error code and the same closure specified at the creation of
+ * the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with our `struct GNUNET_SERVICE_Client`
+ * @param error error code
+ */
+static void
+service_mq_error_handler (void *cls,
+                          enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  struct GNUNET_SERVICE_Handle *sh = client->sh;
+
+  if ( (GNUNET_MQ_ERROR_NO_MATCH == error) &&
+       (GNUNET_NO == sh->require_found) )
   {
-    GNUNET_SERVICE_stop (sctx);
-    return NULL;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "No handler for message of type %u found\n",
+                (unsigned int) client->warn_type);
+    GNUNET_SERVICE_client_continue (client);
+    return; /* ignore error */
+  }
+  GNUNET_SERVICE_client_drop (client);
+}
+
+
+/**
+ * Task run to warn about missing calls to #GNUNET_SERVICE_client_continue().
+ *
+ * @param cls our `struct GNUNET_SERVICE_Client *` to process more requests from
+ */
+static void
+warn_no_client_continue (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+
+  GNUNET_break (0 != client->warn_type); /* type should never be 0 here, as we don't use 0 */
+  client->warn_task
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_no_client_continue,
+                                    client);
+  LOG (GNUNET_ERROR_TYPE_WARNING,
+       _("Processing code for message of type %u did not call `GNUNET_SERVICE_client_continue' after %s\n"),
+       (unsigned int) client->warn_type,
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (client->warn_start),
+                                               GNUNET_YES));
+}
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer for a client.
+ *
+ * Do not call #GNUNET_MST_destroy() from within
+ * the scope of this callback.
+ *
+ * @param cls closure with the `struct GNUNET_SERVICE_Client *`
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if the client was dropped
+ */
+static int
+service_client_mst_cb (void *cls,
+                       const struct GNUNET_MessageHeader *message)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received message of type %u and size %u from client\n",
+       ntohs (message->type),
+       ntohs (message->size));
+  GNUNET_assert (GNUNET_NO == client->needs_continue);
+  client->needs_continue = GNUNET_YES;
+  client->warn_type = ntohs (message->type);
+  client->warn_start = GNUNET_TIME_absolute_get ();
+  GNUNET_assert (NULL == client->warn_task);
+  client->warn_task
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_no_client_continue,
+                                    client);
+  GNUNET_MQ_inject_message (client->mq,
+                            message);
+  if (NULL != client->drop_task)
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * A client sent us data. Receive and process it.  If we are done,
+ * reschedule this task.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client` that sent us data.
+ */
+static void
+service_client_recv (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  int ret;
+
+  client->recv_task = NULL;
+  ret = GNUNET_MST_read (client->mst,
+                         client->sock,
+                         GNUNET_NO,
+                         GNUNET_YES);
+  if (GNUNET_SYSERR == ret)
+  {
+    /* client closed connection (or IO error) */
+    if (NULL == client->drop_task)
+    {
+      GNUNET_assert (GNUNET_NO == client->needs_continue);
+      GNUNET_SERVICE_client_drop (client);
+    }
+    return;
   }
+  if (GNUNET_NO == ret)
+    return; /* more messages in buffer, wait for application
+               to be done processing */
+  GNUNET_assert (GNUNET_OK == ret);
+  if (GNUNET_YES == client->needs_continue)
+    return;
+  if (NULL != client->recv_task)
+    return;
+  /* MST needs more data, re-schedule read job */
+  client->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     client->sock,
+                                     &service_client_recv,
+                                     client);
+}
+
+
+/**
+ * We have successfully accepted a connection from a client.  Now
+ * setup the client (with the scheduler) and tell the application.
+ *
+ * @param sh service that accepted the client
+ * @param sock socket associated with the client
+ */
+static void
+start_client (struct GNUNET_SERVICE_Handle *sh,
+              struct GNUNET_NETWORK_Handle *csock)
+{
+  struct GNUNET_SERVICE_Client *client;
+
+  client = GNUNET_new (struct GNUNET_SERVICE_Client);
+  GNUNET_CONTAINER_DLL_insert (sh->clients_head,
+                               sh->clients_tail,
+                               client);
+  client->sh = sh;
+  client->sock = csock;
+  client->mq = GNUNET_MQ_queue_for_callbacks (&service_mq_send,
+                                              NULL,
+                                              &service_mq_cancel,
+                                              client,
+                                              sh->handlers,
+                                              &service_mq_error_handler,
+                                              client);
+  client->mst = GNUNET_MST_create (&service_client_mst_cb,
+                                   client);
+  if (NULL != sh->connect_cb)
+    client->user_context = sh->connect_cb (sh->cb_cls,
+                                           client,
+                                           client->mq);
+  GNUNET_MQ_set_handlers_closure (client->mq,
+                                  client->user_context);
+  client->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     client->sock,
+                                     &service_client_recv,
+                                     client);
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param add the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
+                   const struct in_addr *add)
+{
+  unsigned int i;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  i = 0;
+  while ( (0 != list[i].network.s_addr) ||
+          (0 != list[i].netmask.s_addr) )
+  {
+    if ((add->s_addr & list[i].netmask.s_addr) ==
+        (list[i].network.s_addr & list[i].netmask.s_addr))
+      return GNUNET_YES;
+    i++;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param ip the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
+                   const struct in6_addr *ip)
+{
+  unsigned int i;
+  unsigned int j;
+  struct in6_addr zero;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  memset (&zero,
+          0,
+          sizeof (struct in6_addr));
+  i = 0;
+NEXT:
+  while (0 != memcmp (&zero,
+                      &list[i].network,
+                      sizeof (struct in6_addr)))
+  {
+    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
+      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
+          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
+      {
+        i++;
+        goto NEXT;
+      }
+    return GNUNET_YES;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * We have a client. Accept the incoming socket(s) (and reschedule
+ * the listen task).
+ *
+ * @param cls the `struct ServiceListenContext` of the ready listen socket
+ */
+static void
+accept_client (void *cls)
+{
+  struct ServiceListenContext *slc = cls;
+  struct GNUNET_SERVICE_Handle *sh = slc->sh;
+
+  slc->listen_task = NULL;
+  while (1)
+  {
+    struct GNUNET_NETWORK_Handle *sock;
+    const struct sockaddr_in *v4;
+    const struct sockaddr_in6 *v6;
+    struct sockaddr_storage sa;
+    socklen_t addrlen;
+    int ok;
+
+    addrlen = sizeof (sa);
+    sock = GNUNET_NETWORK_socket_accept (slc->listen_socket,
+                                         (struct sockaddr *) &sa,
+                                         &addrlen);
+    if (NULL == sock)
+      break;
+    switch (sa.ss_family)
+    {
+    case AF_INET:
+      GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
+      v4 = (const struct sockaddr_in *) &sa;
+      ok = ( ( (NULL == sh->v4_allowed) ||
+               (check_ipv4_listed (sh->v4_allowed,
+                                   &v4->sin_addr))) &&
+             ( (NULL == sh->v4_denied) ||
+               (! check_ipv4_listed (sh->v4_denied,
+                                     &v4->sin_addr)) ) );
+      break;
+    case AF_INET6:
+      GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
+      v6 = (const struct sockaddr_in6 *) &sa;
+      ok = ( ( (NULL == sh->v6_allowed) ||
+               (check_ipv6_listed (sh->v6_allowed,
+                                   &v6->sin6_addr))) &&
+             ( (NULL == sh->v6_denied) ||
+               (! check_ipv6_listed (sh->v6_denied,
+                                     &v6->sin6_addr)) ) );
+      break;
 #ifndef WINDOWS
-  if (NULL != sctx->addrs)
-    for (i = 0; NULL != sctx->addrs[i]; i++)
-      if ((AF_UNIX == sctx->addrs[i]->sa_family)
-          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
-        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
-                                     sctx->match_uid,
-                                     sctx->match_gid);
+    case AF_UNIX:
+      ok = GNUNET_OK;            /* controlled using file-system ACL now */
+      break;
 #endif
-  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
-  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
-  i = 0;
-  while ((sctx->my_handlers[i].callback != NULL))
-    sctx->my_handlers[i++].callback_cls = sctx;
-  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
-  return sctx;
+    default:
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           _("Unknown address family %d\n"),
+           sa.ss_family);
+      return;
+    }
+    if (! ok)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Service rejected incoming connection from %s due to policy.\n",
+           GNUNET_a2s ((const struct sockaddr *) &sa,
+                       addrlen));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (sock));
+      continue;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Service accepted incoming connection from %s.\n",
+         GNUNET_a2s ((const struct sockaddr *) &sa,
+                     addrlen));
+    start_client (slc->sh,
+                  sock);
+  }
+  slc->listen_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     slc->listen_socket,
+                                     &accept_client,
+                                     slc);
+}
+
+
+/**
+ * Resume accepting connections from the listen socket.
+ *
+ * @param sh service to resume accepting connections.
+ */
+void
+GNUNET_SERVICE_resume (struct GNUNET_SERVICE_Handle *sh)
+{
+  struct ServiceListenContext *slc;
+
+  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
+  {
+    GNUNET_assert (NULL == slc->listen_task);
+    slc->listen_task
+      = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                       slc->listen_socket,
+                                       &accept_client,
+                                       slc);
+  }
 }
 
 
 /**
- * Obtain the server used by a service.  Note that the server must NOT
- * be destroyed by the caller.
+ * Task run to resume receiving data from the client after
+ * the client called #GNUNET_SERVICE_client_continue().
  *
- * @param ctx the service context returned from the start function
- * @return handle to the server for this service, NULL if there is none
+ * @param cls our `struct GNUNET_SERVICE_Client`
  */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVICE_get_server (struct GNUNET_SERVICE_Context *ctx)
+static void
+resume_client_receive (void *cls)
 {
-  return ctx->server;
+  struct GNUNET_SERVICE_Client *c = cls;
+  int ret;
+
+  c->recv_task = NULL;
+  /* first, check if there is still something in the buffer */
+  ret = GNUNET_MST_next (c->mst,
+                         GNUNET_YES);
+  if (GNUNET_SYSERR == ret)
+  {
+    if (NULL != c->drop_task)
+      GNUNET_SERVICE_client_drop (c);
+    return;
+  }
+  if (GNUNET_NO == ret)
+    return; /* done processing, wait for more later */
+  GNUNET_assert (GNUNET_OK == ret);
+  if (GNUNET_YES == c->needs_continue)
+    return; /* #GNUNET_MST_next() did give a message to the client */
+  /* need to receive more data from the network first */
+  if (NULL != c->recv_task)
+    return;
+  c->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     c->sock,
+                                     &service_client_recv,
+                                     c);
 }
 
 
 /**
- * Get the NULL-terminated array of listen sockets for this service.
+ * Continue receiving further messages from the given client.
+ * Must be called after each message received.
  *
- * @param ctx service context to query
- * @return NULL if there are no listen sockets, otherwise NULL-terminated
- *              array of listen sockets.
+ * @param c the client to continue receiving from
  */
-struct GNUNET_NETWORK_Handle *const*
-GNUNET_SERVICE_get_listen_sockets (struct GNUNET_SERVICE_Context *ctx)
+void
+GNUNET_SERVICE_client_continue (struct GNUNET_SERVICE_Client *c)
 {
-  return ctx->lsocks;
+  GNUNET_assert (GNUNET_YES == c->needs_continue);
+  GNUNET_assert (NULL == c->recv_task);
+  c->needs_continue = GNUNET_NO;
+  if (NULL != c->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
+  }
+  c->recv_task
+    = GNUNET_SCHEDULER_add_now (&resume_client_receive,
+                                c);
 }
 
 
 /**
- * Stop a service that was started with "GNUNET_SERVICE_start".
+ * Disable the warning the server issues if a message is not
+ * acknowledged in a timely fashion.  Use this call if a client is
+ * intentionally delayed for a while.  Only applies to the current
+ * message.
  *
- * @param sctx the service context returned from the start function
+ * @param c client for which to disable the warning
  */
 void
-GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Context *sctx)
+GNUNET_SERVICE_client_disable_continue_warning (struct GNUNET_SERVICE_Client *c)
 {
-  unsigned int i;
+  GNUNET_break (NULL != c->warn_task);
+  if (NULL != c->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
+  }
+}
 
-#if HAVE_MALLINFO
+
+/**
+ * Asynchronously finish dropping the client.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client`.
+ */
+static void
+finish_client_drop (void *cls)
+{
+  struct GNUNET_SERVICE_Client *c = cls;
+  struct GNUNET_SERVICE_Handle *sh = c->sh;
+
+  c->drop_task = NULL;
+  GNUNET_assert (NULL == c->send_task);
+  GNUNET_assert (NULL == c->recv_task);
+  GNUNET_assert (NULL == c->warn_task);
+  GNUNET_MST_destroy (c->mst);
+  GNUNET_MQ_destroy (c->mq);
+  if (GNUNET_NO == c->persist)
   {
-    char *counter;
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (c->sock));
+  }
+  else
+  {
+    GNUNET_NETWORK_socket_free_memory_only_ (c->sock);
+  }
+  GNUNET_free (c);
+  if ( (GNUNET_YES == sh->got_shutdown) &&
+       (GNUNET_NO == have_non_monitor_clients (sh)) )
+    GNUNET_SERVICE_shutdown (sh);
+}
 
-    if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name,
-                                           "GAUGER_HEAP")) &&
-         (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sctx->cfg, sctx->service_name,
-                                                 "GAUGER_HEAP",
-                                                 &counter)) )
-    {
-      struct mallinfo mi;
 
-      mi = mallinfo ();
-      GAUGER (sctx->service_name, counter, mi.usmblks, "blocks");
-      GNUNET_free (counter);
-    }
+/**
+ * Ask the server to disconnect from the given client.  This is the
+ * same as returning #GNUNET_SYSERR within the check procedure when
+ * handling a message, wexcept that it allows dropping of a client even
+ * when not handling a message from that client.  The `disconnect_cb`
+ * will be called on @a c even if the application closes the connection
+ * using this function.
+ *
+ * @param c client to disconnect now
+ */
+void
+GNUNET_SERVICE_client_drop (struct GNUNET_SERVICE_Client *c)
+{
+  struct GNUNET_SERVICE_Handle *sh = c->sh;
+
+  if (NULL != c->drop_task)
+  {
+    /* asked to drop twice! */
+    GNUNET_assert (0);
+    return;
   }
-#endif
-  if (NULL != sctx->shutdown_task)
+  GNUNET_CONTAINER_DLL_remove (sh->clients_head,
+                               sh->clients_tail,
+                               c);
+  if (NULL != sh->disconnect_cb)
+    sh->disconnect_cb (sh->cb_cls,
+                       c,
+                       c->user_context);
+  if (NULL != c->warn_task)
   {
-    GNUNET_SCHEDULER_cancel (sctx->shutdown_task);
-    sctx->shutdown_task = NULL;
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
   }
-  if (NULL != sctx->server)
-    GNUNET_SERVER_destroy (sctx->server);
-  GNUNET_free_non_null (sctx->my_handlers);
-  if (NULL != sctx->addrs)
+  if (NULL != c->recv_task)
   {
-    i = 0;
-    while (NULL != sctx->addrs[i])
-      GNUNET_free (sctx->addrs[i++]);
-    GNUNET_free (sctx->addrs);
-  }
-  GNUNET_free_non_null (sctx->addrlens);
-  GNUNET_free_non_null (sctx->v4_denied);
-  GNUNET_free_non_null (sctx->v6_denied);
-  GNUNET_free_non_null (sctx->v4_allowed);
-  GNUNET_free_non_null (sctx->v6_allowed);
-  GNUNET_free (sctx);
+    GNUNET_SCHEDULER_cancel (c->recv_task);
+    c->recv_task = NULL;
+  }
+  if (NULL != c->send_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->send_task);
+    c->send_task = NULL;
+  }
+  c->drop_task = GNUNET_SCHEDULER_add_now (&finish_client_drop,
+                                           c);
+}
+
+
+/**
+ * Explicitly stops the service.
+ *
+ * @param sh server to shutdown
+ */
+void
+GNUNET_SERVICE_shutdown (struct GNUNET_SERVICE_Handle *sh)
+{
+  struct GNUNET_SERVICE_Client *client;
+
+  GNUNET_SERVICE_suspend (sh);
+  sh->got_shutdown = GNUNET_NO;
+  while (NULL != (client = sh->clients_head))
+    GNUNET_SERVICE_client_drop (client);
+}
+
+
+/**
+ * Set the 'monitor' flag on this client.  Clients which have been
+ * marked as 'monitors' won't prevent the server from shutting down
+ * once #GNUNET_SERVICE_stop_listening() has been invoked.  The idea is
+ * that for "normal" clients we likely want to allow them to process
+ * their requests; however, monitor-clients are likely to 'never'
+ * disconnect during shutdown and thus will not be considered when
+ * determining if the server should continue to exist after
+ * shutdown has been triggered.
+ *
+ * @param c client to mark as a monitor
+ */
+void
+GNUNET_SERVICE_client_mark_monitor (struct GNUNET_SERVICE_Client *c)
+{
+  c->is_monitor = GNUNET_YES;
+  if ( (GNUNET_YES == c->sh->got_shutdown) &&
+       (GNUNET_NO == have_non_monitor_clients (c->sh)) )
+    GNUNET_SERVICE_shutdown (c->sh);
+}
+
+
+/**
+ * Set the persist option on this client.  Indicates that the
+ * underlying socket or fd should never really be closed.  Used for
+ * indicating process death.
+ *
+ * @param c client to persist the socket (never to be closed)
+ */
+void
+GNUNET_SERVICE_client_persist (struct GNUNET_SERVICE_Client *c)
+{
+  c->persist = GNUNET_YES;
+}
+
+
+/**
+ * Obtain the message queue of @a c.  Convenience function.
+ *
+ * @param c the client to continue receiving from
+ * @return the message queue of @a c
+ */
+struct GNUNET_MQ_Handle *
+GNUNET_SERVICE_client_get_mq (struct GNUNET_SERVICE_Client *c)
+{
+  return c->mq;
 }
 
 
-/* end of service.c */
+/* end of service_new.c */
diff --git a/src/util/service_new.c b/src/util/service_new.c
deleted file mode 100644
index a893d287b..000000000
--- a/src/util/service_new.c
+++ /dev/null
@@ -1,2617 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/service_new.c
- * @brief functions related to starting services (redesign)
- * @author Christian Grothoff
- * @author Florian Dold
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_protocols.h"
-#include "gnunet_constants.h"
-#include "gnunet_resolver_service.h"
-#include "speedup.h"
-
-#if HAVE_MALLINFO
-#include <malloc.h>
-#include "gauger.h"
-#endif
-
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
-
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
-
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
-
-
-/**
- * Information the service tracks per listen operation.
- */
-struct ServiceListenContext
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct ServiceListenContext *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct ServiceListenContext *prev;
-
-  /**
-   * Service this listen context belongs to.
-   */
-  struct GNUNET_SERVICE_Handle *sh;
-
-  /**
-   * Socket we are listening on.
-   */
-  struct GNUNET_NETWORK_Handle *listen_socket;
-
-  /**
-   * Task scheduled to do the listening.
-   */
-  struct GNUNET_SCHEDULER_Task *listen_task;
-
-};
-
-
-/**
- * Handle to a service.
- */
-struct GNUNET_SERVICE_Handle
-{
-  /**
-   * Our configuration.
-   */
-  const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-  /**
-   * Name of our service.
-   */
-  const char *service_name;
-
-  /**
-   * Main service-specific task to run.
-   */
-  GNUNET_SERVICE_InitCallback service_init_cb;
-
-  /**
-   * Function to call when clients connect.
-   */
-  GNUNET_SERVICE_ConnectHandler connect_cb;
-
-  /**
-   * Function to call when clients disconnect / are disconnected.
-   */
-  GNUNET_SERVICE_DisconnectHandler disconnect_cb;
-
-  /**
-   * Closure for @e service_init_cb, @e connect_cb, @e disconnect_cb.
-   */
-  void *cb_cls;
-
-  /**
-   * DLL of listen sockets used to accept new connections.
-   */
-  struct ServiceListenContext *slc_head;
-
-  /**
-   * DLL of listen sockets used to accept new connections.
-   */
-  struct ServiceListenContext *slc_tail;
-
-  /**
-   * Our clients, kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *clients_head;
-
-  /**
-   * Our clients, kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *clients_tail;
-
-  /**
-   * Message handlers to use for all clients.
-   */
-  struct GNUNET_MQ_MessageHandler *handlers;
-
-  /**
-   * Closure for @e task.
-   */
-  void *task_cls;
-
-  /**
-   * IPv4 addresses that are not allowed to connect.
-   */
-  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_denied;
-
-  /**
-   * IPv6 addresses that are not allowed to connect.
-   */
-  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_denied;
-
-  /**
-   * IPv4 addresses that are allowed to connect (if not
-   * set, all are allowed).
-   */
-  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_allowed;
-
-  /**
-   * IPv6 addresses that are allowed to connect (if not
-   * set, all are allowed).
-   */
-  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
-
-  /**
-   * Do we require a matching UID for UNIX domain socket connections?
-   * #GNUNET_NO means that the UID does not have to match (however,
-   * @e match_gid may still impose other access control checks).
-   */
-  int match_uid;
-
-  /**
-   * Do we require a matching GID for UNIX domain socket connections?
-   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
-   * checking that the client's UID is in our group OR that the
-   * client's GID is our GID.  If both "match_gid" and @e match_uid are
-   * #GNUNET_NO, all users on the local system have access.
-   */
-  int match_gid;
-
-  /**
-   * Set to #GNUNET_YES if we got a shutdown signal and terminate
-   * the service if #have_non_monitor_clients() returns #GNUNET_YES.
-   */
-  int got_shutdown;
-
-  /**
-   * Our options.
-   */
-  enum GNUNET_SERVICE_Options options;
-
-  /**
-   * If we are daemonizing, this FD is set to the
-   * pipe to the parent.  Send '.' if we started
-   * ok, '!' if not.  -1 if we are not daemonizing.
-   */
-  int ready_confirm_fd;
-
-  /**
-   * Overall success/failure of the service start.
-   */
-  int ret;
-
-  /**
-   * If #GNUNET_YES, consider unknown message types an error where the
-   * client is disconnected.
-   */
-  int require_found;
-};
-
-
-/**
- * Handle to a client that is connected to a service.
- */
-struct GNUNET_SERVICE_Client
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *prev;
-
-  /**
-   * Service that this client belongs to.
-   */
-  struct GNUNET_SERVICE_Handle *sh;
-
-  /**
-   * Socket of this client.
-   */
-  struct GNUNET_NETWORK_Handle *sock;
-
-  /**
-   * Message queue for the client.
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Tokenizer we use for processing incoming data.
-   */
-  struct GNUNET_MessageStreamTokenizer *mst;
-
-  /**
-   * Task that warns about missing calls to
-   * #GNUNET_SERVICE_client_continue().
-   */
-  struct GNUNET_SCHEDULER_Task *warn_task;
-
-  /**
-   * Task run to finish dropping the client after the stack has
-   * properly unwound.
-   */
-  struct GNUNET_SCHEDULER_Task *drop_task;
-
-  /**
-   * Task that receives data from the client to
-   * pass it to the handlers.
-   */
-  struct GNUNET_SCHEDULER_Task *recv_task;
-
-  /**
-   * Task that transmit data to the client.
-   */
-  struct GNUNET_SCHEDULER_Task *send_task;
-
-  /**
-   * Pointer to the message to be transmitted by @e send_task.
-   */
-  const struct GNUNET_MessageHeader *msg;
-
-  /**
-   * User context value, value returned from
-   * the connect callback.
-   */
-  void *user_context;
-
-  /**
-   * Time when we last gave a message from this client
-   * to the application.
-   */
-  struct GNUNET_TIME_Absolute warn_start;
-
-  /**
-   * Current position in @e msg at which we are transmitting.
-   */
-  size_t msg_pos;
-
-  /**
-   * Persist the file handle for this client no matter what happens,
-   * force the OS to close once the process actually dies.  Should only
-   * be used in special cases!
-   */
-  int persist;
-
-  /**
-   * Is this client a 'monitor' client that should not be counted
-   * when deciding on destroying the server during soft shutdown?
-   * (see also #GNUNET_SERVICE_start)
-   */
-  int is_monitor;
-
-  /**
-   * Are we waiting for the application to call #GNUNET_SERVICE_client_continue()?
-   */
-  int needs_continue;
-
-  /**
-   * Type of last message processed (for warn_no_receive_done).
-   */
-  uint16_t warn_type;
-};
-
-
-/**
- * Check if any of the clients we have left are unrelated to
- * monitoring.
- *
- * @param sh service to check clients for
- * @return #GNUNET_YES if we have non-monitoring clients left
- */
-static int
-have_non_monitor_clients (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  for (client = sh->clients_head;NULL != client; client = client->next)
-  {
-    if (client->is_monitor)
-      continue;
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * Shutdown task triggered when a service should be terminated.
- * This considers active clients and the service options to see
- * how this specific service is to be terminated, and depending
- * on this proceeds with the shutdown logic.
- *
- * @param cls our `struct GNUNET_SERVICE_Handle`
- */
-static void
-service_shutdown (void *cls)
-{
-  struct GNUNET_SERVICE_Handle *sh = cls;
-
-  switch (sh->options)
-  {
-  case GNUNET_SERVICE_OPTION_NONE:
-    GNUNET_SERVICE_shutdown (sh);
-    break;
-  case GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN:
-    /* This task should never be run if we are using
-       the manual shutdown. */
-    GNUNET_assert (0);
-    break;
-  case GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN:
-    sh->got_shutdown = GNUNET_YES;
-    GNUNET_SERVICE_suspend (sh);
-    if (GNUNET_NO == have_non_monitor_clients (sh))
-      GNUNET_SERVICE_shutdown (sh);
-    break;
-  }
-}
-
-
-/**
- * First task run by any service.  Initializes our shutdown task,
- * starts the listening operation on our listen sockets and launches
- * the custom logic of the application service.
- *
- * @param cls our `struct GNUNET_SERVICE_Handle`
- */
-static void
-service_main (void *cls)
-{
-  struct GNUNET_SERVICE_Handle *sh = cls;
-
-  if (GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN != sh->options)
-    GNUNET_SCHEDULER_add_shutdown (&service_shutdown,
-                                   sh);
-  GNUNET_SERVICE_resume (sh);
-
-  if (-1 != sh->ready_confirm_fd)
-  {
-    GNUNET_break (1 == WRITE (sh->ready_confirm_fd, ".", 1));
-    GNUNET_break (0 == CLOSE (sh->ready_confirm_fd));
-    sh->ready_confirm_fd = -1;
-  }
-
-  if (NULL != sh->service_init_cb)
-    sh->service_init_cb (sh->cb_cls,
-                         sh->cfg,
-                         sh);
-}
-
-
-/**
- * Parse an IPv4 access control list.
- *
- * @param ret location where to write the ACL (set)
- * @param sh service context to use to get the configuration
- * @param option name of the ACL option to parse
- * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
- *         no ACL configured)
- */
-static int
-process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Handle *sh,
-              const char *option)
-{
-  char *opt;
-
-  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
-                                         sh->service_name,
-                                         option))
-  {
-    *ret = NULL;
-    return GNUNET_OK;
-  }
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
-                                                       sh->service_name,
-                                                       option,
-                                                       &opt));
-  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
-         opt,
-         sh->service_name,
-         option);
-    GNUNET_free (opt);
-    return GNUNET_SYSERR;
-  }
-  GNUNET_free (opt);
-  return GNUNET_OK;
-}
-
-
-/**
- * Parse an IPv6 access control list.
- *
- * @param ret location where to write the ACL (set)
- * @param sh service context to use to get the configuration
- * @param option name of the ACL option to parse
- * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
- *         no ACL configured)
- */
-static int
-process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Handle *sh,
-              const char *option)
-{
-  char *opt;
-
-  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
-                                         sh->service_name,
-                                         option))
-  {
-    *ret = NULL;
-    return GNUNET_OK;
-  }
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
-                                                       sh->service_name,
-                                                       option,
-                                                       &opt));
-  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
-         opt,
-         sh->service_name,
-         option);
-    GNUNET_free (opt);
-    return GNUNET_SYSERR;
-  }
-  GNUNET_free (opt);
-  return GNUNET_OK;
-}
-
-
-/**
- * Add the given UNIX domain path as an address to the
- * list (as the first entry).
- *
- * @param saddrs array to update
- * @param saddrlens where to store the address length
- * @param unixpath path to add
- * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
- *          parameter is ignore on systems other than LINUX
- */
-static void
-add_unixpath (struct sockaddr **saddrs,
-              socklen_t *saddrlens,
-              const char *unixpath,
-              int abstract)
-{
-#ifdef AF_UNIX
-  struct sockaddr_un *un;
-
-  un = GNUNET_new (struct sockaddr_un);
-  un->sun_family = AF_UNIX;
-  strncpy (un->sun_path,
-           unixpath,
-           sizeof (un->sun_path) - 1);
-#ifdef LINUX
-  if (GNUNET_YES == abstract)
-    un->sun_path[0] = '\0';
-#endif
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  un->sun_len = (u_char) sizeof (struct sockaddr_un);
-#endif
-  *saddrs = (struct sockaddr *) un;
-  *saddrlens = sizeof (struct sockaddr_un);
-#else
-  /* this function should never be called
-   * unless AF_UNIX is defined! */
-  GNUNET_assert (0);
-#endif
-}
-
-
-/**
- * Get the list of addresses that a server for the given service
- * should bind to.
- *
- * @param service_name name of the service
- * @param cfg configuration (which specifies the addresses)
- * @param addrs set (call by reference) to an array of pointers to the
- *              addresses the server should bind to and listen on; the
- *              array will be NULL-terminated (on success)
- * @param addr_lens set (call by reference) to an array of the lengths
- *              of the respective `struct sockaddr` struct in the @a addrs
- *              array (on success)
- * @return number of addresses found on success,
- *              #GNUNET_SYSERR if the configuration
- *              did not specify reasonable finding information or
- *              if it specified a hostname that could not be resolved;
- *              #GNUNET_NO if the number of addresses configured is
- *              zero (in this case, `*addrs` and `*addr_lens` will be
- *              set to NULL).
- */
-static int
-get_server_addresses (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      struct sockaddr ***addrs,
-                      socklen_t **addr_lens)
-{
-  int disablev6;
-  struct GNUNET_NETWORK_Handle *desc;
-  unsigned long long port;
-  char *unixpath;
-  struct addrinfo hints;
-  struct addrinfo *res;
-  struct addrinfo *pos;
-  struct addrinfo *next;
-  unsigned int i;
-  int resi;
-  int ret;
-  int abstract;
-  struct sockaddr **saddrs;
-  socklen_t *saddrlens;
-  char *hostname;
-
-  *addrs = NULL;
-  *addr_lens = NULL;
-  desc = NULL;
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "DISABLEV6"))
-  {
-    if (GNUNET_SYSERR ==
-        (disablev6 =
-         GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                                               service_name,
-                                               "DISABLEV6")))
-      return GNUNET_SYSERR;
-  }
-  else
-    disablev6 = GNUNET_NO;
-
-  if (! disablev6)
-  {
-    /* probe IPv6 support */
-    desc = GNUNET_NETWORK_socket_create (PF_INET6,
-                                         SOCK_STREAM,
-                                         0);
-    if (NULL == desc)
-    {
-      if ( (ENOBUFS == errno) ||
-           (ENOMEM == errno) ||
-           (ENFILE == errno) ||
-           (EACCES == errno) )
-      {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                      "socket");
-        return GNUNET_SYSERR;
-      }
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
-           service_name,
-           STRERROR (errno));
-      disablev6 = GNUNET_YES;
-    }
-    else
-    {
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (desc));
-      desc = NULL;
-    }
-  }
-
-  port = 0;
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "PORT"))
-  {
-    if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_number (cfg,
-                                               service_name,
-                                               "PORT",
-                                               &port))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Require valid port number for service `%s' in configuration!\n"),
-           service_name);
-    }
-    if (port > 65535)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Require valid port number for service `%s' in configuration!\n"),
-           service_name);
-      return GNUNET_SYSERR;
-    }
-  }
-
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "BINDTO"))
-  {
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_string (cfg,
-                                                         service_name,
-                                                         "BINDTO",
-                                                         &hostname));
-  }
-  else
-    hostname = NULL;
-
-  unixpath = NULL;
-  abstract = GNUNET_NO;
-#ifdef AF_UNIX
-  if ((GNUNET_YES ==
-       GNUNET_CONFIGURATION_have_value (cfg,
-                                        service_name,
-                                        "UNIXPATH")) &&
-      (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_filename (cfg,
-                                                service_name,
-                                                "UNIXPATH",
-                                                &unixpath)) &&
-      (0 < strlen (unixpath)))
-  {
-    /* probe UNIX support */
-    struct sockaddr_un s_un;
-
-    if (strlen (unixpath) >= sizeof (s_un.sun_path))
-    {
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("UNIXPATH `%s' too long, maximum length is %llu\n"),
-           unixpath,
-           (unsigned long long) sizeof (s_un.sun_path));
-      unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Using `%s' instead\n"),
-           unixpath);
-    }
-#ifdef LINUX
-    abstract = GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                                                     "TESTING",
-                                                     "USE_ABSTRACT_SOCKETS");
-    if (GNUNET_SYSERR == abstract)
-      abstract = GNUNET_NO;
-#endif
-    if ( (GNUNET_YES != abstract) &&
-         (GNUNET_OK !=
-          GNUNET_DISK_directory_create_for_file (unixpath)) )
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
-                                "mkdir",
-                                unixpath);
-  }
-  if (NULL != unixpath)
-  {
-    desc = GNUNET_NETWORK_socket_create (AF_UNIX,
-                                         SOCK_STREAM,
-                                         0);
-    if (NULL == desc)
-    {
-      if ((ENOBUFS == errno) ||
-          (ENOMEM == errno) ||
-          (ENFILE == errno) ||
-          (EACCES == errno))
-      {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                      "socket");
-        GNUNET_free_non_null (hostname);
-        GNUNET_free (unixpath);
-        return GNUNET_SYSERR;
-      }
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Disabling UNIX domain socket support for service `%s', failed to create UNIX domain socket: %s\n"),
-           service_name,
-           STRERROR (errno));
-      GNUNET_free (unixpath);
-      unixpath = NULL;
-    }
-    else
-    {
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (desc));
-      desc = NULL;
-    }
-  }
-#endif
-
-  if ((0 == port) && (NULL == unixpath))
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Have neither PORT nor UNIXPATH for service `%s', but one is required\n"),
-         service_name);
-    GNUNET_free_non_null (hostname);
-    return GNUNET_SYSERR;
-  }
-  if (0 == port)
-  {
-    saddrs = GNUNET_new_array (2,
-                               struct sockaddr *);
-    saddrlens = GNUNET_new_array (2,
-                                  socklen_t);
-    add_unixpath (saddrs,
-                  saddrlens,
-                  unixpath,
-                  abstract);
-    GNUNET_free_non_null (unixpath);
-    GNUNET_free_non_null (hostname);
-    *addrs = saddrs;
-    *addr_lens = saddrlens;
-    return 1;
-  }
-
-  if (NULL != hostname)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Resolving `%s' since that is where `%s' will bind to.\n",
-         hostname,
-         service_name);
-    memset (&hints,
-            0,
-            sizeof (struct addrinfo));
-    if (disablev6)
-      hints.ai_family = AF_INET;
-    hints.ai_protocol = IPPROTO_TCP;
-    if ((0 != (ret = getaddrinfo (hostname,
-                                  NULL,
-                                  &hints,
-                                  &res))) ||
-        (NULL == res))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Failed to resolve `%s': %s\n"),
-           hostname,
-           gai_strerror (ret));
-      GNUNET_free (hostname);
-      GNUNET_free_non_null (unixpath);
-      return GNUNET_SYSERR;
-    }
-    next = res;
-    i = 0;
-    while (NULL != (pos = next))
-    {
-      next = pos->ai_next;
-      if ( (disablev6) &&
-           (pos->ai_family == AF_INET6) )
-        continue;
-      i++;
-    }
-    if (0 == i)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Failed to find %saddress for `%s'.\n"),
-           disablev6 ? "IPv4 " : "",
-           hostname);
-      freeaddrinfo (res);
-      GNUNET_free (hostname);
-      GNUNET_free_non_null (unixpath);
-      return GNUNET_SYSERR;
-    }
-    resi = i;
-    if (NULL != unixpath)
-      resi++;
-    saddrs = GNUNET_new_array (resi + 1,
-                               struct sockaddr *);
-    saddrlens = GNUNET_new_array (resi + 1,
-                                  socklen_t);
-    i = 0;
-    if (NULL != unixpath)
-    {
-      add_unixpath (saddrs,
-                    saddrlens,
-                    unixpath,
-                    abstract);
-      i++;
-    }
-    next = res;
-    while (NULL != (pos = next))
-    {
-      next = pos->ai_next;
-      if ( (disablev6) &&
-           (AF_INET6 == pos->ai_family) )
-        continue;
-      if ( (IPPROTO_TCP != pos->ai_protocol) &&
-           (0 != pos->ai_protocol) )
-        continue;               /* not TCP */
-      if ( (SOCK_STREAM != pos->ai_socktype) &&
-           (0 != pos->ai_socktype) )
-        continue;               /* huh? */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Service `%s' will bind to `%s'\n",
-           service_name,
-           GNUNET_a2s (pos->ai_addr,
-                       pos->ai_addrlen));
-      if (AF_INET == pos->ai_family)
-      {
-        GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
-        saddrlens[i] = pos->ai_addrlen;
-        saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i],
-                       pos->ai_addr,
-                       saddrlens[i]);
-        ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-      }
-      else
-      {
-        GNUNET_assert (AF_INET6 == pos->ai_family);
-        GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
-        saddrlens[i] = pos->ai_addrlen;
-        saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i],
-                       pos->ai_addr,
-                       saddrlens[i]);
-        ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
-      }
-      i++;
-    }
-    GNUNET_free (hostname);
-    freeaddrinfo (res);
-    resi = i;
-  }
-  else
-  {
-    /* will bind against everything, just set port */
-    if (disablev6)
-    {
-      /* V4-only */
-      resi = 1;
-      if (NULL != unixpath)
-        resi++;
-      i = 0;
-      saddrs = GNUNET_new_array (resi + 1,
-                                 struct sockaddr *);
-      saddrlens = GNUNET_new_array (resi + 1,
-                                    socklen_t);
-      if (NULL != unixpath)
-      {
-        add_unixpath (saddrs,
-                      saddrlens,
-                      unixpath,
-                      abstract);
-        i++;
-      }
-      saddrlens[i] = sizeof (struct sockaddr_in);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[i];
-#endif
-      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
-      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-    }
-    else
-    {
-      /* dual stack */
-      resi = 2;
-      if (NULL != unixpath)
-        resi++;
-      saddrs = GNUNET_new_array (resi + 1,
-                                 struct sockaddr *);
-      saddrlens = GNUNET_new_array (resi + 1,
-                                    socklen_t);
-      i = 0;
-      if (NULL != unixpath)
-      {
-        add_unixpath (saddrs,
-                      saddrlens,
-                      unixpath,
-                      abstract);
-        i++;
-      }
-      saddrlens[i] = sizeof (struct sockaddr_in6);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_len = saddrlens[0];
-#endif
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_family = AF_INET6;
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
-      i++;
-      saddrlens[i] = sizeof (struct sockaddr_in);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[1];
-#endif
-      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
-      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-    }
-  }
-  GNUNET_free_non_null (unixpath);
-  *addrs = saddrs;
-  *addr_lens = saddrlens;
-  return resi;
-}
-
-
-#ifdef MINGW
-/**
- * Read listen sockets from the parent process (ARM).
- *
- * @param sh service context to initialize
- * @return NULL-terminated array of sockets on success,
- *         NULL if not ok (must bind yourself)
- */
-static struct GNUNET_NETWORK_Handle **
-receive_sockets_from_parent (struct GNUNET_SERVICE_Handle *sh)
-{
-  static struct GNUNET_NETWORK_Handle **lsocks;
-  const char *env_buf;
-  int fail;
-  uint64_t count;
-  uint64_t i;
-  HANDLE lsocks_pipe;
-
-  env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
-  if ( (NULL == env_buf) ||
-       (strlen (env_buf) <= 0) )
-    return NULL;
-  /* Using W32 API directly here, because this pipe will
-   * never be used outside of this function, and it's just too much of a bother
-   * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
-   */
-  lsocks_pipe = (HANDLE) strtoul (env_buf,
-                                  NULL,
-                                  10);
-  if ( (0 == lsocks_pipe) ||
-       (INVALID_HANDLE_VALUE == lsocks_pipe))
-    return NULL;
-  fail = 1;
-  do
-  {
-    int ret;
-    int fail2;
-    DWORD rd;
-
-    ret = ReadFile (lsocks_pipe,
-                    &count,
-                    sizeof (count),
-                    &rd,
-                    NULL);
-    if ( (0 == ret) ||
-         (sizeof (count) != rd) ||
-         (0 == count) )
-      break;
-    lsocks = GNUNET_new_array (count + 1,
-                               struct GNUNET_NETWORK_Handle *);
-
-    fail2 = 1;
-    for (i = 0; i < count; i++)
-    {
-      WSAPROTOCOL_INFOA pi;
-      uint64_t size;
-      SOCKET s;
-
-      ret = ReadFile (lsocks_pipe,
-                      &size,
-                      sizeof (size),
-                      &rd,
-                      NULL);
-      if ( (0 == ret) ||
-           (sizeof (size) != rd) ||
-           (sizeof (pi) != size) )
-        break;
-      ret = ReadFile (lsocks_pipe,
-                      &pi,
-                      sizeof (pi),
-                      &rd,
-                      NULL);
-      if ( (0 == ret) ||
-           (sizeof (pi) != rd))
-        break;
-      s = WSASocketA (pi.iAddressFamily,
-                      pi.iSocketType,
-                      pi.iProtocol,
-                      &pi,
-                      0,
-                      WSA_FLAG_OVERLAPPED);
-      lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
-      if (NULL == lsocks[i])
-        break;
-      else if (i == count - 1)
-        fail2 = 0;
-    }
-    if (fail2)
-      break;
-    lsocks[count] = NULL;
-    fail = 0;
-  }
-  while (fail);
-  CloseHandle (lsocks_pipe);
-
-  if (fail)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Could not access a pre-bound socket, will try to bind myself\n"));
-    for (i = 0; (i < count) && (NULL != lsocks[i]); i++)
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (lsocks[i]));
-    GNUNET_free (lsocks);
-    return NULL;
-  }
-  return lsocks;
-}
-#endif
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @param server_addr address to listen on
- * @param socklen length of @a server_addr
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket (const struct sockaddr *server_addr,
-                    socklen_t socklen)
-{
-  struct GNUNET_NETWORK_Handle *sock;
-  uint16_t port;
-  int eno;
-
-  switch (server_addr->sa_family)
-  {
-  case AF_INET:
-    port = ntohs (((const struct sockaddr_in *) server_addr)->sin_port);
-    break;
-  case AF_INET6:
-    port = ntohs (((const struct sockaddr_in6 *) server_addr)->sin6_port);
-    break;
-  case AF_UNIX:
-    port = 0;
-    break;
-  default:
-    GNUNET_break (0);
-    port = 0;
-    break;
-  }
-  sock = GNUNET_NETWORK_socket_create (server_addr->sa_family,
-                                       SOCK_STREAM,
-                                       0);
-  if (NULL == sock)
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "socket");
-    errno = 0;
-    return NULL;
-  }
-  /* bind the socket */
-  if (GNUNET_OK != GNUNET_NETWORK_socket_bind (sock,
-                                               server_addr,
-                                               socklen))
-  {
-    eno = errno;
-    if (EADDRINUSE != errno)
-    {
-      /* we don't log 'EADDRINUSE' here since an IPv4 bind may
-       * fail if we already took the port on IPv6; if both IPv4 and
-       * IPv6 binds fail, then our caller will log using the
-       * errno preserved in 'eno' */
-      LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                    "bind");
-      if (0 != port)
-        LOG (GNUNET_ERROR_TYPE_ERROR,
-             _("`%s' failed for port %d (%s).\n"),
-             "bind",
-             port,
-             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
-      eno = 0;
-    }
-    else
-    {
-      if (0 != port)
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             _("`%s' failed for port %d (%s): address already in use\n"),
-             "bind", port,
-             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
-      else if (AF_UNIX == server_addr->sa_family)
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             _("`%s' failed for `%s': address already in use\n"),
-             "bind",
-             GNUNET_a2s (server_addr, socklen));
-      }
-    }
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (sock));
-    errno = eno;
-    return NULL;
-  }
-  if (GNUNET_OK != GNUNET_NETWORK_socket_listen (sock,
-                                                 5))
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "listen");
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (sock));
-    errno = 0;
-    return NULL;
-  }
-  if (0 != port)
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Server starts to listen on port %u.\n",
-         port);
-  return sock;
-}
-
-
-/**
- * Setup service handle
- *
- * Configuration may specify:
- * - PORT (where to bind to for TCP)
- * - UNIXPATH (where to bind to for UNIX domain sockets)
- * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
- * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
- * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
- * - ACCEPT_FROM6 (only allow connections from specified IPv6 subnets)
- * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
- * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
- *
- * @param sh service context to initialize
- * @return #GNUNET_OK if configuration succeeded
- */
-static int
-setup_service (struct GNUNET_SERVICE_Handle *sh)
-{
-  int tolerant;
-  struct GNUNET_NETWORK_Handle **lsocks;
-#ifndef MINGW
-  const char *nfds;
-  unsigned int cnt;
-  int flags;
-#endif
-
-  if (GNUNET_CONFIGURATION_have_value
-      (sh->cfg,
-       sh->service_name,
-       "TOLERANT"))
-  {
-    if (GNUNET_SYSERR ==
-        (tolerant =
-         GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                               sh->service_name,
-                                               "TOLERANT")))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TOLERANT",
-           sh->service_name);
-      return GNUNET_SYSERR;
-    }
-  }
-  else
-    tolerant = GNUNET_NO;
-
-  lsocks = NULL;
-#ifndef MINGW
-  errno = 0;
-  if ( (NULL != (nfds = getenv ("LISTEN_FDS"))) &&
-       (1 == SSCANF (nfds,
-                     "%u",
-                     &cnt)) &&
-       (cnt > 0) &&
-       (cnt < FD_SETSIZE) &&
-       (cnt + 4 < FD_SETSIZE) )
-  {
-    lsocks = GNUNET_new_array (cnt + 1,
-                               struct GNUNET_NETWORK_Handle *);
-    while (0 < cnt--)
-    {
-      flags = fcntl (3 + cnt,
-                     F_GETFD);
-      if ( (flags < 0) ||
-           (0 != (flags & FD_CLOEXEC)) ||
-           (NULL ==
-            (lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
-      {
-        LOG (GNUNET_ERROR_TYPE_ERROR,
-             _("Could not access pre-bound socket %u, will try to bind myself\n"),
-             (unsigned int) 3 + cnt);
-        cnt++;
-        while (NULL != lsocks[cnt])
-          GNUNET_break (GNUNET_OK ==
-                        GNUNET_NETWORK_socket_close (lsocks[cnt++]));
-        GNUNET_free (lsocks);
-        lsocks = NULL;
-        break;
-      }
-    }
-    unsetenv ("LISTEN_FDS");
-  }
-#else
-  if (NULL != getenv ("GNUNET_OS_READ_LSOCKS"))
-  {
-    lsocks = receive_sockets_from_parent (sh);
-    putenv ("GNUNET_OS_READ_LSOCKS=");
-  }
-#endif
-
-  if (NULL != lsocks)
-  {
-    /* listen only on inherited sockets if we have any */
-    struct GNUNET_NETWORK_Handle **ls;
-
-    for (ls = lsocks; NULL != *ls; ls++)
-    {
-      struct ServiceListenContext *slc;
-
-      slc = GNUNET_new (struct ServiceListenContext);
-      slc->sh = sh;
-      slc->listen_socket = *ls;
-      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
-                                   sh->slc_tail,
-                                   slc);
-    }
-    GNUNET_free (lsocks);
-  }
-  else
-  {
-    struct sockaddr **addrs;
-    socklen_t *addrlens;
-    int num;
-
-    num = get_server_addresses (sh->service_name,
-                                sh->cfg,
-                                &addrs,
-                                &addrlens);
-    if (GNUNET_SYSERR == num)
-      return GNUNET_SYSERR;
-
-    for (int i = 0; i < num; i++)
-    {
-      struct ServiceListenContext *slc;
-
-      slc = GNUNET_new (struct ServiceListenContext);
-      slc->sh = sh;
-      slc->listen_socket = open_listen_socket (addrs[i],
-                                               addrlens[i]);
-      if (NULL == slc->listen_socket)
-      {
-        GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
-                             "bind");
-        GNUNET_free (addrs[i++]);
-        GNUNET_free (slc);
-        continue;
-      }
-      GNUNET_free (addrs[i++]);
-      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
-                                   sh->slc_tail,
-                                   slc);
-    }
-    GNUNET_free_non_null (addrlens);
-    GNUNET_free_non_null (addrs);
-    if ( (0 != num) &&
-         (NULL == sh->slc_head) )
-    {
-      /* All attempts to bind failed, hard failure */
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Could not bind to any of the ports I was supposed to, refusing to run!\n"));
-      return GNUNET_SYSERR;
-    }
-  }
-
-  sh->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
-  sh->match_uid
-    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                            sh->service_name,
-                                            "UNIX_MATCH_UID");
-  sh->match_gid
-    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                            sh->service_name,
-                                            "UNIX_MATCH_GID");
-  process_acl4 (&sh->v4_denied,
-                sh,
-                "REJECT_FROM");
-  process_acl4 (&sh->v4_allowed,
-                sh,
-                "ACCEPT_FROM");
-  process_acl6 (&sh->v6_denied,
-                sh,
-                "REJECT_FROM6");
-  process_acl6 (&sh->v6_allowed,
-                sh,
-                "ACCEPT_FROM6");
-  return GNUNET_OK;
-}
-
-
-/**
- * Get the name of the user that'll be used
- * to provide the service.
- *
- * @param sh service context
- * @return value of the 'USERNAME' option
- */
-static char *
-get_user_name (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *un;
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
-                                               sh->service_name,
-                                               "USERNAME",
-                                               &un))
-    return NULL;
-  return un;
-}
-
-
-/**
- * Set user ID.
- *
- * @param sh service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-set_user_id (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *user;
-
-  if (NULL == (user = get_user_name (sh)))
-    return GNUNET_OK;           /* keep */
-#ifndef MINGW
-  struct passwd *pws;
-
-  errno = 0;
-  pws = getpwnam (user);
-  if (NULL == pws)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Cannot obtain information about user `%s': %s\n"),
-         user,
-         errno == 0 ? _("No such user") : STRERROR (errno));
-    GNUNET_free (user);
-    return GNUNET_SYSERR;
-  }
-  if ( (0 != setgid (pws->pw_gid)) ||
-       (0 != setegid (pws->pw_gid)) ||
-#if HAVE_INITGROUPS
-       (0 != initgroups (user,
-                         pws->pw_gid)) ||
-#endif
-       (0 != setuid (pws->pw_uid)) ||
-       (0 != seteuid (pws->pw_uid)))
-  {
-    if ((0 != setregid (pws->pw_gid,
-                        pws->pw_gid)) ||
-        (0 != setreuid (pws->pw_uid,
-                        pws->pw_uid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Cannot change user/group to `%s': %s\n"),
-           user,
-           STRERROR (errno));
-      GNUNET_free (user);
-      return GNUNET_SYSERR;
-    }
-  }
-#endif
-  GNUNET_free (user);
-  return GNUNET_OK;
-}
-
-
-/**
- * Get the name of the file where we will
- * write the PID of the service.
- *
- * @param sh service context
- * @return name of the file for the process ID
- */
-static char *
-get_pid_file_name (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *pif;
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
-                                               sh->service_name,
-                                               "PIDFILE",
-                                               &pif))
-    return NULL;
-  return pif;
-}
-
-
-/**
- * Delete the PID file that was created by our parent.
- *
- * @param sh service context
- */
-static void
-pid_file_delete (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *pif = get_pid_file_name (sh);
-
-  if (NULL == pif)
-    return;                     /* no PID file */
-  if (0 != UNLINK (pif))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
-                       "unlink",
-                       pif);
-  GNUNET_free (pif);
-}
-
-
-/**
- * Detach from terminal.
- *
- * @param sh service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-detach_terminal (struct GNUNET_SERVICE_Handle *sh)
-{
-#ifndef MINGW
-  pid_t pid;
-  int nullfd;
-  int filedes[2];
-
-  if (0 != PIPE (filedes))
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "pipe");
-    return GNUNET_SYSERR;
-  }
-  pid = fork ();
-  if (pid < 0)
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "fork");
-    return GNUNET_SYSERR;
-  }
-  if (0 != pid)
-  {
-    /* Parent */
-    char c;
-
-    GNUNET_break (0 == CLOSE (filedes[1]));
-    c = 'X';
-    if (1 != READ (filedes[0],
-                   &c,
-                   sizeof (char)))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
-                    "read");
-    fflush (stdout);
-    switch (c)
-    {
-    case '.':
-      exit (0);
-    case 'I':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process failed to initialize\n"));
-      break;
-    case 'S':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process could not initialize server function\n"));
-      break;
-    case 'X':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process failed to report status\n"));
-      break;
-    }
-    exit (1);                   /* child reported error */
-  }
-  GNUNET_break (0 == CLOSE (0));
-  GNUNET_break (0 == CLOSE (1));
-  GNUNET_break (0 == CLOSE (filedes[0]));
-  nullfd = OPEN ("/dev/null",
-                 O_RDWR | O_APPEND);
-  if (nullfd < 0)
-    return GNUNET_SYSERR;
-  /* set stdin/stdout to /dev/null */
-  if ( (dup2 (nullfd, 0) < 0) ||
-       (dup2 (nullfd, 1) < 0) )
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "dup2");
-    (void) CLOSE (nullfd);
-    return GNUNET_SYSERR;
-  }
-  (void) CLOSE (nullfd);
-  /* Detach from controlling terminal */
-  pid = setsid ();
-  if (-1 == pid)
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "setsid");
-  sh->ready_confirm_fd = filedes[1];
-#else
-  /* FIXME: we probably need to do something else
-   * elsewhere in order to fork the process itself... */
-  FreeConsole ();
-#endif
-  return GNUNET_OK;
-}
-
-
-/**
- * Tear down the service, closing the listen sockets and
- * freeing the ACLs.
- *
- * @param sh handle to the service to tear down.
- */
-static void
-teardown_service (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  GNUNET_free_non_null (sh->v4_denied);
-  GNUNET_free_non_null (sh->v6_denied);
-  GNUNET_free_non_null (sh->v4_allowed);
-  GNUNET_free_non_null (sh->v6_allowed);
-  while (NULL != (slc = sh->slc_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (sh->slc_head,
-                                 sh->slc_tail,
-                                 slc);
-    if (NULL != slc->listen_task)
-      GNUNET_SCHEDULER_cancel (slc->listen_task);
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (slc->listen_socket));
-    GNUNET_free (slc);
-  }
-}
-
-
-/**
- * Low-level function to start a service if the scheduler
- * is already running.  Should only be used directly in
- * special cases.
- *
- * The function will launch the service with the name @a service_name
- * using the @a service_options to configure its shutdown
- * behavior. When clients connect or disconnect, the respective
- * @a connect_cb or @a disconnect_cb functions will be called. For
- * messages received from the clients, the respective @a handlers will
- * be invoked; for the closure of the handlers we use the return value
- * from the @a connect_cb invocation of the respective client.
- *
- * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
- * message to receive further messages from this client.  If
- * #GNUNET_SERVICE_client_continue() is not called within a short
- * time, a warning will be logged. If delays are expected, services
- * should call #GNUNET_SERVICE_client_disable_continue_warning() to
- * disable the warning.
- *
- * Clients sending invalid messages (based on @a handlers) will be
- * dropped. Additionally, clients can be dropped at any time using
- * #GNUNET_SERVICE_client_drop().
- *
- * The service must be stopped using #GNUNET_SERVICE_stoP().
- *
- * @param service_name name of the service to run
- * @param cfg configuration to use
- * @param connect_cb function to call whenever a client connects
- * @param disconnect_cb function to call whenever a client disconnects
- * @param cls closure argument for @a connect_cb and @a disconnect_cb
- * @param handlers NULL-terminated array of message handlers for the service,
- *                 the closure will be set to the value returned by
- *                 the @a connect_cb for the respective connection
- * @return NULL on error
- */
-struct GNUNET_SERVICE_Handle *
-GNUNET_SERVICE_starT (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      GNUNET_SERVICE_ConnectHandler connect_cb,
-                      GNUNET_SERVICE_DisconnectHandler disconnect_cb,
-                      void *cls,
-                      const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_SERVICE_Handle *sh;
-
-  sh = GNUNET_new (struct GNUNET_SERVICE_Handle);
-  sh->service_name = service_name;
-  sh->cfg = cfg;
-  sh->connect_cb = connect_cb;
-  sh->disconnect_cb = disconnect_cb;
-  sh->cb_cls = cls;
-  if (NULL != handlers)
-  {
-    unsigned int i;
-
-    for (i=0;NULL != handlers[i].cb; i++) ;
-    sh->handlers = GNUNET_new_array (i + 1,
-                                     struct GNUNET_MQ_MessageHandler);
-    GNUNET_memcpy (sh->handlers,
-                   handlers,
-                   i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }
-  if (GNUNET_OK != setup_service (sh))
-  {
-    GNUNET_free (sh->handlers);
-    GNUNET_free (sh);
-    return NULL;
-  }
-  GNUNET_SERVICE_resume (sh);
-  return sh;
-}
-
-
-/**
- * Stops a service that was started with #GNUNET_SERVICE_starT().
- *
- * @param srv service to stop
- */
-void
-GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  GNUNET_SERVICE_suspend (srv);
-  while (NULL != (client = srv->clients_head))
-    GNUNET_SERVICE_client_drop (client);
-  teardown_service (srv);
-  GNUNET_free (srv->handlers);
-  GNUNET_free (srv);
-}
-
-
-/**
- * Creates the "main" function for a GNUnet service.  You
- * should almost always use the #GNUNET_SERVICE_MAIN macro
- * instead of calling this function directly (except
- * for ARM, which should call this function directly).
- *
- * The function will launch the service with the name @a service_name
- * using the @a service_options to configure its shutdown
- * behavior. Once the service is ready, the @a init_cb will be called
- * for service-specific initialization.  @a init_cb will be given the
- * service handler which can be used to control the service's
- * availability.  When clients connect or disconnect, the respective
- * @a connect_cb or @a disconnect_cb functions will be called. For
- * messages received from the clients, the respective @a handlers will
- * be invoked; for the closure of the handlers we use the return value
- * from the @a connect_cb invocation of the respective client.
- *
- * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
- * message to receive further messages from this client.  If
- * #GNUNET_SERVICE_client_continue() is not called within a short
- * time, a warning will be logged. If delays are expected, services
- * should call #GNUNET_SERVICE_client_disable_continue_warning() to
- * disable the warning.
- *
- * Clients sending invalid messages (based on @a handlers) will be
- * dropped. Additionally, clients can be dropped at any time using
- * #GNUNET_SERVICE_client_drop().
- *
- * @param argc number of command-line arguments in @a argv
- * @param argv array of command-line arguments
- * @param service_name name of the service to run
- * @param options options controlling shutdown of the service
- * @param service_init_cb function to call once the service is ready
- * @param connect_cb function to call whenever a client connects
- * @param disconnect_cb function to call whenever a client disconnects
- * @param cls closure argument for @a service_init_cb, @a connect_cb and @a disconnect_cb
- * @param handlers NULL-terminated array of message handlers for the service,
- *                 the closure will be set to the value returned by
- *                 the @a connect_cb for the respective connection
- * @return 0 on success, non-zero on error
- */
-int
-GNUNET_SERVICE_ruN_ (int argc,
-                     char *const *argv,
-                     const char *service_name,
-                     enum GNUNET_SERVICE_Options options,
-                     GNUNET_SERVICE_InitCallback service_init_cb,
-                     GNUNET_SERVICE_ConnectHandler connect_cb,
-                     GNUNET_SERVICE_DisconnectHandler disconnect_cb,
-                     void *cls,
-                     const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_SERVICE_Handle sh;
-  char *cfg_filename;
-  char *opt_cfg_filename;
-  char *loglev;
-  const char *xdg;
-  char *logfile;
-  int do_daemonize;
-  unsigned long long skew_offset;
-  unsigned long long skew_variance;
-  long long clock_offset;
-  struct GNUNET_CONFIGURATION_Handle *cfg;
-  int ret;
-  int err;
-
-  struct GNUNET_GETOPT_CommandLineOption service_options[] = {
-    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_filename),
-    {'d', "daemonize", NULL,
-     gettext_noop ("do daemonize (detach from terminal)"), 0,
-     GNUNET_GETOPT_set_one, &do_daemonize},
-    GNUNET_GETOPT_OPTION_HELP (NULL),
-    GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
-    GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
-    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
-    GNUNET_GETOPT_OPTION_END
-  };
-
-  memset (&sh,
-          0,
-          sizeof (sh));
-  xdg = getenv ("XDG_CONFIG_HOME");
-  if (NULL != xdg)
-    GNUNET_asprintf (&cfg_filename,
-                     "%s%s%s",
-                     xdg,
-                     DIR_SEPARATOR_STR,
-                     GNUNET_OS_project_data_get ()->config_file);
-  else
-    cfg_filename = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
-  sh.ready_confirm_fd = -1;
-  sh.options = options;
-  sh.cfg = cfg = GNUNET_CONFIGURATION_create ();
-  sh.service_init_cb = service_init_cb;
-  sh.connect_cb = connect_cb;
-  sh.disconnect_cb = disconnect_cb;
-  sh.cb_cls = cls;
-  if (NULL != handlers)
-  {
-    unsigned int i;
-
-    for (i=0;NULL != handlers[i].cb; i++) ;
-    sh.handlers = GNUNET_new_array (i + 1,
-                                     struct GNUNET_MQ_MessageHandler);
-    GNUNET_memcpy (sh.handlers,
-                   handlers,
-                   i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }
-  sh.service_name = service_name;
-
-  /* setup subsystems */
-  loglev = NULL;
-  logfile = NULL;
-  opt_cfg_filename = NULL;
-  do_daemonize = 0;
-  ret = GNUNET_GETOPT_run (service_name,
-                           service_options,
-                           argc,
-                           argv);
-  if (GNUNET_SYSERR == ret)
-    goto shutdown;
-  if (GNUNET_NO == ret)
-  {
-    err = 0;
-    goto shutdown;
-  }
-  if (GNUNET_OK != GNUNET_log_setup (service_name,
-                                     loglev,
-                                     logfile))
-  {
-    GNUNET_break (0);
-    goto shutdown;
-  }
-  if (NULL == opt_cfg_filename)
-    opt_cfg_filename = GNUNET_strdup (cfg_filename);
-  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_filename))
-  {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
-                                                    opt_cfg_filename))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Malformed configuration file `%s', exit ...\n"),
-                  opt_cfg_filename);
-      goto shutdown;
-    }
-  }
-  else
-  {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
-                                                    NULL))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Malformed configuration, exit ...\n"));
-      goto shutdown;
-    }
-    if (0 != strcmp (opt_cfg_filename,
-                     cfg_filename))
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Could not access configuration file `%s'\n"),
-                  opt_cfg_filename);
-  }
-  if (GNUNET_OK != setup_service (&sh))
-    goto shutdown;
-  if ( (1 == do_daemonize) &&
-       (GNUNET_OK != detach_terminal (&sh)) )
-  {
-    GNUNET_break (0);
-    goto shutdown;
-  }
-  if (GNUNET_OK != set_user_id (&sh))
-    goto shutdown;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Service `%s' runs with configuration from `%s'\n",
-       service_name,
-       opt_cfg_filename);
-  if ((GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
-                                              "TESTING",
-                                              "SKEW_OFFSET",
-                                              &skew_offset)) &&
-      (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
-                                              "TESTING",
-                                              "SKEW_VARIANCE",
-                                              &skew_variance)))
-  {
-    clock_offset = skew_offset - skew_variance;
-    GNUNET_TIME_set_offset (clock_offset);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Skewing clock by %dll ms\n",
-         clock_offset);
-  }
-  GNUNET_RESOLVER_connect (sh.cfg);
-
-  /* actually run service */
-  err = 0;
-  GNUNET_SCHEDULER_run (&service_main,
-                        &sh);
-  /* shutdown */
-  if (1 == do_daemonize)
-    pid_file_delete (&sh);
-
-shutdown:
-  if (-1 != sh.ready_confirm_fd)
-  {
-    if (1 != WRITE (sh.ready_confirm_fd,
-                    err ? "I" : "S",
-                    1))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
-                    "write");
-    GNUNET_break (0 == CLOSE (sh.ready_confirm_fd));
-  }
-#if HAVE_MALLINFO
-  {
-    char *counter;
-
-    if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sh.cfg,
-                                           service_name,
-                                           "GAUGER_HEAP")) &&
-         (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sh.cfg,
-                                                 service_name,
-                                                 "GAUGER_HEAP",
-                                                 &counter)) )
-    {
-      struct mallinfo mi;
-
-      mi = mallinfo ();
-      GAUGER (service_name,
-              counter,
-              mi.usmblks,
-              "blocks");
-      GNUNET_free (counter);
-    }
-  }
-#endif
-  teardown_service (&sh);
-  GNUNET_free (sh.handlers);
-  GNUNET_SPEEDUP_stop_ ();
-  GNUNET_CONFIGURATION_destroy (cfg);
-  GNUNET_free_non_null (logfile);
-  GNUNET_free_non_null (loglev);
-  GNUNET_free (cfg_filename);
-  GNUNET_free_non_null (opt_cfg_filename);
-
-  return err ? GNUNET_SYSERR : sh.ret;
-}
-
-
-/**
- * Suspend accepting connections from the listen socket temporarily.
- * Resume activity using #GNUNET_SERVICE_resume.
- *
- * @param sh service to stop accepting connections.
- */
-void
-GNUNET_SERVICE_suspend (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
-  {
-    if (NULL != slc->listen_task)
-    {
-      GNUNET_SCHEDULER_cancel (slc->listen_task);
-      slc->listen_task = NULL;
-    }
-  }
-}
-
-
-/**
- * Task run when we are ready to transmit data to the
- * client.
- *
- * @param cls the `struct GNUNET_SERVICE_Client *` to send to
- */
-static void
-do_send (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  ssize_t ret;
-  size_t left;
-  const char *buf;
-
-  client->send_task = NULL;
-  buf = (const char *) client->msg;
-  left = ntohs (client->msg->size) - client->msg_pos;
-  ret = GNUNET_NETWORK_socket_send (client->sock,
-                                    &buf[client->msg_pos],
-                                    left);
-  GNUNET_assert (ret <= (ssize_t) left);
-  if (0 == ret)
-  {
-    GNUNET_MQ_inject_error (client->mq,
-                            GNUNET_MQ_ERROR_WRITE);
-    return;
-  }
-  if (-1 == ret)
-  {
-    if ( (EAGAIN == errno) ||
-         (EINTR == errno) )
-    {
-      /* ignore */
-      ret = 0;
-    }
-    else
-    {
-      if (EPIPE != errno)
-        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
-                             "send");
-      GNUNET_MQ_inject_error (client->mq,
-                              GNUNET_MQ_ERROR_WRITE);
-      return;
-    }
-  }
-  if (0 == client->msg_pos)
-  {
-    GNUNET_MQ_impl_send_in_flight (client->mq);
-  }
-  client->msg_pos += ret;
-  if (left > ret)
-  {
-    client->send_task
-      = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                        client->sock,
-                                        &do_send,
-                                        client);
-    return;
-  }
-  GNUNET_MQ_impl_send_continue (client->mq);
-}
-
-
-/**
- * Signature of functions implementing the sending functionality of a
- * message queue.
- *
- * @param mq the message queue
- * @param msg the message to send
- * @param impl_state our `struct GNUNET_SERVICE_Client *`
- */
-static void
-service_mq_send (struct GNUNET_MQ_Handle *mq,
-                 const struct GNUNET_MessageHeader *msg,
-                 void *impl_state)
-{
-  struct GNUNET_SERVICE_Client *client = impl_state;
-
-  GNUNET_assert (NULL == client->send_task);
-  client->msg = msg;
-  client->msg_pos = 0;
-  client->send_task
-    = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                      client->sock,
-                                      &do_send,
-                                      client);
-}
-
-
-/**
- * Implementation function that cancels the currently sent message.
- *
- * @param mq message queue
- * @param impl_state state specific to the implementation
- */
-static void
-service_mq_cancel (struct GNUNET_MQ_Handle *mq,
-                   void *impl_state)
-{
-  struct GNUNET_SERVICE_Client *client = impl_state;
-
-  GNUNET_assert (0 == client->msg_pos);
-  client->msg = NULL;
-  GNUNET_SCHEDULER_cancel (client->send_task);
-  client->send_task = NULL;
-}
-
-
-/**
- * Generic error handler, called with the appropriate
- * error code and the same closure specified at the creation of
- * the message queue.
- * Not every message queue implementation supports an error handler.
- *
- * @param cls closure with our `struct GNUNET_SERVICE_Client`
- * @param error error code
- */
-static void
-service_mq_error_handler (void *cls,
-                          enum GNUNET_MQ_Error error)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  struct GNUNET_SERVICE_Handle *sh = client->sh;
-
-  if ( (GNUNET_MQ_ERROR_NO_MATCH == error) &&
-       (GNUNET_NO == sh->require_found) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "No handler for message of type %u found\n",
-                (unsigned int) client->warn_type);
-    GNUNET_SERVICE_client_continue (client);
-    return; /* ignore error */
-  }
-  GNUNET_SERVICE_client_drop (client);
-}
-
-
-/**
- * Task run to warn about missing calls to #GNUNET_SERVICE_client_continue().
- *
- * @param cls our `struct GNUNET_SERVICE_Client *` to process more requests from
- */
-static void
-warn_no_client_continue (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  GNUNET_break (0 != client->warn_type); /* type should never be 0 here, as we don't use 0 */
-  client->warn_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_no_client_continue,
-                                    client);
-  LOG (GNUNET_ERROR_TYPE_WARNING,
-       _("Processing code for message of type %u did not call `GNUNET_SERVICE_client_continue' after %s\n"),
-       (unsigned int) client->warn_type,
-       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (client->warn_start),
-                                               GNUNET_YES));
-}
-
-
-/**
- * Functions with this signature are called whenever a
- * complete message is received by the tokenizer for a client.
- *
- * Do not call #GNUNET_MST_destroy() from within
- * the scope of this callback.
- *
- * @param cls closure with the `struct GNUNET_SERVICE_Client *`
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR if the client was dropped
- */
-static int
-service_client_mst_cb (void *cls,
-                       const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  GNUNET_assert (GNUNET_NO == client->needs_continue);
-  client->needs_continue = GNUNET_YES;
-  client->warn_type = ntohs (message->type);
-  client->warn_start = GNUNET_TIME_absolute_get ();
-  GNUNET_assert (NULL == client->warn_task);
-  client->warn_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_no_client_continue,
-                                    client);
-  GNUNET_MQ_inject_message (client->mq,
-                            message);
-  if (NULL != client->drop_task)
-    return GNUNET_SYSERR;
-  return GNUNET_OK;
-}
-
-
-/**
- * A client sent us data. Receive and process it.  If we are done,
- * reschedule this task.
- *
- * @param cls the `struct GNUNET_SERVICE_Client` that sent us data.
- */
-static void
-service_client_recv (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  int ret;
-
-  client->recv_task = NULL;
-  ret = GNUNET_MST_read (client->mst,
-                         client->sock,
-                         GNUNET_NO,
-                         GNUNET_YES);
-  if (GNUNET_SYSERR == ret)
-  {
-    /* client closed connection (or IO error) */
-    if (NULL == client->drop_task)
-    {
-      GNUNET_assert (GNUNET_NO == client->needs_continue);
-      GNUNET_SERVICE_client_drop (client);
-    }
-    return;
-  }
-  if (GNUNET_NO == ret)
-    return; /* more messages in buffer, wait for application
-               to be done processing */
-  GNUNET_assert (GNUNET_OK == ret);
-  if (GNUNET_YES == client->needs_continue)
-    return;
-  if (NULL != client->recv_task)
-    return;
-  /* MST needs more data, re-schedule read job */
-  client->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     client->sock,
-                                     &service_client_recv,
-                                     client);
-}
-
-
-/**
- * We have successfully accepted a connection from a client.  Now
- * setup the client (with the scheduler) and tell the application.
- *
- * @param sh service that accepted the client
- * @param sock socket associated with the client
- */
-static void
-start_client (struct GNUNET_SERVICE_Handle *sh,
-              struct GNUNET_NETWORK_Handle *csock)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  client = GNUNET_new (struct GNUNET_SERVICE_Client);
-  GNUNET_CONTAINER_DLL_insert (sh->clients_head,
-                               sh->clients_tail,
-                               client);
-  client->sh = sh;
-  client->sock = csock;
-  client->mq = GNUNET_MQ_queue_for_callbacks (&service_mq_send,
-                                              NULL,
-                                              &service_mq_cancel,
-                                              client,
-                                              sh->handlers,
-                                              &service_mq_error_handler,
-                                              client);
-  client->mst = GNUNET_MST_create (&service_client_mst_cb,
-                                   client);
-  client->user_context = sh->connect_cb (sh->cb_cls,
-                                         client,
-                                         client->mq);
-  GNUNET_MQ_set_handlers_closure (client->mq,
-                                  client->user_context);
-  client->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     client->sock,
-                                     &service_client_recv,
-                                     client);
-}
-
-
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param add the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
-                   const struct in_addr *add)
-{
-  unsigned int i;
-
-  if (NULL == list)
-    return GNUNET_NO;
-  i = 0;
-  while ( (0 != list[i].network.s_addr) ||
-          (0 != list[i].netmask.s_addr) )
-  {
-    if ((add->s_addr & list[i].netmask.s_addr) ==
-        (list[i].network.s_addr & list[i].netmask.s_addr))
-      return GNUNET_YES;
-    i++;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param ip the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
-                   const struct in6_addr *ip)
-{
-  unsigned int i;
-  unsigned int j;
-  struct in6_addr zero;
-
-  if (NULL == list)
-    return GNUNET_NO;
-  memset (&zero,
-          0,
-          sizeof (struct in6_addr));
-  i = 0;
-NEXT:
-  while (0 != memcmp (&zero,
-                      &list[i].network,
-                      sizeof (struct in6_addr)))
-  {
-    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
-      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
-          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
-      {
-        i++;
-        goto NEXT;
-      }
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * We have a client. Accept the incoming socket(s) (and reschedule
- * the listen task).
- *
- * @param cls the `struct ServiceListenContext` of the ready listen socket
- */
-static void
-accept_client (void *cls)
-{
-  struct ServiceListenContext *slc = cls;
-  struct GNUNET_SERVICE_Handle *sh = slc->sh;
-
-  slc->listen_task = NULL;
-  while (1)
-  {
-    struct GNUNET_NETWORK_Handle *sock;
-    const struct sockaddr_in *v4;
-    const struct sockaddr_in6 *v6;
-    struct sockaddr_storage sa;
-    socklen_t addrlen;
-    int ok;
-
-    addrlen = sizeof (sa);
-    sock = GNUNET_NETWORK_socket_accept (slc->listen_socket,
-                                         (struct sockaddr *) &sa,
-                                         &addrlen);
-    if (NULL == sock)
-      break;
-    switch (sa.ss_family)
-    {
-    case AF_INET:
-      GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-      v4 = (const struct sockaddr_in *) &sa;
-      ok = ( ( (NULL == sh->v4_allowed) ||
-               (check_ipv4_listed (sh->v4_allowed,
-                                   &v4->sin_addr))) &&
-             ( (NULL == sh->v4_denied) ||
-               (! check_ipv4_listed (sh->v4_denied,
-                                     &v4->sin_addr)) ) );
-      break;
-    case AF_INET6:
-      GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
-      v6 = (const struct sockaddr_in6 *) &sa;
-      ok = ( ( (NULL == sh->v6_allowed) ||
-               (check_ipv6_listed (sh->v6_allowed,
-                                   &v6->sin6_addr))) &&
-             ( (NULL == sh->v6_denied) ||
-               (! check_ipv6_listed (sh->v6_denied,
-                                     &v6->sin6_addr)) ) );
-      break;
-#ifndef WINDOWS
-    case AF_UNIX:
-      ok = GNUNET_OK;            /* controlled using file-system ACL now */
-      break;
-#endif
-    default:
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("Unknown address family %d\n"),
-           sa.ss_family);
-      return;
-    }
-    if (! ok)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Service rejected incoming connection from %s due to policy.\n",
-           GNUNET_a2s ((const struct sockaddr *) &sa,
-                       addrlen));
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (sock));
-      continue;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Service accepted incoming connection from %s.\n",
-         GNUNET_a2s ((const struct sockaddr *) &sa,
-                     addrlen));
-    start_client (slc->sh,
-                  sock);
-  }
-  slc->listen_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     slc->listen_socket,
-                                     &accept_client,
-                                     slc);
-}
-
-
-/**
- * Resume accepting connections from the listen socket.
- *
- * @param sh service to resume accepting connections.
- */
-void
-GNUNET_SERVICE_resume (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
-  {
-    GNUNET_assert (NULL == slc->listen_task);
-    slc->listen_task
-      = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                       slc->listen_socket,
-                                       &accept_client,
-                                       slc);
-  }
-}
-
-
-/**
- * Task run to resume receiving data from the client after
- * the client called #GNUNET_SERVICE_client_continue().
- *
- * @param cls our `struct GNUNET_SERVICE_Client`
- */
-static void
-resume_client_receive (void *cls)
-{
-  struct GNUNET_SERVICE_Client *c = cls;
-  int ret;
-
-  c->recv_task = NULL;
-  /* first, check if there is still something in the buffer */
-  ret = GNUNET_MST_next (c->mst,
-                         GNUNET_YES);
-  if (GNUNET_SYSERR == ret)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c);
-    return;
-  }
-  if (GNUNET_NO == ret)
-    return; /* done processing, wait for more later */
-  GNUNET_assert (GNUNET_OK == ret);
-  if (GNUNET_YES == c->needs_continue)
-    return; /* #GNUNET_MST_next() did give a message to the client */
-  /* need to receive more data from the network first */
-  if (NULL != c->recv_task)
-    return;
-  c->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     c->sock,
-                                     &service_client_recv,
-                                     c);
-}
-
-
-/**
- * Continue receiving further messages from the given client.
- * Must be called after each message received.
- *
- * @param c the client to continue receiving from
- */
-void
-GNUNET_SERVICE_client_continue (struct GNUNET_SERVICE_Client *c)
-{
-  GNUNET_assert (GNUNET_YES == c->needs_continue);
-  GNUNET_assert (NULL == c->recv_task);
-  c->needs_continue = GNUNET_NO;
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-  c->recv_task
-    = GNUNET_SCHEDULER_add_now (&resume_client_receive,
-                                c);
-}
-
-
-/**
- * Disable the warning the server issues if a message is not
- * acknowledged in a timely fashion.  Use this call if a client is
- * intentionally delayed for a while.  Only applies to the current
- * message.
- *
- * @param c client for which to disable the warning
- */
-void
-GNUNET_SERVICE_client_disable_continue_warning (struct GNUNET_SERVICE_Client *c)
-{
-  GNUNET_break (NULL != c->warn_task);
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-}
-
-
-/**
- * Asynchronously finish dropping the client.
- *
- * @param cls the `struct GNUNET_SERVICE_Client`.
- */
-static void
-finish_client_drop (void *cls)
-{
-  struct GNUNET_SERVICE_Client *c = cls;
-  struct GNUNET_SERVICE_Handle *sh = c->sh;
-
-  GNUNET_MST_destroy (c->mst);
-  GNUNET_MQ_destroy (c->mq);
-  if (GNUNET_NO == c->persist)
-  {
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (c->sock));
-  }
-  else
-  {
-    GNUNET_NETWORK_socket_free_memory_only_ (c->sock);
-  }
-  GNUNET_free (c);
-  if ( (GNUNET_YES == sh->got_shutdown) &&
-       (GNUNET_NO == have_non_monitor_clients (sh)) )
-    GNUNET_SERVICE_shutdown (sh);
-}
-
-
-/**
- * Ask the server to disconnect from the given client.  This is the
- * same as returning #GNUNET_SYSERR within the check procedure when
- * handling a message, wexcept that it allows dropping of a client even
- * when not handling a message from that client.  The `disconnect_cb`
- * will be called on @a c even if the application closes the connection
- * using this function.
- *
- * @param c client to disconnect now
- */
-void
-GNUNET_SERVICE_client_drop (struct GNUNET_SERVICE_Client *c)
-{
-  struct GNUNET_SERVICE_Handle *sh = c->sh;
-
-  if (NULL != c->drop_task)
-  {
-    /* asked to drop twice! */
-    GNUNET_break (0);
-    return;
-  }
-  GNUNET_CONTAINER_DLL_remove (sh->clients_head,
-                               sh->clients_tail,
-                               c);
-  sh->disconnect_cb (sh->cb_cls,
-                     c,
-                     c->user_context);
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-  if (NULL != c->recv_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->recv_task);
-    c->recv_task = NULL;
-  }
-  if (NULL != c->send_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->send_task);
-    c->send_task = NULL;
-  }
-  c->drop_task = GNUNET_SCHEDULER_add_now (&finish_client_drop,
-                                           c);
-}
-
-
-/**
- * Explicitly stops the service.
- *
- * @param sh server to shutdown
- */
-void
-GNUNET_SERVICE_shutdown (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  GNUNET_SERVICE_suspend (sh);
-  sh->got_shutdown = GNUNET_NO;
-  while (NULL != (client = sh->clients_head))
-    GNUNET_SERVICE_client_drop (client);
-}
-
-
-/**
- * Set the 'monitor' flag on this client.  Clients which have been
- * marked as 'monitors' won't prevent the server from shutting down
- * once #GNUNET_SERVICE_stop_listening() has been invoked.  The idea is
- * that for "normal" clients we likely want to allow them to process
- * their requests; however, monitor-clients are likely to 'never'
- * disconnect during shutdown and thus will not be considered when
- * determining if the server should continue to exist after
- * shutdown has been triggered.
- *
- * @param c client to mark as a monitor
- */
-void
-GNUNET_SERVICE_client_mark_monitor (struct GNUNET_SERVICE_Client *c)
-{
-  c->is_monitor = GNUNET_YES;
-  if ( (GNUNET_YES == c->sh->got_shutdown) &&
-       (GNUNET_NO == have_non_monitor_clients (c->sh)) )
-    GNUNET_SERVICE_shutdown (c->sh);
-}
-
-
-/**
- * Set the persist option on this client.  Indicates that the
- * underlying socket or fd should never really be closed.  Used for
- * indicating process death.
- *
- * @param c client to persist the socket (never to be closed)
- */
-void
-GNUNET_SERVICE_client_persist (struct GNUNET_SERVICE_Client *c)
-{
-  c->persist = GNUNET_YES;
-}
-
-
-/**
- * Obtain the message queue of @a c.  Convenience function.
- *
- * @param c the client to continue receiving from
- * @return the message queue of @a c
- */
-struct GNUNET_MQ_Handle *
-GNUNET_SERVICE_client_get_mq (struct GNUNET_SERVICE_Client *c)
-{
-  return c->mq;
-}
-
-
-/* end of service_new.c */
diff --git a/src/util/signal.c b/src/util/signal.c
index 543fcd899..cb917e36a 100644
--- a/src/util/signal.c
+++ b/src/util/signal.c
@@ -27,7 +27,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-signal", __VA_ARGS__)
 
 
 struct GNUNET_SIGNAL_Context
diff --git a/src/util/socks.c b/src/util/socks.c
index fee79cc80..85548fd79 100644
--- a/src/util/socks.c
+++ b/src/util/socks.c
@@ -29,9 +29,9 @@
 #include "gnunet_util_lib.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "socks", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-socks", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "socks", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-socks", syscall)
 
 
 /* SOCKS5 authentication methods */
@@ -372,15 +372,18 @@ transmit_ready (void *cls,
     return 0;
   }
 
-  GNUNET_assert (1024 >= size && size > 0);
-  GNUNET_assert (SOCKS5_step_done > ih->step && ih->step >= 0);
+  GNUNET_assert ( (1024 >= size) && (size > 0) );
+  GNUNET_assert ( (SOCKS5_step_done > ih->step) && (ih->step >= 0) );
   unsigned char * b = ih->outstep[ih->step];
   unsigned char * e = ih->outstep[ih->step+1];
   GNUNET_assert (e <= &ih->outbuf[1024]);
-  unsigned l = e - b;
-  GNUNET_assert (size >= l && l >= 0);
-  GNUNET_memcpy(buf, b, l);
-  register_reciever (ih, register_reciever_wants(ih));
+  unsigned int l = e - b;
+  GNUNET_assert (size >= l);
+  GNUNET_memcpy (buf,
+                 b,
+                 l);
+  register_reciever (ih,
+                     register_reciever_wants (ih));
   return l;
 }
 
@@ -566,17 +569,25 @@ GNUNET_SOCKS_check_service (const char *service_name,
  */
 struct GNUNET_CONNECTION_Handle *
 GNUNET_SOCKS_do_connect (const char *service_name,
-                          const struct GNUNET_CONFIGURATION_Handle *cfg)
+                         const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_SOCKS_Handshake *ih;
   struct GNUNET_CONNECTION_Handle *socks5; /* *proxied */
-  char *host0,*host1,*user,*pass;
-  unsigned long long port0,port1;
-
-  if (GNUNET_YES != GNUNET_SOCKS_check_service (service_name, cfg))
+  char *host0;
+  char *host1;
+  char *user;
+  char *pass;
+  unsigned long long port0;
+  unsigned long long port1;
+
+  if (GNUNET_YES !=
+      GNUNET_SOCKS_check_service (service_name, cfg))
     return NULL;
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (cfg, service_name, "SOCKSPORT", &port0))
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             service_name,
+                                             "SOCKSPORT",
+                                             &port0))
     port0 = 9050;
   /* A typical Tor client should usually try port 9150 for the TBB too, but
    * GNUnet can probably assume a system Tor installation. */
@@ -588,16 +599,23 @@ GNUNET_SOCKS_do_connect (const char *service_name,
          service_name);
     return NULL;
   }
-  if ((GNUNET_OK !=
-       GNUNET_CONFIGURATION_get_value_number (cfg, service_name, "PORT", &port1))
-      || (port1 > 65535) || (port1 <= 0) ||
+  if ( (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_number (cfg,
+                                               service_name,
+                                               "PORT",
+                                               &port1)) ||
+       (port1 > 65535) ||
+       (port1 <= 0) ||
        (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "HOSTNAME", &host1)))
+        GNUNET_CONFIGURATION_get_value_string (cfg,
+                                               service_name,
+                                               "HOSTNAME",
+                                               &host1)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         _
-         ("Attempting to proxy service `%s' to invalid port %d or hostname `%s'.\n"),
-         service_name,port1,host1);
+         _("Attempting to proxy service `%s' to invalid port %d or hostname.\n"),
+         service_name,
+         port1);
     return NULL;
   }
   /* Appeared to still work after host0 corrupted, so either test case is broken, or
@@ -605,20 +623,32 @@ GNUNET_SOCKS_do_connect (const char *service_name,
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSHOST", &host0))
     host0 = NULL;
-  socks5 = GNUNET_CONNECTION_create_from_connect (cfg, (host0 != NULL)? host0:"127.0.0.1", port0);
+  socks5 = GNUNET_CONNECTION_create_from_connect (cfg,
+                                                  (host0 != NULL)
+                                                  ? host0
+                                                  :"127.0.0.1",
+                                                  port0);
   GNUNET_free_non_null (host0);
 
   /* Sets to NULL if they do not exist */
-  (void) GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSUSER", &user);
-  (void) GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSPASS", &pass);
+  (void) GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                service_name,
+                                                "SOCKSUSER",
+                                                &user);
+  (void) GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                service_name,
+                                                "SOCKSPASS",
+                                                &pass);
   ih = GNUNET_SOCKS_init_handshake(user,pass);
-  if (NULL != user) GNUNET_free (user);
-  if (NULL != pass) GNUNET_free (pass);
+  GNUNET_free_non_null (user);
+  GNUNET_free_non_null (pass);
 
-  GNUNET_SOCKS_set_handshake_destination (ih,host1,port1);
+  GNUNET_SOCKS_set_handshake_destination (ih,
+                                          host1,
+                                          port1);
   GNUNET_free (host1);
-
-  return GNUNET_SOCKS_run_handshake(ih,socks5);
+  return GNUNET_SOCKS_run_handshake (ih,
+                                     socks5);
 }
 
 /* socks.c */
diff --git a/src/util/speedup.c b/src/util/speedup.c
index 97df65c8e..c6a4cf678 100644
--- a/src/util/speedup.c
+++ b/src/util/speedup.c
@@ -27,7 +27,7 @@
 #include "gnunet_util_lib.h"
 #include "speedup.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-speedup", __VA_ARGS__)
 
 
 static struct GNUNET_TIME_Relative interval;
diff --git a/src/util/strings.c b/src/util/strings.c
index 2b51d3e52..d3268f4d9 100644
--- a/src/util/strings.c
+++ b/src/util/strings.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2005-2013 GNUnet e.V.
+     Copyright (C) 2005-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,7 +17,6 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
 /**
  * @file util/strings.c
  * @brief string functions
@@ -35,9 +34,9 @@
 #include <unistr.h>
 #include <uniconv.h>
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-strings", __VA_ARGS__)
 
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util", syscall)
+#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-strings", syscall)
 
 
 /**
@@ -90,6 +89,37 @@ GNUNET_STRINGS_buffer_fill (char *buffer, size_t size, unsigned int count, ...)
 
 
 /**
+ * Convert a peer path to a human-readable string.
+ *
+ * @param pids array of PIDs to convert to a string
+ * @param num_pids length of the @a pids array
+ * @return string representing the array of @a pids
+ */
+char *
+GNUNET_STRINGS_pp2s (const struct GNUNET_PeerIdentity *pids,
+                     unsigned int num_pids)
+{
+  char *buf;
+  size_t off;
+  size_t plen = num_pids * 5 + 1;
+
+  off = 0;
+  buf = GNUNET_malloc (plen);
+  for (unsigned int i = 0;
+       i < num_pids;
+       i++)
+  {
+    off += GNUNET_snprintf (&buf[off],
+                            plen - off,
+                            "%s%s",
+                            GNUNET_i2s (&pids[i]),
+                            (i == num_pids -1) ? "" : "-");
+  }
+  return buf;
+}
+
+
+/**
  * Given a buffer of a given size, find "count"
  * 0-terminated strings in the buffer and assign
  * the count (varargs) of type "const char**" to the
diff --git a/src/util/test_client.c b/src/util/test_client.c
index f60e5b7f7..527b400b0 100644
--- a/src/util/test_client.c
+++ b/src/util/test_client.c
@@ -179,7 +179,7 @@ main (int argc,
     test_argv[2] = "test_client_unix.conf";
   global_ret = 1;
   if (0 !=
-      GNUNET_SERVICE_ruN_ (3,
+      GNUNET_SERVICE_run_ (3,
                            test_argv,
                            "test_client",
                            GNUNET_SERVICE_OPTION_NONE,
diff --git a/src/util/test_common_allocation.c b/src/util/test_common_allocation.c
index 4ef98b629..4d1b6fe7d 100644
--- a/src/util/test_common_allocation.c
+++ b/src/util/test_common_allocation.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001, 2002, 2003, 2005, 2006 GNUnet e.V.
+     Copyright (C) 2001, 2002, 2003, 2005, 2006, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -25,6 +25,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
+
 static int
 check ()
 {
@@ -95,47 +96,57 @@ check ()
   if (ptrs[0] != NULL)
     return 9;
 
-        /* GNUNET_new_array_2d tests */
-        a2 = GNUNET_new_array_2d (17, 22, unsigned int);
-        for (i = 0; i < 17; i++)
-        {
-                for (j = 0; j < 22; j++)
-                {
-                        if (0 != a2[i][j])
-                                return 10;
-                        a2[i][j] = i * 100 + j;
-                }
-        }
-        free (a2);
-
-        /* GNUNET_new_array_3d tests */
-        a3 = GNUNET_new_array_3d (2, 3, 4, char);
-        for (i = 0; i < 2; i++)
-        {
-                for (j = 0; j < 3; j++)
-                {
-                        for (k = 0; k < 4; k++)
-                        {
-                                if (0 != a3[i][j][k])
-                                        return 11;
-                                a3[i][j][k] = i * 100 + j * 10 + k;
-                        }
-                }
-        }
-        free (a3);
+  /* GNUNET_new_array_2d tests */
+  a2 = GNUNET_new_array_2d (17, 22, unsigned int);
+  for (i = 0; i < 17; i++)
+  {
+    for (j = 0; j < 22; j++)
+    {
+      if (0 != a2[i][j])
+      {
+        GNUNET_free (a2);
+        return 10;
+      }
+      a2[i][j] = i * 100 + j;
+    }
+  }
+  GNUNET_free (a2);
 
+  /* GNUNET_new_array_3d tests */
+  a3 = GNUNET_new_array_3d (2, 3, 4, char);
+  for (i = 0; i < 2; i++)
+  {
+    for (j = 0; j < 3; j++)
+    {
+      for (k = 0; k < 4; k++)
+      {
+        if (0 != a3[i][j][k])
+        {
+          GNUNET_free (a3);
+          return 11;
+        }
+        a3[i][j][k] = i * 100 + j * 10 + k;
+      }
+    }
+  }
+  GNUNET_free (a3);
   return 0;
 }
 
+
 int
 main (int argc, char *argv[])
 {
   int ret;
 
-  GNUNET_log_setup ("test-common-allocation", "WARNING", NULL);
+  GNUNET_log_setup ("test-common-allocation",
+                    "WARNING",
+                    NULL);
   ret = check ();
   if (ret != 0)
-    FPRINTF (stderr, "ERROR %d.\n", ret);
+    FPRINTF (stderr,
+             "ERROR %d.\n",
+             ret);
   return ret;
 }
 
diff --git a/src/util/test_connection.c b/src/util/test_connection.c
deleted file mode 100644
index eaca75c2e..000000000
--- a/src/util/test_connection.c
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static size_t sofar;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return -1 on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_port = htons (PORT);
-  sa.sin_family = AF_INET;
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-static void
-receive_check (void *cls, const void *buf, size_t available,
-               const struct sockaddr *addr, socklen_t addrlen, int errCode)
-{
-  int *ok = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive validates incoming data\n");
-  GNUNET_assert (buf != NULL);  /* no timeout */
-  if (0 == memcmp (&"Hello World"[sofar], buf, available))
-    sofar += available;
-  if (sofar < 12)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive needs more data\n");
-    GNUNET_CONNECTION_receive (asock, 1024,
-                               GNUNET_TIME_relative_multiply
-                               (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                               cls);
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive closes accepted socket\n");
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (asock);
-    GNUNET_CONNECTION_destroy (csock);
-  }
-}
-
-
-static void
-run_accept (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test accepts connection\n");
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test destroys listen socket\n");
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Test asks to receive on accepted socket\n");
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                             cls);
-}
-
-
-static size_t
-make_hello (void *cls, size_t size, void *buf)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Test prepares to transmit on connect socket\n");
-  GNUNET_assert (size >= 12);
-  strcpy ((char *) buf, "Hello World");
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test destroys client socket\n");
-  return 12;
-}
-
-
-static void
-task (void *cls)
-{
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test asks for write notification\n");
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &make_hello, NULL));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test prepares to accept\n");
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL, ls, &run_accept,
-                                 cls);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection",
-                    "WARNING",
-                    NULL);
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection.c */
diff --git a/src/util/test_connection_addressing.c b/src/util/test_connection_addressing.c
deleted file mode 100644
index a6345b10a..000000000
--- a/src/util/test_connection_addressing.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_addressing.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static size_t sofar;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != 0);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  if (GNUNET_OK !=
-      GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                  sizeof (sa)))
-  {
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                         "bind");
-    GNUNET_assert (0);
-  }
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static void
-receive_check (void *cls, const void *buf, size_t available,
-               const struct sockaddr *addr, socklen_t addrlen, int errCode)
-{
-  int *ok = cls;
-
-  GNUNET_assert (buf != NULL);  /* no timeout */
-  if (0 == memcmp (&"Hello World"[sofar], buf, available))
-    sofar += available;
-  if (sofar < 12)
-  {
-    GNUNET_CONNECTION_receive (asock, 1024,
-                               GNUNET_TIME_relative_multiply
-                               (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                               cls);
-  }
-  else
-  {
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (csock);
-    GNUNET_CONNECTION_destroy (asock);
-  }
-}
-
-
-static void
-run_accept (void *cls)
-{
-  void *addr;
-  size_t alen;
-  struct sockaddr_in *v4;
-  struct sockaddr_in expect;
-
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONNECTION_get_address (asock, &addr, &alen));
-  GNUNET_assert (alen == sizeof (struct sockaddr_in));
-  v4 = addr;
-  memset (&expect, 0, sizeof (expect));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  expect.sin_len = sizeof (expect);
-#endif
-  expect.sin_family = AF_INET;
-  expect.sin_port = v4->sin_port;
-  expect.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  GNUNET_assert (0 == memcmp (&expect, v4, alen));
-  GNUNET_free (addr);
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                             cls);
-}
-
-static size_t
-make_hello (void *cls, size_t size, void *buf)
-{
-  GNUNET_assert (size >= 12);
-  strcpy ((char *) buf, "Hello World");
-  return 12;
-}
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in v4;
-
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  v4.sin_len = sizeof (v4);
-#endif
-  v4.sin_family = AF_INET;
-  v4.sin_port = htons (PORT);
-  v4.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  csock =
-      GNUNET_CONNECTION_create_from_sockaddr (AF_INET,
-                                              (const struct sockaddr *) &v4,
-                                              sizeof (v4));
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &make_hello, NULL));
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL, ls, &run_accept,
-                                 cls);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_addressing",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-/* end of test_connection_addressing.c */
diff --git a/src/util/test_connection_receive_cancel.c b/src/util/test_connection_receive_cancel.c
deleted file mode 100644
index 9c0ab699e..000000000
--- a/src/util/test_connection_receive_cancel.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_receive_cancel.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static void
-dead_receive (void *cls,
-              const void *buf,
-              size_t available,
-              const struct sockaddr *addr,
-              socklen_t addrlen,
-              int errCode)
-{
-  GNUNET_assert (0);
-}
-
-
-static void
-run_accept_cancel (void *cls)
-{
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5),
-                             &dead_receive, cls);
-}
-
-
-static void
-receive_cancel_task (void *cls)
-{
-  int *ok = cls;
-
-  GNUNET_CONNECTION_receive_cancel (asock);
-  GNUNET_CONNECTION_destroy (csock);
-  GNUNET_CONNECTION_destroy (asock);
-  *ok = 0;
-}
-
-
-static void
-task_receive_cancel (void *cls)
-{
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                 ls,
-                                 &run_accept_cancel,
-                                 cls);
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                &receive_cancel_task,
-                                cls);
-}
-
-
-/**
- * Main method, starts scheduler with task_timeout.
- */
-static int
-check_receive_cancel ()
-{
-  int ok;
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "resolver",
-                                         "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_receive_cancel, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_connection_receive_cancel", "WARNING", NULL);
-  ret += check_receive_cancel ();
-
-  return ret;
-}
-
-/* end of test_connection_receive_cancel.c */
diff --git a/src/util/test_connection_timeout.c b/src/util/test_connection_timeout.c
deleted file mode 100644
index e78cec669..000000000
--- a/src/util/test_connection_timeout.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_timeout.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static size_t
-send_kilo (void *cls, size_t size, void *buf)
-{
-  int *ok = cls;
-
-  if (size == 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got the desired timeout!\n");
-    GNUNET_assert (buf == NULL);
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (lsock);
-    GNUNET_CONNECTION_destroy (csock);
-    return 0;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending kilo to fill buffer.\n");
-  GNUNET_assert (size >= 1024);
-  memset (buf, 42, 1024);
-
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &send_kilo, cls));
-  return 1024;
-}
-
-
-static void
-task_timeout (void *cls)
-{
-
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &send_kilo, cls));
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_timeout",
-                    "WARNING",
-                    NULL);
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_timeout, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_timeout.c */
diff --git a/src/util/test_connection_timeout_no_connect.c b/src/util/test_connection_timeout_no_connect.c
deleted file mode 100644
index ebcd4b71e..000000000
--- a/src/util/test_connection_timeout_no_connect.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_timeout_no_connect.c
- * @brief tests for connection.c, doing timeout which connect failure
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 13425
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static size_t
-handle_timeout (void *cls, size_t size, void *buf)
-{
-  int *ok = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received timeout signal.\n");
-  GNUNET_assert (size == 0);
-  GNUNET_assert (buf == NULL);
-  *ok = 0;
-  return 0;
-}
-
-
-static void
-task_timeout (void *cls)
-{
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &handle_timeout,
-                                                          cls));
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_timeout_no_connect",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_timeout, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_timeout_no_connect.c */
diff --git a/src/util/test_connection_transmit_cancel.c b/src/util/test_connection_transmit_cancel.c
deleted file mode 100644
index 9ef0720ed..000000000
--- a/src/util/test_connection_transmit_cancel.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_transmit_cancel.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-static size_t
-not_run (void *cls, size_t size, void *buf)
-{
-  GNUNET_assert (0);
-  return 0;
-}
-
-
-static void
-task_transmit_cancel (void *cls)
-{
-  int *ok = cls;
-  struct GNUNET_CONNECTION_TransmitHandle *th;
-  struct GNUNET_CONNECTION_Handle *csock;
-
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  th = GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                GNUNET_TIME_UNIT_MINUTES,
-                                                &not_run, cls);
-  GNUNET_assert (NULL != th);
-  GNUNET_CONNECTION_notify_transmit_ready_cancel (th);
-  GNUNET_CONNECTION_destroy (csock);
-  *ok = 0;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_transmit_cancel",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_transmit_cancel, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_transmit_cancel.c */
diff --git a/src/util/test_container_dll.c b/src/util/test_container_dll.c
new file mode 100644
index 000000000..4d2830807
--- /dev/null
+++ b/src/util/test_container_dll.c
@@ -0,0 +1,112 @@
+/*
+ This file is part of GNUnet.
+ Copyright (C) 2017 GNUnet e.V.
+
+ GNUnet is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ GNUnet is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GNUnet; see the file COPYING.  If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @author Christian Grothoff
+ * @file util/test_container_dll.c
+ * @brief Test of DLL operations
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+
+/**
+ * Element in the DLL.
+ */
+struct Element
+{
+  /**
+   * Required pointer to previous element.
+   */
+  struct Element *prev;
+
+  /**
+   * Required pointer to next element.
+   */
+  struct Element *next;
+
+  /**
+   * Used to sort.
+   */
+  unsigned int value;
+};
+
+
+/**
+ * Compare two elements.
+ *
+ * @param cls closure, NULL
+ * @param e1 an element of to sort
+ * @param e2 another element to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
+ */
+static int
+cmp_elem (void *cls,
+          struct Element *e1,
+          struct Element *e2)
+{
+  if (e1->value == e2->value)
+    return 0;
+  return (e1->value < e2->value) ? 1 : -1;
+}
+
+
+int
+main (int argc, char **argv)
+{
+  unsigned int values[] = {
+    4, 5, 8, 6, 9, 3, 7, 2, 1, 0
+  };
+  struct Element *head = NULL;
+  struct Element *tail = NULL;
+  struct Element *e;
+  unsigned int want;
+
+  GNUNET_log_setup ("test-container-dll",
+                    "WARNING",
+                    NULL);
+  for (unsigned int off=0;
+       0 != values[off];
+       off++)
+  {
+    e = GNUNET_new (struct Element);
+    e->value = values[off];
+    GNUNET_CONTAINER_DLL_insert_sorted (struct Element,
+                                        cmp_elem,
+                                        NULL,
+                                        head,
+                                        tail,
+                                        e);
+  }
+
+  want = 1;
+  while (NULL != (e = head))
+  {
+    GNUNET_assert (e->value == want);
+    GNUNET_CONTAINER_DLL_remove (head,
+                                 tail,
+                                 e);
+    GNUNET_free (e);
+    want++;
+  }
+  return 0;
+}
+
+/* end of test_container_heap.c */
diff --git a/src/util/test_container_meta_data.c b/src/util/test_container_meta_data.c
index d84935a1e..cd23674a3 100644
--- a/src/util/test_container_meta_data.c
+++ b/src/util/test_container_meta_data.c
@@ -27,10 +27,9 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-#if HAVE_EXTRACTOR_H
-
 #define ABORT(m) { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); if (m != NULL) GNUNET_CONTAINER_meta_data_destroy(m); return 1; }
 
+
 static int
 testMeta (int i)
 {
@@ -106,10 +105,15 @@ testMeta (int i)
     ABORT (m);
   for (j = 0; j < i; j++)
   {
-    GNUNET_snprintf (val, sizeof (val), "%s.%d",
-                     "A teststring that should compress well.", j);
+    GNUNET_snprintf (val,
+                     sizeof (val),
+                     "%s.%d",
+                     "A teststring that should compress well.",
+                     j);
     if (GNUNET_OK !=
-        GNUNET_CONTAINER_meta_data_delete (m, EXTRACTOR_METATYPE_UNKNOWN, val,
+        GNUNET_CONTAINER_meta_data_delete (m,
+                                           EXTRACTOR_METATYPE_UNKNOWN,
+                                           val,
                                            strlen (val) + 1))
     {
       ABORT (m);
@@ -121,7 +125,8 @@ testMeta (int i)
   return 0;
 }
 
-int
+
+static int
 testMetaMore (int i)
 {
   struct GNUNET_CONTAINER_MetaData *meta;
@@ -135,7 +140,7 @@ testMetaMore (int i)
   {
     GNUNET_snprintf (txt, 128, "%u -- %u\n", i, q);
     GNUNET_CONTAINER_meta_data_insert (meta, "<test>",
-                                       q % EXTRACTOR_metatype_get_max (),
+                                       q % 42 /* EXTRACTOR_metatype_get_max () */,
                                        EXTRACTOR_METAFORMAT_UTF8, "text/plain",
                                        txt, strlen (txt) + 1);
   }
@@ -153,6 +158,7 @@ testMetaMore (int i)
   return 0;
 }
 
+
 static int
 testMetaLink ()
 {
@@ -188,7 +194,8 @@ testMetaLink ()
   return 0;
 }
 
-int
+
+static int
 check ()
 {
   struct GNUNET_CONTAINER_MetaData *meta;
@@ -345,16 +352,5 @@ main (int argc, char *argv[])
   return 0;
 }
 
-#else
-
-int
-main (int argc, char *argv[])
-{
-  fprintf (stderr,
-           "GNU libextractor not found, skipping test.\n");
-  return 0;
-}
-
-#endif
 
 /* end of test_container_meta_data.c */
diff --git a/src/util/test_getopt.c b/src/util/test_getopt.c
index 8e578640d..16f2df8d1 100644
--- a/src/util/test_getopt.c
+++ b/src/util/test_getopt.c
@@ -56,7 +56,7 @@ testVerbose ()
   unsigned int vflags = 0;
 
   const struct GNUNET_GETOPT_CommandLineOption verboseoptionlist[] = {
-    GNUNET_GETOPT_OPTION_VERBOSE (&vflags),
+    GNUNET_GETOPT_option_verbose (&vflags),
     GNUNET_GETOPT_OPTION_END
   };
 
@@ -83,7 +83,7 @@ testVersion ()
     NULL
   };
   const struct GNUNET_GETOPT_CommandLineOption versionoptionlist[] = {
-    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
+    GNUNET_GETOPT_option_version (PACKAGE_VERSION " " VCS_VERSION),
     GNUNET_GETOPT_OPTION_END
   };
 
@@ -105,7 +105,7 @@ testAbout ()
     NULL
   };
   const struct GNUNET_GETOPT_CommandLineOption aboutoptionlist[] = {
-    GNUNET_GETOPT_OPTION_HELP ("Testing"),
+    GNUNET_GETOPT_option_help ("Testing"),
     GNUNET_GETOPT_OPTION_END
   };
 
@@ -131,18 +131,21 @@ testLogOpts ()
   char *fn = NULL;
 
   const struct GNUNET_GETOPT_CommandLineOption logoptionlist[] = {
-    GNUNET_GETOPT_OPTION_LOGFILE (&fn),
-    GNUNET_GETOPT_OPTION_LOGLEVEL (&level),
+    GNUNET_GETOPT_option_logfile (&fn),
+    GNUNET_GETOPT_option_loglevel (&level),
     GNUNET_GETOPT_OPTION_END
   };
 
-  if (5 != GNUNET_GETOPT_run ("test_getopt", logoptionlist, 5, myargv))
+  if (5 != GNUNET_GETOPT_run ("test_getopt",
+                              logoptionlist,
+                              5, myargv))
   {
     GNUNET_break (0);
     return 1;
   }
-  GNUNET_assert (fn != NULL);
-  if ((0 != strcmp (level, "WARNING")) || (0 != strcmp (fn, "filename")))
+  GNUNET_assert (NULL != fn);
+  if ( (0 != strcmp (level, "WARNING")) ||
+       (NULL == strstr (fn, "/filename")) )
   {
     GNUNET_break (0);
     GNUNET_free (level);
@@ -170,21 +173,35 @@ testFlagNum ()
   unsigned long long lnum = 0;
 
   const struct GNUNET_GETOPT_CommandLineOption logoptionlist[] = {
-    {'f', "--flag", NULL, "helptext", 0, &GNUNET_GETOPT_set_one,
-     (void *) &flag},
-    {'n', "--num", "ARG", "helptext", 1, &GNUNET_GETOPT_set_uint,
-     (void *) &num},
-    {'N', "--lnum", "ARG", "helptext", 1, &GNUNET_GETOPT_set_ulong,
-     (void *) &lnum},
+    GNUNET_GETOPT_option_flag ('f',
+                                  "--flag",
+                                  "helptext",
+                                  &flag),
+    GNUNET_GETOPT_option_uint ('n',
+                                   "--num",
+                                   "ARG",
+                                   "helptext",
+                                   &num),
+    GNUNET_GETOPT_option_ulong ('N',
+                                    "--lnum",
+                                    "ARG",
+                                    "helptext",
+                                    &lnum),
     GNUNET_GETOPT_OPTION_END
   };
 
-  if (6 != GNUNET_GETOPT_run ("test_getopt", logoptionlist, 6, myargv))
+  if (6 !=
+      GNUNET_GETOPT_run ("test_getopt",
+                         logoptionlist,
+                         6,
+                         myargv))
   {
     GNUNET_break (0);
     return 1;
   }
-  if ((1 != flag) || (42 != num) || (42 != lnum))
+  if ( (1 != flag) ||
+       (42 != num) ||
+       (42 != lnum))
   {
     GNUNET_break (0);
     return 1;
@@ -198,7 +215,9 @@ main (int argc, char *argv[])
 {
   int errCnt = 0;
 
-  GNUNET_log_setup ("test_getopt", "WARNING", NULL);
+  GNUNET_log_setup ("test_getopt",
+                    "WARNING",
+                    NULL);
   /* suppress output from -h, -v options */
 #ifndef MINGW
   GNUNET_break (0 == CLOSE (1));
diff --git a/src/util/test_program.c b/src/util/test_program.c
index 669cee7bd..6d51b1872 100644
--- a/src/util/test_program.c
+++ b/src/util/test_program.c
@@ -24,37 +24,19 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-static int setme1, setme2;
-
-static struct GNUNET_GETOPT_CommandLineOption options1[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options2[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'N', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options3[] = {
-  {'N', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options4[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'n', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
+
+static int setme1;
+
+static int setme2;
+
 
 /**
  * Main function that will be run.
  */
-
 static void
-runner (void *cls, char *const *args, const char *cfgfile,
+runner (void *cls,
+        char *const *args,
+        const char *cfgfile,
         const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   int *ok = cls;
@@ -62,21 +44,16 @@ runner (void *cls, char *const *args, const char *cfgfile,
   GNUNET_assert (setme1 == 1);
   GNUNET_assert (0 == strcmp (args[0], "extra"));
   GNUNET_assert (args[1] == NULL);
-  GNUNET_assert (0 == strcmp (cfgfile, "test_program_data.conf"));
-
+  GNUNET_assert (NULL != strstr (cfgfile, "/test_program_data.conf"));
   *ok = 0;
 }
 
-/**
- * Main method, starts scheduler with task1,
- * checks that "ok" is correct at the end.
- */
-static int
-check ()
+
+int
+main (int argc, char *argv[])
 {
   int ok = 1;
-
-  char *const argv[] = {
+  char *const argvx[] = {
     "test_program",
     "-c",
     "test_program_data.conf",
@@ -86,33 +63,75 @@ check ()
     "extra",
     NULL
   };
+  struct GNUNET_GETOPT_CommandLineOption options1[] = {
+    GNUNET_GETOPT_option_flag ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options2[] = {
+    GNUNET_GETOPT_option_flag ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_option_flag ('N',
+                                  "number",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options3[] = {
+    GNUNET_GETOPT_option_flag ('N',
+                                  "number",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_option_flag ('n',
+                                  "name",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options4[] = {
+    GNUNET_GETOPT_option_flag ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_option_flag ('n',
+                                  "name",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
 
+
+  GNUNET_log_setup ("test_program",
+                    "WARNING",
+                    NULL);
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options1, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program",
+                                     "A test",
+                                     options1,
+                                     &runner, &ok));
 
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options2, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options2,
+                                     &runner, &ok));
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options3, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options3,
+                                     &runner, &ok));
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options4, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options4,
+                                     &runner, &ok));
 
   return ok;
 }
 
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_program", "WARNING", NULL);
-  ret += check ();
-
-  return ret;
-}
-
 /* end of test_program.c */
diff --git a/src/util/test_server.c b/src/util/test_server.c
deleted file mode 100644
index 8003adbf4..000000000
--- a/src/util/test_server.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2014, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server.c
- * @brief tests for server.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * TCP port to use for the server.
- */
-#define PORT 12435
-
-/**
- * Timeout to use for operations.
- */
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2)
-
-/**
- * Test message type.
- */
-#define MY_TYPE 128
-
-/**
- * Test message type.
- */
-#define MY_TYPE2 129
-
-/**
- * Handle for the server.
- */
-static struct GNUNET_SERVER_Handle *server;
-
-/**
- * Handle for the client.
- */
-static struct GNUNET_MQ_Handle *mq;
-
-/**
- * Handle of the server for the client.
- */
-static struct GNUNET_SERVER_Client *argclient;
-
-/**
- * Our configuration.
- */
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Number indiciating in which phase of the test we are.
- */
-static int ok;
-
-
-/**
- * Final task invoked to clean up.
- *
- * @param cls NULL
- */
-static void
-finish_up (void *cls)
-{
-  GNUNET_assert (7 == ok);
-  ok = 0;
-  GNUNET_SERVER_destroy (server);
-  GNUNET_MQ_destroy (mq);
-  GNUNET_CONFIGURATION_destroy (cfg);
-}
-
-
-/**
- * The server has received the second message, initiate clean up.
- *
- * @param cls NULL
- * @param client client we got the message from
- * @param message the message
- */
-static void
-recv_fin_cb (void *cls,
-             struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (6 == ok);
-  ok = 7;
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-  GNUNET_SCHEDULER_add_now (&finish_up, NULL);
-}
-
-
-/**
- * We have received the reply from the server, check that we are at
- * the right stage and queue the next message to the server.  Cleans
- * up #argclient.
- *
- * @param cls NULL
- * @param msg message we got from the server
- */
-static void
-handle_reply (void *cls,
-              const struct GNUNET_MessageHeader *msg)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *m;
-
-  GNUNET_assert (4 == ok);
-  ok = 6;
-  env = GNUNET_MQ_msg (m,
-                       MY_TYPE2);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Send a reply of type #MY_TYPE from the server to the client.
- * Checks that we are in the right phase and transmits the
- * reply.  Cleans up #argclient state.
- *
- * @param cls NULL
- * @param size number of bytes we are allowed to send
- * @param buf where to copy the reply
- * @return number of bytes written to @a buf
- */
-static size_t
-reply_msg (void *cls,
-           size_t size,
-           void *buf)
-{
-  struct GNUNET_MessageHeader msg;
-
-  GNUNET_assert (3 == ok);
-  ok = 4;
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  msg.type = htons (MY_TYPE);
-  msg.size = htons (sizeof (struct GNUNET_MessageHeader));
-  GNUNET_memcpy (buf, &msg, sizeof (struct GNUNET_MessageHeader));
-  GNUNET_assert (NULL != argclient);
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-  GNUNET_SERVER_client_drop (argclient);
-  argclient = NULL;
-  return sizeof (struct GNUNET_MessageHeader);
-}
-
-
-/**
- * Function called whenever the server receives a message of
- * type #MY_TYPE.  Checks that we are at the stage where
- * we expect the first message, then sends a reply.  Stores
- * the handle to the client in #argclient.
- *
- * @param cls NULL
- * @param client client that sent the message
- * @param message the message we received
- */
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *client,
-         const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (2 == ok);
-  ok = 3;
-  argclient = client;
-  GNUNET_SERVER_client_keep (argclient);
-  GNUNET_assert (sizeof (struct GNUNET_MessageHeader) == ntohs (message->size));
-  GNUNET_assert (MY_TYPE == ntohs (message->type));
-  GNUNET_assert (NULL !=
-                 GNUNET_SERVER_notify_transmit_ready (client,
-                                                      ntohs (message->size),
-                                                      TIMEOUT,
-                                                      &reply_msg,
-                                                      NULL));
-}
-
-
-/**
- * Message handlers for the server.
- */
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {&recv_fin_cb, NULL, MY_TYPE2, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-/**
- * Generic error handler, called with the appropriate error code and
- * the same closure specified at the creation of the message queue.
- * Not every message queue implementation supports an error handler.
- *
- * @param cls closure with the `struct GNUNET_STATISTICS_Handle *`
- * @param error error code
- */
-static void
-mq_error_handler (void *cls,
-                  enum GNUNET_MQ_Error error)
-{
-  GNUNET_assert (0); /* should never happen */
-}
-
-
-/**
- * First task run by the scheduler.  Initializes the server and
- * a client and asks for a transmission from the client to the
- * server.
- *
- * @param cls NULL
- */
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-  struct GNUNET_MQ_MessageHandler chandlers[] = {
-    GNUNET_MQ_hd_fixed_size (reply,
-                             MY_TYPE,
-                             struct GNUNET_MessageHeader,
-                             cls),
-    GNUNET_MQ_handler_end ()
-  };
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server = GNUNET_SERVER_create (NULL, NULL,
-                                 sap, slens,
-                                 TIMEOUT, GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg,
-                                         "test-server",
-                                         "PORT",
-                                         PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "test-server",
-                                         "HOSTNAME",
-                                         "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "resolver",
-                                         "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test-server",
-                              chandlers,
-                              &mq_error_handler,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Runs the test.
- *
- * @param argc length of @a argv
- * @param argv command line arguments (ignored)
- * @return 0 on success, otherwise phase of failure
- */
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-/* end of test_server.c */
diff --git a/src/util/test_server_disconnect.c b/src/util/test_server_disconnect.c
deleted file mode 100644
index c3d003e90..000000000
--- a/src/util/test_server_disconnect.c
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_disconnect.c
- * @brief tests for server.c,  specifically GNUNET_SERVER_client_disconnect
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define PORT 12435
-
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250)
-
-#define MY_TYPE 128
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-finish_up (void *cls)
-{
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  GNUNET_SERVER_destroy (server);
-  GNUNET_MQ_destroy (mq);
-  GNUNET_CONFIGURATION_destroy (cfg);
-}
-
-
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *clientarg)
-{
-  if (NULL == clientarg)
-    return;
-  GNUNET_assert (ok == 4);
-  ok = 5;
-  GNUNET_SCHEDULER_add_now (&finish_up, NULL);
-}
-
-
-static void
-server_disconnect (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok = 4;
-  GNUNET_SERVER_client_disconnect (argclient);
-  GNUNET_SERVER_client_drop (argclient);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *client,
-         const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (ok == 2);
-  ok = 3;
-  GNUNET_SERVER_client_keep (client);
-  GNUNET_SCHEDULER_add_now (&server_disconnect, client);
-  GNUNET_assert (sizeof (struct GNUNET_MessageHeader) == ntohs (message->size));
-  GNUNET_assert (MY_TYPE == ntohs (message->type));
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server = GNUNET_SERVER_create (NULL, NULL, sap, slens, TIMEOUT, GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, NULL);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg, "test-server", "PORT", PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test-server", "HOSTNAME",
-                                         "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test-server",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Main method, starts scheduler with task1,
- * checks that "ok" is correct at the end.
- */
-static int
-check ()
-{
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_server_disconnect", "WARNING", NULL);
-  ret += check ();
-
-  return ret;
-}
-
-/* end of test_server_disconnect.c */
diff --git a/src/util/test_server_mst_interrupt.c b/src/util/test_server_mst_interrupt.c
deleted file mode 100644
index 3141a75bd..000000000
--- a/src/util/test_server_mst_interrupt.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_mst_interrupt.c
- * @brief test for interrupt message processing in server_mst.c
- */
-#include "platform.h"
-#include "gnunet_protocols.h"
-#include "gnunet_util_lib.h"
-
-static struct GNUNET_SERVER_MessageStreamTokenizer * mst;
-
-
-/* Callback destroying mst with data in buffer */
-static int
-mst_cb (void *cls, void *client,
-        const struct GNUNET_MessageHeader * message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MST gave me message, destroying\n");
-  GNUNET_SERVER_mst_destroy (mst);
-  return GNUNET_SYSERR;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  struct GNUNET_PeerIdentity id;
-  struct GNUNET_MessageHeader msg[2];
-
-  GNUNET_log_setup ("test_server_mst_interrupt", "WARNING", NULL);
-  memset (&id, 0, sizeof (id));
-  msg[0].size = htons (sizeof (msg));
-  msg[0].type = htons (sizeof (GNUNET_MESSAGE_TYPE_DUMMY));
-  mst = GNUNET_SERVER_mst_create(mst_cb, NULL);
-  GNUNET_SERVER_mst_receive (mst, &id,
-                             (const char *) &msg, 2 * sizeof (msg),
-                             GNUNET_NO, GNUNET_NO);
-  /* If we reach this line, it did not crash */
-  return 0;
-}
-
-/* end of test_server_mst_interrupt.c */
diff --git a/src/util/test_server_with_client.c b/src/util/test_server_with_client.c
deleted file mode 100644
index 63bfda00c..000000000
--- a/src/util/test_server_with_client.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_with_client.c
- * @brief tests for server.c and client.c,
- *       specifically disconnect_notify,
- *       client_get_address and receive_done (resume processing)
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 22335
-
-#define MY_TYPE 128
-
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-send_done (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok++;
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *argclient,
-         const struct GNUNET_MessageHeader *message)
-{
-  void *addr;
-  size_t addrlen;
-  struct sockaddr_in sa;
-  struct sockaddr_in *have;
-
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_SERVER_client_get_address (argclient,
-                                                   &addr,
-                                                   &addrlen));
-
-  GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-  have = addr;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = have->sin_port;
-  sa.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  GNUNET_assert (0 == memcmp (&sa, addr, addrlen));
-  GNUNET_free (addr);
-  switch (ok)
-  {
-  case 2:
-    ok++;
-    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                  (GNUNET_TIME_UNIT_MILLISECONDS, 50),
-                                  &send_done,
-                                  argclient);
-    break;
-  case 4:
-    ok++;
-    GNUNET_MQ_destroy (mq);
-    GNUNET_SERVER_receive_done (argclient,
-                                GNUNET_OK);
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-
-}
-
-
-static void
-clean_up (void *cls)
-{
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
-  GNUNET_CONFIGURATION_destroy (cfg);
-  cfg = NULL;
-}
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *client)
-{
-  if (client == NULL)
-    return;
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  GNUNET_SCHEDULER_add_now (&clean_up, NULL);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server =
-      GNUNET_SERVER_create (NULL, NULL, sap, slens,
-                            GNUNET_TIME_relative_multiply
-                            (GNUNET_TIME_UNIT_MILLISECONDS, 250), GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  handlers[0].callback_cls = cls;
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, cls);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg, "test", "PORT", PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test", "HOSTNAME", "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server_with_client",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, NULL);
-  return ok;
-}
-
-/* end of test_server_with_client.c */
diff --git a/src/util/test_server_with_client_unix.c b/src/util/test_server_with_client_unix.c
deleted file mode 100644
index d240f1a88..000000000
--- a/src/util/test_server_with_client_unix.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_with_client_unix.c
- * @brief tests for server.c and client.c,
- *       specifically disconnect_notify,
- *       client_get_address and receive_done (resume processing)
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define MY_TYPE 128
-
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-send_done (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok++;
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *argclient,
-         const struct GNUNET_MessageHeader *message)
-{
-  switch (ok)
-  {
-  case 2:
-    ok++;
-    (void) GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                         (GNUNET_TIME_UNIT_MILLISECONDS, 50),
-                                         &send_done,
-                                         argclient);
-    break;
-  case 4:
-    ok++;
-    GNUNET_MQ_destroy (mq);
-    GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-
-}
-
-
-static void
-clean_up (void *cls)
-{
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
-  GNUNET_CONFIGURATION_destroy (cfg);
-  cfg = NULL;
-}
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *client)
-{
-  if (client == NULL)
-    return;
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  (void) GNUNET_SCHEDULER_add_now (&clean_up, NULL);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_un un;
-  const char *unixpath = "/tmp/testsock";
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  memset (&un, 0, sizeof (un));
-  un.sun_family = AF_UNIX;
-  strncpy(un.sun_path, unixpath, sizeof (un.sun_path) - 1);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  un.sun_len = (u_char) sizeof (un);
-#endif
-
-  sap[0] = (struct sockaddr *) &un;
-  slens[0] = sizeof (un);
-  sap[1] = NULL;
-  slens[1] = 0;
-  server =
-      GNUNET_SERVER_create (NULL, NULL, sap, slens,
-                            GNUNET_TIME_relative_multiply
-                            (GNUNET_TIME_UNIT_MILLISECONDS, 250), GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  handlers[0].callback_cls = cls;
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, cls);
-  cfg = GNUNET_CONFIGURATION_create ();
-
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test", "UNIXPATH", unixpath);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server_with_client_unix",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, NULL);
-  return ok;
-}
-
-/* end of test_server_with_client_unix.c */
diff --git a/src/util/test_service.c b/src/util/test_service.c
index d2136b42f..1567c97ce 100644
--- a/src/util/test_service.c
+++ b/src/util/test_service.c
@@ -148,7 +148,7 @@ check (const char *sname)
               sname);
   global_ret = 1;
   GNUNET_assert (0 ==
-                 GNUNET_SERVICE_ruN_ (3,
+                 GNUNET_SERVICE_run_ (3,
                                       argv,
                                       sname,
                                       GNUNET_SERVICE_OPTION_NONE,
diff --git a/src/util/time.c b/src/util/time.c
index 89b0c2d44..19100ac36 100644
--- a/src/util/time.c
+++ b/src/util/time.c
@@ -27,7 +27,7 @@
 #include "gnunet_crypto_lib.h"
 #include "gnunet_time_lib.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-time", __VA_ARGS__)
 
 /**
  * Variable used to simulate clock skew.  Used for testing, never in production.
diff --git a/src/util/util.conf b/src/util/util.conf
index ecc94ead0..ceb5fdcbb 100644
--- a/src/util/util.conf
+++ b/src/util/util.conf
@@ -37,9 +37,8 @@ GNUNET_RUNTIME_DIR = ${TMPDIR:-${TMP:-/tmp}}/gnunet-system-runtime/
 GNUNET_USER_RUNTIME_DIR = ${TMPDIR:-${TMP:-/tmp}}/gnunet-${USERHOME:-${USER:-user}}-runtime/
 
 
-# Legacy option...
-# GNUNET_TEST_HOME = ~/.gnunet/
-# GNUNET_TEST_HOME = /var/lib/gnunet/
+# Override for GNUNET_HOME used by test cases.
+# GNUNET_TEST_HOME = /tmp/foo/bar
 
 # DEFAULTCONFIG = /etc/gnunet.conf
 # If 'DEFAULTCONFIG' is not defined, the current
diff --git a/src/util/win.c b/src/util/win.c
index 7cd7e0f3c..97877b0ca 100644
--- a/src/util/win.c
+++ b/src/util/win.c
@@ -534,10 +534,9 @@ EnumNICs3 (struct EnumNICs3_results **results, int *results_count)
           for (i = 0; !found && i < interfaces4_len / sizeof (INTERFACE_INFO); i++)
           {
             struct sockaddr_in *m = (struct sockaddr_in *) &r->mask;
-            if (GNUNET_memcpy (&interfaces4[i].iiAddress.Address,
+            GNUNET_memcpy (&interfaces4[i].iiAddress.Address,
                 unicast->Address.lpSockaddr,
-                unicast->Address.iSockaddrLength) != 0)
-              continue;
+                unicast->Address.iSockaddrLength);
             found = 1;
             GNUNET_memcpy (&r->address, &interfaces4[i].iiAddress.Address,
                 sizeof (struct sockaddr_in));
@@ -557,10 +556,9 @@ EnumNICs3 (struct EnumNICs3_results **results, int *results_count)
               interfaces6 != NULL && !found && i < interfaces6->iAddressCount;
               i++)
           {
-            if (GNUNET_memcpy (interfaces6->Address[i].lpSockaddr,
+            GNUNET_memcpy (interfaces6->Address[i].lpSockaddr,
                 unicast->Address.lpSockaddr,
-                unicast->Address.iSockaddrLength) != 0)
-              continue;
+                unicast->Address.iSockaddrLength);
             found = 1;
             GNUNET_memcpy (&r->address, interfaces6->Address[i].lpSockaddr,
                 sizeof (struct sockaddr_in6));
diff --git a/src/util/winproc.c b/src/util/winproc.c
index 66c225ce5..318b68a88 100644
--- a/src/util/winproc.c
+++ b/src/util/winproc.c
@@ -69,7 +69,7 @@ TAddAce GNAddAce;
 TAddAccessAllowedAce GNAddAccessAllowedAce;
 TSetNamedSecurityInfo GNSetNamedSecurityInfo;
 
-#define LOG(kind,...) GNUNET_log_from (kind, "winproc", __VA_ARGS__)
+#define LOG(kind,...) GNUNET_log_from (kind, "util-winproc", __VA_ARGS__)
 /**
  * Log (panic) messages from PlibC
  */
diff --git a/src/vpn/Makefile.am b/src/vpn/Makefile.am
index 5517a45e3..5c16fa349 100644
--- a/src/vpn/Makefile.am
+++ b/src/vpn/Makefile.am
@@ -79,5 +79,3 @@ libgnunetvpn_la_LIBADD = \
  $(top_builddir)/src/util/libgnunetutil.la $(XLIB)
 libgnunetvpn_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS)
-
-
diff --git a/src/vpn/gnunet-helper-vpn-windows.c b/src/vpn/gnunet-helper-vpn-windows.c
index a9596752a..e74a0aa2f 100644
--- a/src/vpn/gnunet-helper-vpn-windows.c
+++ b/src/vpn/gnunet-helper-vpn-windows.c
@@ -77,7 +77,7 @@
 static boolean privilege_testing = FALSE;
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/vpn/gnunet-helper-vpn.c b/src/vpn/gnunet-helper-vpn.c
index 02889d65b..4ed4e079e 100644
--- a/src/vpn/gnunet-helper-vpn.c
+++ b/src/vpn/gnunet-helper-vpn.c
@@ -53,7 +53,7 @@
 #define DEBUG GNUNET_NO
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/vpn/gnunet-service-vpn.c b/src/vpn/gnunet-service-vpn.c
index c66023c85..d9daaa7e2 100644
--- a/src/vpn/gnunet-service-vpn.c
+++ b/src/vpn/gnunet-service-vpn.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2010, 2011, 2012, 2016 Christian Grothoff
+     Copyright (C) 2010, 2011, 2012, 2016, 2017 Christian Grothoff
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -221,11 +221,6 @@ struct ChannelState
   struct GNUNET_REGEX_Search *search;
 
   /**
-   * Active transmission handle, NULL for none.
-   */
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  /**
    * Entry for this entry in the channel_heap, NULL as long as this
    * channel state is not fully bound.
    */
@@ -535,11 +530,6 @@ free_channel_state (struct ChannelState *ts)
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Cleaning up channel state\n");
-  if (NULL != ts->th)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (ts->th);
-    ts->th = NULL;
-  }
   if (NULL != (channel = ts->channel))
   {
     ts->channel = NULL;
@@ -585,97 +575,32 @@ free_channel_state (struct ChannelState *ts)
 
 
 /**
- * Send a message from the message queue via cadet.
- *
- * @param cls the `struct ChannelState` with the message queue
- * @param size number of bytes available in @a buf
- * @param buf where to copy the message
- * @return number of bytes copied to @a buf
- */
-static size_t
-send_to_peer_notify_callback (void *cls, size_t size, void *buf)
-{
-  struct ChannelState *ts = cls;
-  struct ChannelMessageQueueEntry *tnq;
-  size_t ret;
-
-  ts->th = NULL;
-  if (NULL == buf)
-    return 0;
-  tnq = ts->tmq_head;
-  GNUNET_assert (NULL != tnq);
-  GNUNET_assert (size >= tnq->len);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending %u bytes via cadet channel\n",
-              (unsigned int) tnq->len);
-  GNUNET_CONTAINER_DLL_remove (ts->tmq_head,
-                               ts->tmq_tail,
-                               tnq);
-  ts->tmq_length--;
-  GNUNET_memcpy (buf, tnq->msg, tnq->len);
-  ret = tnq->len;
-  GNUNET_free (tnq);
-  if (NULL != (tnq = ts->tmq_head))
-  {
-    if (NULL == ts->th)
-      ts->th = GNUNET_CADET_notify_transmit_ready (ts->channel,
-                                                   GNUNET_NO /* cork */,
-                                                   GNUNET_TIME_UNIT_FOREVER_REL,
-                                                   tnq->len,
-                                                   &send_to_peer_notify_callback,
-                                                   ts);
-  }
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# Bytes given to cadet for transmission"),
-                            ret, GNUNET_NO);
-  return ret;
-}
-
-
-/**
  * Add the given message to the given channel and trigger the
  * transmission process.
  *
- * @param tnq message to queue
  * @param ts channel to queue the message for
+ * @param env message to queue
  */
 static void
-send_to_channel (struct ChannelMessageQueueEntry *tnq,
-                struct ChannelState *ts)
+send_to_channel (struct ChannelState *ts,
+                 struct GNUNET_MQ_Envelope *env)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Queueing %u bytes for transmission via cadet channel\n",
-              (unsigned int) tnq->len);
+  struct GNUNET_MQ_Handle *mq;
+
   GNUNET_assert (NULL != ts->channel);
-  GNUNET_CONTAINER_DLL_insert_tail (ts->tmq_head,
-                                    ts->tmq_tail,
-                                    tnq);
-  ts->tmq_length++;
-  if (ts->tmq_length > MAX_MESSAGE_QUEUE_SIZE)
+  mq = GNUNET_CADET_get_mq (ts->channel);
+  GNUNET_MQ_send (mq,
+                  env);
+  if (GNUNET_MQ_get_length (mq) > MAX_MESSAGE_QUEUE_SIZE)
   {
-    struct ChannelMessageQueueEntry *dq;
-
-    dq = ts->tmq_head;
-    GNUNET_assert (dq != tnq);
-    GNUNET_CONTAINER_DLL_remove (ts->tmq_head,
-                                 ts->tmq_tail,
-                                 dq);
-    ts->tmq_length--;
-    GNUNET_CADET_notify_transmit_ready_cancel (ts->th);
-    ts->th = NULL;
+    env = GNUNET_MQ_unsent_head (mq);
+    GNUNET_assert (NULL != env);
     GNUNET_STATISTICS_update (stats,
-                              gettext_noop ("# Bytes dropped in cadet queue (overflow)"),
-                              dq->len,
+                              gettext_noop ("# Messages dropped in cadet queue (overflow)"),
+                              1,
                               GNUNET_NO);
-    GNUNET_free (dq);
+    GNUNET_MQ_discard (env);
   }
-  if (NULL == ts->th)
-    ts->th = GNUNET_CADET_notify_transmit_ready (ts->channel,
-                                                GNUNET_NO /* cork */,
-                                                GNUNET_TIME_UNIT_FOREVER_REL,
-                                                tnq->len,
-                                                &send_to_peer_notify_callback,
-                                                ts);
 }
 
 
@@ -710,6 +635,767 @@ print_channel_destination (const struct DestinationEntry *de)
 
 
 /**
+ * Function called whenever a channel is destroyed.  Should clean up
+ * any associated state.
+ *
+ * @param cls our `struct ChannelState`
+ * @param channel connection to the other end (henceforth invalid)
+ */
+static void
+channel_cleaner (void *cls,
+                 const struct GNUNET_CADET_Channel *channel)
+{
+  struct ChannelState *ts = cls;
+
+  ts->channel = NULL; /* we must not call GNUNET_CADET_channel_destroy() anymore */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "CADET notified us about death of channel to `%s'\n",
+              print_channel_destination (&ts->destination));
+  free_channel_state (ts);
+}
+
+
+/**
+ * Synthesize a plausible ICMP payload for an ICMP error
+ * response on the given channel.
+ *
+ * @param ts channel information
+ * @param ipp IPv4 header to fill in (ICMP payload)
+ * @param udp "UDP" header to fill in (ICMP payload); might actually
+ *            also be the first 8 bytes of the TCP header
+ */
+static void
+make_up_icmpv4_payload (struct ChannelState *ts,
+                        struct GNUNET_TUN_IPv4Header *ipp,
+                        struct GNUNET_TUN_UdpHeader *udp)
+{
+  GNUNET_TUN_initialize_ipv4_header (ipp,
+                                     ts->protocol,
+                                     sizeof (struct GNUNET_TUN_TcpHeader),
+                                     &ts->source_ip.v4,
+                                     &ts->destination_ip.v4);
+  udp->source_port = htons (ts->source_port);
+  udp->destination_port = htons (ts->destination_port);
+  udp->len = htons (0);
+  udp->crc = htons (0);
+}
+
+
+/**
+ * Synthesize a plausible ICMP payload for an ICMP error
+ * response on the given channel.
+ *
+ * @param ts channel information
+ * @param ipp IPv6 header to fill in (ICMP payload)
+ * @param udp "UDP" header to fill in (ICMP payload); might actually
+ *            also be the first 8 bytes of the TCP header
+ */
+static void
+make_up_icmpv6_payload (struct ChannelState *ts,
+                        struct GNUNET_TUN_IPv6Header *ipp,
+                        struct GNUNET_TUN_UdpHeader *udp)
+{
+  GNUNET_TUN_initialize_ipv6_header (ipp,
+                                     ts->protocol,
+                                     sizeof (struct GNUNET_TUN_TcpHeader),
+                                     &ts->source_ip.v6,
+                                     &ts->destination_ip.v6);
+  udp->source_port = htons (ts->source_port);
+  udp->destination_port = htons (ts->destination_port);
+  udp->len = htons (0);
+  udp->crc = htons (0);
+}
+
+
+/**
+ * We got an ICMP packet back from the CADET channel.  Check it is OK.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param message the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_icmp_back (void *cls,
+                 const struct GNUNET_EXIT_IcmpToVPNMessage *i2v)
+{
+  struct ChannelState *ts = cls;
+
+  if (NULL == ts->heap_node)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (AF_UNSPEC == ts->af)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We got an ICMP packet back from the CADET channel.  Pass it on to the
+ * local virtual interface via the helper.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param message the actual message
+ */
+static void
+handle_icmp_back (void *cls,
+                  const struct GNUNET_EXIT_IcmpToVPNMessage *i2v)
+{
+  struct ChannelState *ts = cls;
+  size_t mlen;
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# ICMP packets received from cadet"),
+                            1,
+                            GNUNET_NO);
+  mlen = ntohs (i2v->header.size) - sizeof (struct GNUNET_EXIT_IcmpToVPNMessage);
+  {
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received ICMP packet from cadet, sending %u bytes from %s -> %s via TUN\n",
+                (unsigned int) mlen,
+                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
+                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)));
+  }
+  switch (ts->af)
+  {
+  case AF_INET:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
+        + sizeof (struct GNUNET_TUN_IcmpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        /* reserve some extra space in case we have an ICMP type here where
+           we will need to make up the payload ourselves */
+        char buf[size + sizeof (struct GNUNET_TUN_IPv4Header) + 8] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
+        struct GNUNET_TUN_IcmpHeader *icmp = (struct GNUNET_TUN_IcmpHeader *) &ipv4[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV4);
+        GNUNET_TUN_initialize_ipv4_header (ipv4,
+                                           IPPROTO_ICMP,
+                                           sizeof (struct GNUNET_TUN_IcmpHeader) + mlen,
+                                           &ts->destination_ip.v4,
+                                           &ts->source_ip.v4);
+        *icmp = i2v->icmp_header;
+        GNUNET_memcpy (&icmp[1],
+                       &i2v[1],
+                       mlen);
+        /* For some ICMP types, we need to adjust (make up) the payload here.
+           Also, depending on the AF used on the other side, we have to
+           do ICMP PT (translate ICMP types) */
+        switch (ntohl (i2v->af))
+        {
+        case AF_INET:
+          switch (icmp->type)
+          {
+          case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
+          case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
+            break;
+          case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
+          case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
+          case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
+            {
+              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv4_payload (ts, ipp, udp);
+            }
+            break;
+          default:
+            GNUNET_break_op (0);
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
+                                      1, GNUNET_NO);
+            return;
+          }
+          /* end AF_INET */
+          break;
+        case AF_INET6:
+          /* ICMP PT 6-to-4 and possibly making up payloads */
+          switch (icmp->type)
+          {
+          case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
+            icmp->type = GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE;
+            {
+              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv4_payload (ts, ipp, udp);
+            }
+            break;
+          case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
+            icmp->type = GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED;
+            {
+              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv4_payload (ts, ipp, udp);
+            }
+            break;
+          case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
+          case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv6 packets dropped (impossible PT to v4)"),
+                                      1, GNUNET_NO);
+            return;
+          case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
+            icmp->type = GNUNET_TUN_ICMPTYPE_ECHO_REQUEST;
+            break;
+          case GNUNET_TUN_ICMPTYPE6_ECHO_REPLY:
+            icmp->type = GNUNET_TUN_ICMPTYPE_ECHO_REPLY;
+            break;
+          default:
+            GNUNET_break_op (0);
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
+                                      1, GNUNET_NO);
+            return;
+          }
+          /* end AF_INET6 */
+          break;
+        default:
+          GNUNET_break_op (0);
+          return;
+        }
+        msg->size = htons (size);
+        GNUNET_TUN_calculate_icmp_checksum (icmp,
+                                            &i2v[1],
+                                            mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  case AF_INET6:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
+        + sizeof (struct GNUNET_TUN_IcmpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        char buf[size + sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
+        struct GNUNET_TUN_IcmpHeader *icmp = (struct GNUNET_TUN_IcmpHeader *) &ipv6[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV6);
+        GNUNET_TUN_initialize_ipv6_header (ipv6,
+                                           IPPROTO_ICMPV6,
+                                           sizeof (struct GNUNET_TUN_IcmpHeader) + mlen,
+                                           &ts->destination_ip.v6,
+                                           &ts->source_ip.v6);
+        *icmp = i2v->icmp_header;
+        GNUNET_memcpy (&icmp[1],
+                &i2v[1],
+                mlen);
+
+        /* For some ICMP types, we need to adjust (make up) the payload here.
+           Also, depending on the AF used on the other side, we have to
+           do ICMP PT (translate ICMP types) */
+        switch (ntohl (i2v->af))
+        {
+        case AF_INET:
+          /* ICMP PT 4-to-6 and possibly making up payloads */
+          switch (icmp->type)
+          {
+          case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
+            icmp->type = GNUNET_TUN_ICMPTYPE6_ECHO_REPLY;
+            break;
+          case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
+            icmp->type = GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST;
+            break;
+          case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
+            icmp->type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
+            {
+              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv6_payload (ts, ipp, udp);
+            }
+            break;
+          case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
+            icmp->type = GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED;
+            {
+              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv6_payload (ts, ipp, udp);
+            }
+            break;
+          case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv4 packets dropped (impossible PT to v6)"),
+                                      1, GNUNET_NO);
+            return;
+          default:
+            GNUNET_break_op (0);
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
+                                      1, GNUNET_NO);
+            return;
+          }
+          /* end AF_INET */
+          break;
+        case AF_INET6:
+          switch (icmp->type)
+          {
+          case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
+          case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
+          case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
+          case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
+            {
+              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
+              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
+
+              if (mlen != 0)
+                {
+                  /* sender did not strip ICMP payload? */
+                  GNUNET_break_op (0);
+                  return;
+                }
+              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
+              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
+              make_up_icmpv6_payload (ts, ipp, udp);
+            }
+            break;
+          case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
+            break;
+          default:
+            GNUNET_break_op (0);
+            GNUNET_STATISTICS_update (stats,
+                                      gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
+                                      1, GNUNET_NO);
+            return;
+          }
+          /* end AF_INET6 */
+          break;
+        default:
+          GNUNET_break_op (0);
+          return;
+        }
+        msg->size = htons (size);
+        GNUNET_TUN_calculate_icmp_checksum (icmp,
+                                            &i2v[1], mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  default:
+    GNUNET_assert (0);
+  }
+  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
+                                     GNUNET_TIME_absolute_get ().abs_value_us);
+  GNUNET_CADET_receive_done (ts->channel);
+}
+
+
+/**
+ * We got a UDP packet back from the CADET channel.  Check that it is OK.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param reply the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_udp_back (void *cls,
+                  const struct GNUNET_EXIT_UdpReplyMessage *reply)
+{
+  struct ChannelState *ts = cls;
+
+  if (NULL == ts->heap_node)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (AF_UNSPEC == ts->af)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We got a UDP packet back from the CADET channel.  Pass it on to the
+ * local virtual interface via the helper.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param reply the actual message
+ */
+static void
+handle_udp_back (void *cls,
+                 const struct GNUNET_EXIT_UdpReplyMessage *reply)
+{
+  struct ChannelState *ts = cls;
+  size_t mlen;
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# UDP packets received from cadet"),
+                            1,
+                            GNUNET_NO);
+  mlen = ntohs (reply->header.size) - sizeof (struct GNUNET_EXIT_UdpReplyMessage);
+  {
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received UDP reply from cadet, sending %u bytes from [%s]:%u -> [%s]:%u via TUN\n",
+                (unsigned int) mlen,
+                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
+                ts->destination_port,
+                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)),
+                ts->source_port);
+  }
+  switch (ts->af)
+  {
+  case AF_INET:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
+        + sizeof (struct GNUNET_TUN_UdpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        char buf[size] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
+        struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipv4[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        msg->size = htons (size);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV4);
+        GNUNET_TUN_initialize_ipv4_header (ipv4,
+                                           IPPROTO_UDP,
+                                           sizeof (struct GNUNET_TUN_UdpHeader) + mlen,
+                                           &ts->destination_ip.v4,
+                                           &ts->source_ip.v4);
+        if (0 == ntohs (reply->source_port))
+          udp->source_port = htons (ts->destination_port);
+        else
+          udp->source_port = reply->source_port;
+        if (0 == ntohs (reply->destination_port))
+          udp->destination_port = htons (ts->source_port);
+        else
+          udp->destination_port = reply->destination_port;
+        udp->len = htons (mlen + sizeof (struct GNUNET_TUN_UdpHeader));
+        GNUNET_TUN_calculate_udp4_checksum (ipv4,
+                                            udp,
+                                            &reply[1],
+                                            mlen);
+        GNUNET_memcpy (&udp[1],
+                &reply[1],
+                mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  case AF_INET6:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
+        + sizeof (struct GNUNET_TUN_UdpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        char buf[size] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
+        struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipv6[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        msg->size = htons (size);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV6);
+        GNUNET_TUN_initialize_ipv6_header (ipv6,
+                                           IPPROTO_UDP,
+                                           sizeof (struct GNUNET_TUN_UdpHeader) + mlen,
+                                           &ts->destination_ip.v6,
+                                           &ts->source_ip.v6);
+        if (0 == ntohs (reply->source_port))
+          udp->source_port = htons (ts->destination_port);
+        else
+          udp->source_port = reply->source_port;
+        if (0 == ntohs (reply->destination_port))
+          udp->destination_port = htons (ts->source_port);
+        else
+          udp->destination_port = reply->destination_port;
+        udp->len = htons (mlen + sizeof (struct GNUNET_TUN_UdpHeader));
+        GNUNET_TUN_calculate_udp6_checksum (ipv6,
+                                            udp,
+                                            &reply[1], mlen);
+        GNUNET_memcpy (&udp[1],
+                &reply[1],
+                mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  default:
+    GNUNET_assert (0);
+  }
+  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
+                                     GNUNET_TIME_absolute_get ().abs_value_us);
+  GNUNET_CADET_receive_done (ts->channel);
+}
+
+
+/**
+ * We got a TCP packet back from the CADET channel.  Check it is OK.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param data the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_tcp_back (void *cls,
+                  const struct GNUNET_EXIT_TcpDataMessage *data)
+{
+  struct ChannelState *ts = cls;
+
+  if (NULL == ts->heap_node)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (data->tcp_header.off * 4 < sizeof (struct GNUNET_TUN_TcpHeader))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We got a TCP packet back from the CADET channel.  Pass it on to the
+ * local virtual interface via the helper.
+ *
+ * @param cls our `struct ChannelState *`
+ * @param data the actual message
+ */
+static void
+handle_tcp_back (void *cls,
+                 const struct GNUNET_EXIT_TcpDataMessage *data)
+{
+  struct ChannelState *ts = cls;
+  size_t mlen;
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# TCP packets received from cadet"),
+                            1,
+                            GNUNET_NO);
+  mlen = ntohs (data->header.size) - sizeof (struct GNUNET_EXIT_TcpDataMessage);
+  {
+    char sbuf[INET6_ADDRSTRLEN];
+    char dbuf[INET6_ADDRSTRLEN];
+
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Received TCP reply from cadet, sending %u bytes from [%s]:%u -> [%s]:%u via TUN\n",
+                (unsigned int) mlen,
+                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
+                ts->destination_port,
+                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)),
+                ts->source_port);
+  }
+  switch (ts->af)
+  {
+  case AF_INET:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
+        + sizeof (struct GNUNET_TUN_TcpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        char buf[size] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
+        struct GNUNET_TUN_TcpHeader *tcp = (struct GNUNET_TUN_TcpHeader *) &ipv4[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        msg->size = htons (size);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV4);
+        GNUNET_TUN_initialize_ipv4_header (ipv4,
+                                           IPPROTO_TCP,
+                                           sizeof (struct GNUNET_TUN_TcpHeader) + mlen,
+                                           &ts->destination_ip.v4,
+                                           &ts->source_ip.v4);
+        *tcp = data->tcp_header;
+        tcp->source_port = htons (ts->destination_port);
+        tcp->destination_port = htons (ts->source_port);
+        GNUNET_TUN_calculate_tcp4_checksum (ipv4,
+                                            tcp,
+                                            &data[1],
+                                            mlen);
+        GNUNET_memcpy (&tcp[1],
+                       &data[1],
+                       mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  case AF_INET6:
+    {
+      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
+        + sizeof (struct GNUNET_TUN_TcpHeader)
+        + sizeof (struct GNUNET_MessageHeader) +
+        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
+        mlen;
+      {
+        char buf[size] GNUNET_ALIGN;
+        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
+        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
+        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
+        struct GNUNET_TUN_TcpHeader *tcp = (struct GNUNET_TUN_TcpHeader *) &ipv6[1];
+        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
+        msg->size = htons (size);
+        tun->flags = htons (0);
+        tun->proto = htons (ETH_P_IPV6);
+        GNUNET_TUN_initialize_ipv6_header (ipv6,
+                                           IPPROTO_TCP,
+                                           sizeof (struct GNUNET_TUN_TcpHeader) + mlen,
+                                           &ts->destination_ip.v6,
+                                           &ts->source_ip.v6);
+        *tcp = data->tcp_header;
+        tcp->source_port = htons (ts->destination_port);
+        tcp->destination_port = htons (ts->source_port);
+        GNUNET_TUN_calculate_tcp6_checksum (ipv6,
+                                            tcp,
+                                            &data[1],
+                                            mlen);
+        GNUNET_memcpy (&tcp[1],
+                       &data[1],
+                       mlen);
+        (void) GNUNET_HELPER_send (helper_handle,
+                                   msg,
+                                   GNUNET_YES,
+                                   NULL, NULL);
+      }
+    }
+    break;
+  }
+  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
+                                     GNUNET_TIME_absolute_get ().abs_value_us);
+  GNUNET_CADET_receive_done (ts->channel);
+}
+
+
+/**
+ * Create a channel for @a ts to @a target at @a port
+ *
+ * @param ts channel state to create the channel for
+ * @param target peer to connect to
+ * @param port destination port
+ * @return the channel handle
+ */
+static struct GNUNET_CADET_Channel *
+create_channel (struct ChannelState *ts,
+                const struct GNUNET_PeerIdentity *target,
+                const struct GNUNET_HashCode *port)
+{
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_var_size (udp_back,
+                           GNUNET_MESSAGE_TYPE_VPN_UDP_REPLY,
+                           struct GNUNET_EXIT_UdpReplyMessage,
+                           ts),
+    GNUNET_MQ_hd_var_size (tcp_back,
+                           GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_VPN,
+                           struct GNUNET_EXIT_TcpDataMessage,
+                           ts),
+    GNUNET_MQ_hd_var_size (icmp_back,
+                           GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_VPN,
+                           struct GNUNET_EXIT_IcmpToVPNMessage,
+                           ts),
+    GNUNET_MQ_handler_end()
+  };
+
+  return GNUNET_CADET_channel_create (cadet_handle,
+                                      ts,
+                                      target,
+                                      port,
+                                      GNUNET_CADET_OPTION_DEFAULT,
+                                      NULL,
+                                      &channel_cleaner,
+                                      cadet_handlers);
+}
+
+
+/**
  * Regex has found a potential exit peer for us; consider using it.
  *
  * @param cls the `struct ChannelState`
@@ -758,11 +1444,9 @@ handle_regex_result (void *cls,
               "Creating tunnel to %s for destination %s!\n",
               GNUNET_i2s (id),
               print_channel_destination (&ts->destination));
-  ts->channel = GNUNET_CADET_channel_create (cadet_handle,
-                                             ts,
-                                             id,
-                                             &port,
-                                             GNUNET_CADET_OPTION_DEFAULT);
+  ts->channel = create_channel (ts,
+                                id,
+                                &port);
 }
 
 
@@ -795,12 +1479,10 @@ create_channel_to_destination (struct DestinationChannel *dt,
     GNUNET_TUN_compute_service_cadet_port (&ts->destination.details.service_destination.service_descriptor,
                                            ts->destination_port,
                                            &cadet_port);
-    ts->channel
-      = GNUNET_CADET_channel_create (cadet_handle,
-                                     ts,
-                                     &dt->destination->details.service_destination.target,
-                                     &cadet_port,
-                                     GNUNET_CADET_OPTION_DEFAULT);
+    ts->channel = create_channel (ts,
+                                  &dt->destination->details.service_destination.target,
+                                  &cadet_port);
+
     if (NULL == ts->channel)
     {
       GNUNET_break (0);
@@ -906,9 +1588,9 @@ route_packet (struct DestinationEntry *destination,
 {
   struct GNUNET_HashCode key;
   struct ChannelState *ts;
-  struct ChannelMessageQueueEntry *tnq;
   size_t alen;
   size_t mlen;
+  struct GNUNET_MQ_Envelope *env;
   const struct GNUNET_TUN_UdpHeader *udp;
   const struct GNUNET_TUN_TcpHeader *tcp;
   const struct GNUNET_TUN_IcmpHeader *icmp;
@@ -1157,17 +1839,14 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_UdpServiceMessage) +
         payload_length - sizeof (struct GNUNET_TUN_UdpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
       }
-      tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-      tnq->len = mlen;
-      tnq->msg = &tnq[1];
-      usm = (struct GNUNET_EXIT_UdpServiceMessage *) &tnq[1];
-      usm->header.size = htons ((uint16_t) mlen);
-      usm->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_UDP_TO_SERVICE);
+      env = GNUNET_MQ_msg_extra (usm,
+                                 payload_length - sizeof (struct GNUNET_TUN_UdpHeader),
+                                 GNUNET_MESSAGE_TYPE_VPN_UDP_TO_SERVICE);
       /* if the source port is below 32000, we assume it has a special
          meaning; if not, we pick a random port (this is a heuristic) */
       usm->source_port = (ntohs (udp->source_port) < 32000) ? udp->source_port : 0;
@@ -1185,17 +1864,14 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_UdpInternetMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_UdpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
       }
-      tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-      tnq->len = mlen;
-      tnq->msg = &tnq[1];
-      uim = (struct GNUNET_EXIT_UdpInternetMessage *) &tnq[1];
-      uim->header.size = htons ((uint16_t) mlen);
-      uim->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_UDP_TO_INTERNET);
+      env = GNUNET_MQ_msg_extra (uim,
+                                 payload_length + alen - sizeof (struct GNUNET_TUN_UdpHeader),
+                                 GNUNET_MESSAGE_TYPE_VPN_UDP_TO_INTERNET);
       uim->af = htonl (destination->details.exit_destination.af);
       uim->source_port = (ntohs (udp->source_port) < 32000) ? udp->source_port : 0;
       uim->destination_port = udp->destination_port;
@@ -1215,8 +1891,8 @@ route_packet (struct DestinationEntry *destination,
         GNUNET_assert (0);
       }
       GNUNET_memcpy (payload,
-              &udp[1],
-              payload_length - sizeof (struct GNUNET_TUN_UdpHeader));
+                     &udp[1],
+                     payload_length - sizeof (struct GNUNET_TUN_UdpHeader));
     }
     break;
   case IPPROTO_TCP:
@@ -1228,17 +1904,14 @@ route_packet (struct DestinationEntry *destination,
 
         mlen = sizeof (struct GNUNET_EXIT_TcpServiceStartMessage) +
           payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-        if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
         {
           GNUNET_break (0);
           return;
         }
-        tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-        tnq->len = mlen;
-        tnq->msg = &tnq[1];
-        tsm = (struct  GNUNET_EXIT_TcpServiceStartMessage *) &tnq[1];
-        tsm->header.size = htons ((uint16_t) mlen);
-        tsm->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_TCP_TO_SERVICE_START);
+        env = GNUNET_MQ_msg_extra (tsm,
+                                   payload_length - sizeof (struct GNUNET_TUN_TcpHeader),
+                                   GNUNET_MESSAGE_TYPE_VPN_TCP_TO_SERVICE_START);
         tsm->reserved = htonl (0);
         tsm->tcp_header = *tcp;
         GNUNET_memcpy (&tsm[1],
@@ -1254,17 +1927,14 @@ route_packet (struct DestinationEntry *destination,
 
         mlen = sizeof (struct GNUNET_EXIT_TcpInternetStartMessage) +
           alen + payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-        if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
         {
           GNUNET_break (0);
           return;
         }
-        tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-        tnq->len = mlen;
-        tnq->msg = &tnq[1];
-        tim = (struct  GNUNET_EXIT_TcpInternetStartMessage *) &tnq[1];
-        tim->header.size = htons ((uint16_t) mlen);
-        tim->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_TCP_TO_INTERNET_START);
+        env = GNUNET_MQ_msg_extra (tim,
+                                   payload_length + alen - sizeof (struct GNUNET_TUN_TcpHeader),
+                                   GNUNET_MESSAGE_TYPE_VPN_TCP_TO_INTERNET_START);
         tim->af = htonl (destination->details.exit_destination.af);
         tim->tcp_header = *tcp;
         switch (destination->details.exit_destination.af)
@@ -1283,8 +1953,8 @@ route_packet (struct DestinationEntry *destination,
           GNUNET_assert (0);
         }
         GNUNET_memcpy (payload,
-                &tcp[1],
-                payload_length - sizeof (struct GNUNET_TUN_TcpHeader));
+                       &tcp[1],
+                       payload_length - sizeof (struct GNUNET_TUN_TcpHeader));
       }
     }
     else
@@ -1293,22 +1963,19 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_TcpDataMessage) +
         payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
       }
-      tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-      tnq->len = mlen;
-      tnq->msg = &tnq[1];
-      tdm = (struct  GNUNET_EXIT_TcpDataMessage *) &tnq[1];
-      tdm->header.size = htons ((uint16_t) mlen);
-      tdm->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_EXIT);
+      env = GNUNET_MQ_msg_extra (tdm,
+                                 payload_length - sizeof (struct GNUNET_TUN_TcpHeader),
+                                 GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_EXIT);
       tdm->reserved = htonl (0);
       tdm->tcp_header = *tcp;
       GNUNET_memcpy (&tdm[1],
-              &tcp[1],
-              payload_length - sizeof (struct GNUNET_TUN_TcpHeader));
+                     &tcp[1],
+                     payload_length - sizeof (struct GNUNET_TUN_TcpHeader));
      }
     break;
   case IPPROTO_ICMP:
@@ -1317,19 +1984,6 @@ route_packet (struct DestinationEntry *destination,
     {
       struct GNUNET_EXIT_IcmpServiceMessage *ism;
 
-      mlen = sizeof (struct GNUNET_EXIT_IcmpServiceMessage) +
-        payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-      {
-        GNUNET_break (0);
-        return;
-      }
-      tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-      tnq->msg = &tnq[1];
-      ism = (struct GNUNET_EXIT_IcmpServiceMessage *) &tnq[1];
-      ism->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_SERVICE);
-      ism->af = htonl (af); /* need to tell destination ICMP protocol family! */
-      ism->icmp_header = *icmp;
       /* ICMP protocol translation will be done by the receiver (as we don't know
          the target AF); however, we still need to possibly discard the payload
          depending on the ICMP type */
@@ -1384,12 +2038,20 @@ route_packet (struct DestinationEntry *destination,
       /* update length calculations, as payload_length may have changed */
       mlen = sizeof (struct GNUNET_EXIT_IcmpServiceMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      tnq->len = mlen;
-      ism->header.size = htons ((uint16_t) mlen);
-      /* finally, copy payload (if there is any left...) */
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
+      {
+        GNUNET_break (0);
+        return;
+      }
+
+      env = GNUNET_MQ_msg_extra (ism,
+                                 payload_length - sizeof (struct GNUNET_TUN_IcmpHeader),
+                                 GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_SERVICE);
+      ism->af = htonl (af); /* need to tell destination ICMP protocol family! */
+      ism->icmp_header = *icmp;
       GNUNET_memcpy (&ism[1],
-              &icmp[1],
-              payload_length - sizeof (struct GNUNET_TUN_IcmpHeader));
+                     &icmp[1],
+                     payload_length - sizeof (struct GNUNET_TUN_IcmpHeader));
     }
     else
     {
@@ -1397,19 +2059,9 @@ route_packet (struct DestinationEntry *destination,
       struct in_addr *ip4dst;
       struct in6_addr *ip6dst;
       void *payload;
+      uint8_t new_type;
 
-      mlen = sizeof (struct GNUNET_EXIT_IcmpInternetMessage) +
-        alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
-      {
-        GNUNET_break (0);
-        return;
-      }
-      tnq = GNUNET_malloc (sizeof (struct ChannelMessageQueueEntry) + mlen);
-      tnq->msg = &tnq[1];
-      iim = (struct GNUNET_EXIT_IcmpInternetMessage *) &tnq[1];
-      iim->header.type = htons (GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_INTERNET);
-      iim->icmp_header = *icmp;
+      new_type = icmp->type;
       /* Perform ICMP protocol-translation (depending on destination AF and source AF)
          and throw away ICMP payload depending on ICMP message type */
       switch (af)
@@ -1419,21 +2071,21 @@ route_packet (struct DestinationEntry *destination,
         {
         case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
           if (destination->details.exit_destination.af == AF_INET6)
-            iim->icmp_header.type = GNUNET_TUN_ICMPTYPE6_ECHO_REPLY;
+            new_type = GNUNET_TUN_ICMPTYPE6_ECHO_REPLY;
           break;
         case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
           if (destination->details.exit_destination.af == AF_INET6)
-            iim->icmp_header.type = GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST;
+            new_type = GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST;
           break;
         case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
           if (destination->details.exit_destination.af == AF_INET6)
-            iim->icmp_header.type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
+            new_type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
           /* throw away IP-payload, exit will have to make it up anyway */
           payload_length = sizeof (struct GNUNET_TUN_IcmpHeader);
           break;
         case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
           if (destination->details.exit_destination.af == AF_INET6)
-            iim->icmp_header.type = GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED;
+            new_type = GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED;
           /* throw away IP-payload, exit will have to make it up anyway */
           payload_length = sizeof (struct GNUNET_TUN_IcmpHeader);
           break;
@@ -1443,7 +2095,6 @@ route_packet (struct DestinationEntry *destination,
               GNUNET_STATISTICS_update (stats,
                                         gettext_noop ("# ICMPv4 packets dropped (impossible PT to v6)"),
                                         1, GNUNET_NO);
-              GNUNET_free (tnq);
               return;
             }
           /* throw away IP-payload, exit will have to make it up anyway */
@@ -1453,7 +2104,6 @@ route_packet (struct DestinationEntry *destination,
           GNUNET_STATISTICS_update (stats,
                                     gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
                                     1, GNUNET_NO);
-          GNUNET_free (tnq);
           return;
         }
         /* end of AF_INET */
@@ -1462,14 +2112,14 @@ route_packet (struct DestinationEntry *destination,
         switch (icmp->type)
           {
           case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
-            if (destination->details.exit_destination.af == AF_INET6)
-              iim->icmp_header.type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
+            if (destination->details.exit_destination.af == AF_INET)
+              new_type = GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE;
             /* throw away IP-payload, exit will have to make it up anyway */
             payload_length = sizeof (struct GNUNET_TUN_IcmpHeader);
             break;
           case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
             if (destination->details.exit_destination.af == AF_INET)
-              iim->icmp_header.type = GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED;
+              new_type = GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED;
             /* throw away IP-payload, exit will have to make it up anyway */
             payload_length = sizeof (struct GNUNET_TUN_IcmpHeader);
             break;
@@ -1479,7 +2129,6 @@ route_packet (struct DestinationEntry *destination,
               GNUNET_STATISTICS_update (stats,
                                         gettext_noop ("# ICMPv6 packets dropped (impossible PT to v4)"),
                                         1, GNUNET_NO);
-              GNUNET_free (tnq);
               return;
             }
             /* throw away IP-payload, exit will have to make it up anyway */
@@ -1491,7 +2140,6 @@ route_packet (struct DestinationEntry *destination,
               GNUNET_STATISTICS_update (stats,
                                         gettext_noop ("# ICMPv6 packets dropped (impossible PT to v4)"),
                                         1, GNUNET_NO);
-              GNUNET_free (tnq);
               return;
             }
             /* throw away IP-payload, exit will have to make it up anyway */
@@ -1499,17 +2147,16 @@ route_packet (struct DestinationEntry *destination,
             break;
           case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
             if (destination->details.exit_destination.af == AF_INET)
-              iim->icmp_header.type = GNUNET_TUN_ICMPTYPE_ECHO_REQUEST;
+              new_type = GNUNET_TUN_ICMPTYPE_ECHO_REQUEST;
             break;
           case GNUNET_TUN_ICMPTYPE6_ECHO_REPLY:
             if (destination->details.exit_destination.af == AF_INET)
-              iim->icmp_header.type = GNUNET_TUN_ICMPTYPE_ECHO_REPLY;
+              new_type = GNUNET_TUN_ICMPTYPE_ECHO_REPLY;
             break;
           default:
             GNUNET_STATISTICS_update (stats,
                                       gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
                                       1, GNUNET_NO);
-            GNUNET_free (tnq);
             return;
           }
         /* end of AF_INET6 */
@@ -1517,13 +2164,20 @@ route_packet (struct DestinationEntry *destination,
       default:
         GNUNET_assert (0);
       }
+
       /* update length calculations, as payload_length may have changed */
       mlen = sizeof (struct GNUNET_EXIT_IcmpInternetMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      tnq->len = mlen;
-      iim->header.size = htons ((uint16_t) mlen);
-
-      /* need to tell destination ICMP protocol family! */
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
+      {
+        GNUNET_break (0);
+        return;
+      }
+      env = GNUNET_MQ_msg_extra (iim,
+                                 alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader),
+                                 GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_INTERNET);
+      iim->icmp_header = *icmp;
+      iim->icmp_header.type = new_type;
       iim->af = htonl (destination->details.exit_destination.af);
       switch (destination->details.exit_destination.af)
       {
@@ -1541,8 +2195,8 @@ route_packet (struct DestinationEntry *destination,
         GNUNET_assert (0);
       }
       GNUNET_memcpy (payload,
-              &icmp[1],
-              payload_length - sizeof (struct GNUNET_TUN_IcmpHeader));
+                     &icmp[1],
+                     payload_length - sizeof (struct GNUNET_TUN_IcmpHeader));
     }
     break;
   default:
@@ -1551,7 +2205,8 @@ route_packet (struct DestinationEntry *destination,
     break;
   }
   ts->is_established = GNUNET_YES;
-  send_to_channel (tnq, ts);
+  send_to_channel (ts,
+                   env);
 }
 
 
@@ -1562,12 +2217,10 @@ route_packet (struct DestinationEntry *destination,
  * and forward the packet.
  *
  * @param cls closure, NULL
- * @param client NULL
  * @param message message we got from the client (VPN channel interface)
  */
 static int
 message_token (void *cls,
-               void *client,
                const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TUN_Layer2PacketHeader *tun;
@@ -1678,694 +2331,6 @@ message_token (void *cls,
 
 
 /**
- * Synthesize a plausible ICMP payload for an ICMP error
- * response on the given channel.
- *
- * @param ts channel information
- * @param ipp IPv4 header to fill in (ICMP payload)
- * @param udp "UDP" header to fill in (ICMP payload); might actually
- *            also be the first 8 bytes of the TCP header
- */
-static void
-make_up_icmpv4_payload (struct ChannelState *ts,
-                        struct GNUNET_TUN_IPv4Header *ipp,
-                        struct GNUNET_TUN_UdpHeader *udp)
-{
-  GNUNET_TUN_initialize_ipv4_header (ipp,
-                                     ts->protocol,
-                                     sizeof (struct GNUNET_TUN_TcpHeader),
-                                     &ts->source_ip.v4,
-                                     &ts->destination_ip.v4);
-  udp->source_port = htons (ts->source_port);
-  udp->destination_port = htons (ts->destination_port);
-  udp->len = htons (0);
-  udp->crc = htons (0);
-}
-
-
-/**
- * Synthesize a plausible ICMP payload for an ICMP error
- * response on the given channel.
- *
- * @param ts channel information
- * @param ipp IPv6 header to fill in (ICMP payload)
- * @param udp "UDP" header to fill in (ICMP payload); might actually
- *            also be the first 8 bytes of the TCP header
- */
-static void
-make_up_icmpv6_payload (struct ChannelState *ts,
-                        struct GNUNET_TUN_IPv6Header *ipp,
-                        struct GNUNET_TUN_UdpHeader *udp)
-{
-  GNUNET_TUN_initialize_ipv6_header (ipp,
-                                     ts->protocol,
-                                     sizeof (struct GNUNET_TUN_TcpHeader),
-                                     &ts->source_ip.v6,
-                                     &ts->destination_ip.v6);
-  udp->source_port = htons (ts->source_port);
-  udp->destination_port = htons (ts->destination_port);
-  udp->len = htons (0);
-  udp->crc = htons (0);
-}
-
-
-/**
- * We got an ICMP packet back from the CADET channel.  Pass it on to the
- * local virtual interface via the helper.
- *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-receive_icmp_back (void *cls,
-                   struct GNUNET_CADET_Channel *channel,
-                   void **channel_ctx,
-                   const struct GNUNET_MessageHeader *message)
-{
-  struct ChannelState *ts = *channel_ctx;
-  const struct GNUNET_EXIT_IcmpToVPNMessage *i2v;
-  size_t mlen;
-
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# ICMP packets received from cadet"),
-                            1, GNUNET_NO);
-  mlen = ntohs (message->size);
-  if (mlen < sizeof (struct GNUNET_EXIT_IcmpToVPNMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL == ts->heap_node)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (AF_UNSPEC == ts->af)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  i2v = (const struct GNUNET_EXIT_IcmpToVPNMessage *) message;
-  mlen -= sizeof (struct GNUNET_EXIT_IcmpToVPNMessage);
-  {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received ICMP packet from cadet, sending %u bytes from %s -> %s via TUN\n",
-                (unsigned int) mlen,
-                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
-                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)));
-  }
-  switch (ts->af)
-  {
-  case AF_INET:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
-        + sizeof (struct GNUNET_TUN_IcmpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        /* reserve some extra space in case we have an ICMP type here where
-           we will need to make up the payload ourselves */
-        char buf[size + sizeof (struct GNUNET_TUN_IPv4Header) + 8] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
-        struct GNUNET_TUN_IcmpHeader *icmp = (struct GNUNET_TUN_IcmpHeader *) &ipv4[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV4);
-        GNUNET_TUN_initialize_ipv4_header (ipv4,
-                                           IPPROTO_ICMP,
-                                           sizeof (struct GNUNET_TUN_IcmpHeader) + mlen,
-                                           &ts->destination_ip.v4,
-                                           &ts->source_ip.v4);
-        *icmp = i2v->icmp_header;
-        GNUNET_memcpy (&icmp[1],
-                &i2v[1],
-                mlen);
-        /* For some ICMP types, we need to adjust (make up) the payload here.
-           Also, depending on the AF used on the other side, we have to
-           do ICMP PT (translate ICMP types) */
-        switch (ntohl (i2v->af))
-        {
-        case AF_INET:
-          switch (icmp->type)
-          {
-          case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
-          case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
-            break;
-          case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
-          case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
-          case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
-            {
-              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv4_payload (ts, ipp, udp);
-            }
-            break;
-          default:
-            GNUNET_break_op (0);
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
-                                      1, GNUNET_NO);
-            return GNUNET_SYSERR;
-          }
-          /* end AF_INET */
-          break;
-        case AF_INET6:
-          /* ICMP PT 6-to-4 and possibly making up payloads */
-          switch (icmp->type)
-          {
-          case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
-            icmp->type = GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE;
-            {
-              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv4_payload (ts, ipp, udp);
-            }
-            break;
-          case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
-            icmp->type = GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED;
-            {
-              struct GNUNET_TUN_IPv4Header *ipp = (struct GNUNET_TUN_IPv4Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv4Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv4_payload (ts, ipp, udp);
-            }
-            break;
-          case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
-          case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv6 packets dropped (impossible PT to v4)"),
-                                      1, GNUNET_NO);
-            return GNUNET_OK;
-          case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
-            icmp->type = GNUNET_TUN_ICMPTYPE_ECHO_REQUEST;
-            break;
-          case GNUNET_TUN_ICMPTYPE6_ECHO_REPLY:
-            icmp->type = GNUNET_TUN_ICMPTYPE_ECHO_REPLY;
-            break;
-          default:
-            GNUNET_break_op (0);
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
-                                      1, GNUNET_NO);
-            return GNUNET_SYSERR;
-          }
-          /* end AF_INET6 */
-          break;
-        default:
-          GNUNET_break_op (0);
-          return GNUNET_SYSERR;
-        }
-        msg->size = htons (size);
-        GNUNET_TUN_calculate_icmp_checksum (icmp,
-                                            &i2v[1],
-                                            mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  case AF_INET6:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
-        + sizeof (struct GNUNET_TUN_IcmpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        char buf[size + sizeof (struct GNUNET_TUN_IPv6Header) + 8] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
-        struct GNUNET_TUN_IcmpHeader *icmp = (struct GNUNET_TUN_IcmpHeader *) &ipv6[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV6);
-        GNUNET_TUN_initialize_ipv6_header (ipv6,
-                                           IPPROTO_ICMPV6,
-                                           sizeof (struct GNUNET_TUN_IcmpHeader) + mlen,
-                                           &ts->destination_ip.v6,
-                                           &ts->source_ip.v6);
-        *icmp = i2v->icmp_header;
-        GNUNET_memcpy (&icmp[1],
-                &i2v[1],
-                mlen);
-
-        /* For some ICMP types, we need to adjust (make up) the payload here.
-           Also, depending on the AF used on the other side, we have to
-           do ICMP PT (translate ICMP types) */
-        switch (ntohl (i2v->af))
-        {
-        case AF_INET:
-          /* ICMP PT 4-to-6 and possibly making up payloads */
-          switch (icmp->type)
-          {
-          case GNUNET_TUN_ICMPTYPE_ECHO_REPLY:
-            icmp->type = GNUNET_TUN_ICMPTYPE6_ECHO_REPLY;
-            break;
-          case GNUNET_TUN_ICMPTYPE_ECHO_REQUEST:
-            icmp->type = GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST;
-            break;
-          case GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE:
-            icmp->type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
-            {
-              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv6_payload (ts, ipp, udp);
-            }
-            break;
-          case GNUNET_TUN_ICMPTYPE_TIME_EXCEEDED:
-            icmp->type = GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED;
-            {
-              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv6_payload (ts, ipp, udp);
-            }
-            break;
-          case GNUNET_TUN_ICMPTYPE_SOURCE_QUENCH:
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv4 packets dropped (impossible PT to v6)"),
-                                      1, GNUNET_NO);
-            return GNUNET_OK;
-          default:
-            GNUNET_break_op (0);
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv4 packets dropped (type not allowed)"),
-                                      1, GNUNET_NO);
-            return GNUNET_SYSERR;
-          }
-          /* end AF_INET */
-          break;
-        case AF_INET6:
-          switch (icmp->type)
-          {
-          case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
-          case GNUNET_TUN_ICMPTYPE6_TIME_EXCEEDED:
-          case GNUNET_TUN_ICMPTYPE6_PACKET_TOO_BIG:
-          case GNUNET_TUN_ICMPTYPE6_PARAMETER_PROBLEM:
-            {
-              struct GNUNET_TUN_IPv6Header *ipp = (struct GNUNET_TUN_IPv6Header *) &icmp[1];
-              struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipp[1];
-
-              if (mlen != 0)
-                {
-                  /* sender did not strip ICMP payload? */
-                  GNUNET_break_op (0);
-                  return GNUNET_SYSERR;
-                }
-              size += sizeof (struct GNUNET_TUN_IPv6Header) + 8;
-              GNUNET_assert (8 == sizeof (struct GNUNET_TUN_UdpHeader));
-              make_up_icmpv6_payload (ts, ipp, udp);
-            }
-            break;
-          case GNUNET_TUN_ICMPTYPE6_ECHO_REQUEST:
-            break;
-          default:
-            GNUNET_break_op (0);
-            GNUNET_STATISTICS_update (stats,
-                                      gettext_noop ("# ICMPv6 packets dropped (type not allowed)"),
-                                      1, GNUNET_NO);
-            return GNUNET_SYSERR;
-          }
-          /* end AF_INET6 */
-          break;
-        default:
-          GNUNET_break_op (0);
-          return GNUNET_SYSERR;
-        }
-        msg->size = htons (size);
-        GNUNET_TUN_calculate_icmp_checksum (icmp,
-                                            &i2v[1], mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
-                                     GNUNET_TIME_absolute_get ().abs_value_us);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
-}
-
-
-/**
- * We got a UDP packet back from the CADET channel.  Pass it on to the
- * local virtual interface via the helper.
- *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our 'struct ChannelState *'
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-receive_udp_back (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct ChannelState *ts = *channel_ctx;
-  const struct GNUNET_EXIT_UdpReplyMessage *reply;
-  size_t mlen;
-
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# UDP packets received from cadet"),
-                            1, GNUNET_NO);
-  mlen = ntohs (message->size);
-  if (mlen < sizeof (struct GNUNET_EXIT_UdpReplyMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL == ts->heap_node)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (AF_UNSPEC == ts->af)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  reply = (const struct GNUNET_EXIT_UdpReplyMessage *) message;
-  mlen -= sizeof (struct GNUNET_EXIT_UdpReplyMessage);
-  {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received UDP reply from cadet, sending %u bytes from [%s]:%u -> [%s]:%u via TUN\n",
-                (unsigned int) mlen,
-                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
-                ts->destination_port,
-                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)),
-                ts->source_port);
-  }
-  switch (ts->af)
-  {
-  case AF_INET:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
-        + sizeof (struct GNUNET_TUN_UdpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        char buf[size] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
-        struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipv4[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        msg->size = htons (size);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV4);
-        GNUNET_TUN_initialize_ipv4_header (ipv4,
-                                           IPPROTO_UDP,
-                                           sizeof (struct GNUNET_TUN_UdpHeader) + mlen,
-                                           &ts->destination_ip.v4,
-                                           &ts->source_ip.v4);
-        if (0 == ntohs (reply->source_port))
-          udp->source_port = htons (ts->destination_port);
-        else
-          udp->source_port = reply->source_port;
-        if (0 == ntohs (reply->destination_port))
-          udp->destination_port = htons (ts->source_port);
-        else
-          udp->destination_port = reply->destination_port;
-        udp->len = htons (mlen + sizeof (struct GNUNET_TUN_UdpHeader));
-        GNUNET_TUN_calculate_udp4_checksum (ipv4,
-                                            udp,
-                                            &reply[1],
-                                            mlen);
-        GNUNET_memcpy (&udp[1],
-                &reply[1],
-                mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  case AF_INET6:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
-        + sizeof (struct GNUNET_TUN_UdpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        char buf[size] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
-        struct GNUNET_TUN_UdpHeader *udp = (struct GNUNET_TUN_UdpHeader *) &ipv6[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        msg->size = htons (size);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV6);
-        GNUNET_TUN_initialize_ipv6_header (ipv6,
-                                           IPPROTO_UDP,
-                                           sizeof (struct GNUNET_TUN_UdpHeader) + mlen,
-                                           &ts->destination_ip.v6,
-                                           &ts->source_ip.v6);
-        if (0 == ntohs (reply->source_port))
-          udp->source_port = htons (ts->destination_port);
-        else
-          udp->source_port = reply->source_port;
-        if (0 == ntohs (reply->destination_port))
-          udp->destination_port = htons (ts->source_port);
-        else
-          udp->destination_port = reply->destination_port;
-        udp->len = htons (mlen + sizeof (struct GNUNET_TUN_UdpHeader));
-        GNUNET_TUN_calculate_udp6_checksum (ipv6,
-                                            udp,
-                                            &reply[1], mlen);
-        GNUNET_memcpy (&udp[1],
-                &reply[1],
-                mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
-                                     GNUNET_TIME_absolute_get ().abs_value_us);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
-}
-
-
-/**
- * We got a TCP packet back from the CADET channel.  Pass it on to the
- * local virtual interface via the helper.
- *
- * @param cls closure, NULL
- * @param channel connection to the other end
- * @param channel_ctx pointer to our `struct ChannelState *`
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-receive_tcp_back (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct ChannelState *ts = *channel_ctx;
-  const struct GNUNET_EXIT_TcpDataMessage *data;
-  size_t mlen;
-
-  GNUNET_STATISTICS_update (stats,
-                            gettext_noop ("# TCP packets received from cadet"),
-                            1, GNUNET_NO);
-  mlen = ntohs (message->size);
-  if (mlen < sizeof (struct GNUNET_EXIT_TcpDataMessage))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  if (NULL == ts->heap_node)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  data = (const struct GNUNET_EXIT_TcpDataMessage *) message;
-  mlen -= sizeof (struct GNUNET_EXIT_TcpDataMessage);
-  {
-    char sbuf[INET6_ADDRSTRLEN];
-    char dbuf[INET6_ADDRSTRLEN];
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Received TCP reply from cadet, sending %u bytes from [%s]:%u -> [%s]:%u via TUN\n",
-                (unsigned int) mlen,
-                inet_ntop (ts->af, &ts->destination_ip, sbuf, sizeof (sbuf)),
-                ts->destination_port,
-                inet_ntop (ts->af, &ts->source_ip, dbuf, sizeof (dbuf)),
-                ts->source_port);
-  }
-  if (data->tcp_header.off * 4 < sizeof (struct GNUNET_TUN_TcpHeader))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  switch (ts->af)
-  {
-  case AF_INET:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv4Header)
-        + sizeof (struct GNUNET_TUN_TcpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        char buf[size] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv4Header *ipv4 = (struct GNUNET_TUN_IPv4Header *) &tun[1];
-        struct GNUNET_TUN_TcpHeader *tcp = (struct GNUNET_TUN_TcpHeader *) &ipv4[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        msg->size = htons (size);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV4);
-        GNUNET_TUN_initialize_ipv4_header (ipv4,
-                                           IPPROTO_TCP,
-                                           sizeof (struct GNUNET_TUN_TcpHeader) + mlen,
-                                           &ts->destination_ip.v4,
-                                           &ts->source_ip.v4);
-        *tcp = data->tcp_header;
-        tcp->source_port = htons (ts->destination_port);
-        tcp->destination_port = htons (ts->source_port);
-        GNUNET_TUN_calculate_tcp4_checksum (ipv4,
-                                            tcp,
-                                            &data[1],
-                                            mlen);
-        GNUNET_memcpy (&tcp[1],
-                &data[1],
-                mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  case AF_INET6:
-    {
-      size_t size = sizeof (struct GNUNET_TUN_IPv6Header)
-        + sizeof (struct GNUNET_TUN_TcpHeader)
-        + sizeof (struct GNUNET_MessageHeader) +
-        sizeof (struct GNUNET_TUN_Layer2PacketHeader) +
-        mlen;
-      {
-        char buf[size] GNUNET_ALIGN;
-        struct GNUNET_MessageHeader *msg = (struct GNUNET_MessageHeader *) buf;
-        struct GNUNET_TUN_Layer2PacketHeader *tun = (struct GNUNET_TUN_Layer2PacketHeader*) &msg[1];
-        struct GNUNET_TUN_IPv6Header *ipv6 = (struct GNUNET_TUN_IPv6Header *) &tun[1];
-        struct GNUNET_TUN_TcpHeader *tcp = (struct GNUNET_TUN_TcpHeader *) &ipv6[1];
-        msg->type = htons (GNUNET_MESSAGE_TYPE_VPN_HELPER);
-        msg->size = htons (size);
-        tun->flags = htons (0);
-        tun->proto = htons (ETH_P_IPV6);
-        GNUNET_TUN_initialize_ipv6_header (ipv6,
-                                           IPPROTO_TCP,
-                                           sizeof (struct GNUNET_TUN_TcpHeader) + mlen,
-                                           &ts->destination_ip.v6,
-                                           &ts->source_ip.v6);
-        *tcp = data->tcp_header;
-        tcp->source_port = htons (ts->destination_port);
-        tcp->destination_port = htons (ts->source_port);
-        GNUNET_TUN_calculate_tcp6_checksum (ipv6,
-                                            tcp,
-                                            &data[1],
-                                            mlen);
-        GNUNET_memcpy (&tcp[1],
-                &data[1],
-                mlen);
-        (void) GNUNET_HELPER_send (helper_handle,
-                                   msg,
-                                   GNUNET_YES,
-                                   NULL, NULL);
-      }
-    }
-    break;
-  }
-  GNUNET_CONTAINER_heap_update_cost (ts->heap_node,
-                                     GNUNET_TIME_absolute_get ().abs_value_us);
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
-}
-
-
-/**
  * Allocate an IPv4 address from the range of the channel
  * for a new redirection.
  *
@@ -2813,29 +2778,6 @@ handle_client_redirect_to_service (void *cls,
 }
 
 
-/**
- * Function called whenever a channel is destroyed.  Should clean up
- * any associated state.
- *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored (our `struct ChannelState`)
- */
-static void
-channel_cleaner (void *cls,
-                 const struct GNUNET_CADET_Channel *channel,
-                 void *channel_ctx)
-{
-  struct ChannelState *ts = channel_ctx;
-
-  ts->channel = NULL; /* we must not call GNUNET_CADET_channel_destroy() anymore */
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "CADET notified us about death of channel to `%s'\n",
-              print_channel_destination (&ts->destination));
-  free_channel_state (ts);
-}
-
 
 /**
  * Free memory occupied by an entry in the destination map.
@@ -2984,12 +2926,6 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *cfg_,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    { &receive_udp_back, GNUNET_MESSAGE_TYPE_VPN_UDP_REPLY, 0},
-    { &receive_tcp_back, GNUNET_MESSAGE_TYPE_VPN_TCP_DATA_TO_VPN, 0},
-    { &receive_icmp_back, GNUNET_MESSAGE_TYPE_VPN_ICMP_TO_VPN, 0},
-    {NULL, 0, 0}
-  };
   char *ifname;
   char *ipv6addr;
   char *ipv6prefix_s;
@@ -3130,11 +3066,7 @@ run (void *cls,
   }
   vpn_argv[6] = NULL;
 
-  cadet_handle
-    = GNUNET_CADET_connect (cfg_,
-                            NULL,
-                            &channel_cleaner,
-                            cadet_handlers);
+  cadet_handle = GNUNET_CADET_connect (cfg_);
     // FIXME never opens ports???
   helper_handle = GNUNET_HELPER_start (GNUNET_NO,
                                        "gnunet-helper-vpn", vpn_argv,
diff --git a/src/vpn/gnunet-vpn.c b/src/vpn/gnunet-vpn.c
index 2e7daf7f7..34c545339 100644
--- a/src/vpn/gnunet-vpn.c
+++ b/src/vpn/gnunet-vpn.c
@@ -78,7 +78,7 @@ static int udp;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 /**
  * Global return value.
@@ -286,33 +286,53 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'4', "ipv4", NULL,
-     gettext_noop ("request that result should be an IPv4 address"),
-     0, &GNUNET_GETOPT_set_one, &ipv4},
-    {'6', "ipv6", NULL,
-     gettext_noop ("request that result should be an IPv6 address"),
-     0, &GNUNET_GETOPT_set_one, &ipv6},
-    {'d', "duration", "TIME",
-     gettext_noop ("how long should the mapping be valid for new tunnels?"),
-     1, &GNUNET_GETOPT_set_relative_time, &duration},
-    {'i', "ip", "IP",
-     gettext_noop ("destination IP for the tunnel"),
-     1, &GNUNET_GETOPT_set_string, &target_ip},
-    {'p', "peer", "PEERID",
-     gettext_noop ("peer offering the service we would like to access"),
-     1, &GNUNET_GETOPT_set_string, &peer_id},
-    {'s', "service", "NAME",
-     gettext_noop ("name of the service we would like to access"),
-     1, &GNUNET_GETOPT_set_string, &service_name},
-    {'t', "tcp", NULL,
-     gettext_noop ("service is offered via TCP"),
-     0, &GNUNET_GETOPT_set_one, &tcp},
-    {'u', "udp", NULL,
-     gettext_noop ("service is offered via UDP"),
-     0, &GNUNET_GETOPT_set_one, &udp},
-
-    GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_flag ('4',
+                                  "ipv4",
+                                  gettext_noop ("request that result should be an IPv4 address"),
+                                  &ipv4),
+
+    GNUNET_GETOPT_option_flag ('6',
+                                  "ipv6",
+                                  gettext_noop ("request that result should be an IPv6 address"),
+                                  &ipv6),
+
+    GNUNET_GETOPT_option_relative_time ('d',
+                                            "duration",
+                                            "TIME",
+                                            gettext_noop ("how long should the mapping be valid for new tunnels?"),
+                                            &duration),
+
+    GNUNET_GETOPT_option_string ('i',
+                                 "ip",
+                                 "IP",
+                                 gettext_noop ("destination IP for the tunnel"),
+                                 &target_ip),
+
+    GNUNET_GETOPT_option_string ('p',
+                                 "peer",
+                                 "PEERID",
+                                 gettext_noop ("peer offering the service we would like to access"),
+                                 &peer_id),
+
+    GNUNET_GETOPT_option_string ('s', 
+                                 "service",
+                                 "NAME",
+                                 gettext_noop ("name of the service we would like to access"),
+                                 &service_name),
+
+    GNUNET_GETOPT_option_flag ('t',
+                                  "tcp",
+                                  gettext_noop ("service is offered via TCP"),
+                                  &tcp),
+
+    GNUNET_GETOPT_option_flag ('u',
+                                  "udp",
+                                  gettext_noop ("service is offered via UDP"),
+                                  &udp),
+
+    GNUNET_GETOPT_option_verbose (&verbosity),
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))