40 files changed, 18852 insertions, 0 deletions

diff --git a/src/service/cadet/.gitignore b/src/service/cadet/.gitignore
new file mode 100644
index 000000000..bed27a3d3
--- /dev/null
+++ b/src/service/cadet/.gitignore
@@ -0,0 +1,2 @@
+gnunet-service-cadet
+test_cadet_local_mq
diff --git a/src/service/cadet/Makefile.am b/src/service/cadet/Makefile.am
new file mode 100644
index 000000000..0046ffc70
--- /dev/null
+++ b/src/service/cadet/Makefile.am
@@ -0,0 +1,81 @@
+# This Makefile.am is in the public domain
+AM_CPPFLAGS = -I$(top_srcdir)/src/include
+
+if USE_COVERAGE
+  AM_CFLAGS = --coverage -O0
+  XLIB = -lgcov
+endif
+
+pkgcfgdir= $(pkgdatadir)/config.d/
+
+libexecdir= $(pkglibdir)/libexec/
+
+pkgcfg_DATA = \
+  cadet.conf
+
+plugindir = $(libdir)/gnunet
+
+AM_CLFAGS = -g
+
+libexec_PROGRAMS = \
+ gnunet-service-cadet
+
+lib_LTLIBRARIES = \
+  libgnunetcadet.la
+
+libgnunetcadet_la_SOURCES = \
+  cadet_api.c \
+  cadet_api_drop_message.c \
+  cadet_api_get_channel.c \
+  cadet_api_get_path.c \
+  cadet_api_list_peers.c \
+  cadet_api_list_tunnels.c \
+  cadet_api_helper.c
+libgnunetcadet_la_LIBADD = \
+  $(top_builddir)/src/lib/util/libgnunetutil.la \
+  $(XLIB) \
+  $(LTLIBINTL)
+libgnunetcadet_la_LDFLAGS = \
+  $(GN_LIB_LDFLAGS)  \
+  -version-info 7:0:0
+
+gnunet_service_cadet_SOURCES = \
+ gnunet-service-cadet.c gnunet-service-cadet.h \
+ gnunet-service-cadet_channel.c gnunet-service-cadet_channel.h \
+ gnunet-service-cadet_connection.c gnunet-service-cadet_connection.h \
+ gnunet-service-cadet_core.c gnunet-service-cadet_core.h \
+ gnunet-service-cadet_dht.c gnunet-service-cadet_dht.h \
+ gnunet-service-cadet_hello.c gnunet-service-cadet_hello.h \
+ gnunet-service-cadet_tunnels.c gnunet-service-cadet_tunnels.h \
+ gnunet-service-cadet_paths.c gnunet-service-cadet_paths.h \
+ gnunet-service-cadet_peer.c gnunet-service-cadet_peer.h
+gnunet_service_cadet_LDADD = \
+  $(top_builddir)/src/lib/util/libgnunetutil.la \
+  $(top_builddir)/src/service/core/libgnunetcore.la \
+  $(top_builddir)/src/service/dht/libgnunetdht.la \
+  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/service/transport/libgnunettransportapplication.la \
+  $(top_builddir)/src/service/peerstore/libgnunetpeerstore.la \
+  $(top_builddir)/src/lib/hello/libgnunethello.la \
+  $(top_builddir)/src/lib/block/libgnunetblock.la
+if LINUX
+gnunet_service_cadet_LDFLAGS = -lrt \
+    $(GN_LIBINTL)
+endif
+
+
+test_cadet_local_mq_SOURCES = \
+  test_cadet_local_mq.c
+test_cadet_local_mq_LDADD = \
+  libgnunetcadet.la \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+if ENABLE_TEST_RUN
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
+TESTS = $(check_PROGRAMS)
+endif
+
+EXTRA_DIST = \
+  cadet.h cadet_protocol.h \
+  test_cadet.conf
diff --git a/src/service/cadet/TODO b/src/service/cadet/TODO
new file mode 100644
index 000000000..06567b0ad
--- /dev/null
+++ b/src/service/cadet/TODO
@@ -0,0 +1,36 @@
+- URGENT:
+  + if 'client-not-ready', we do not ACK at all, and sender keeps
+    retransmitting again and again; would be good to do flow-control notification instead
+    of not ACKing that we got the data but are simply not ready for more!
+  + Congestion/flow control (CHANNEL):
+    estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+    (and figure out how/where to use this!)
+
+- HIGH: revisit handling of 'unbuffered' traffic! (CHANNEL/TUNNEL)
+        (need to push down through tunnel into connection selection);
+        At Tunnel-level, try to create connections that match channel
+        preferences (buffered/unbuffered) and select connections for
+        channel traffic that match channel preferences.
+        BUT: not sure this is ideal, discloses traffic type to
+        routers. We don't want that! (Maybe revise decision to do this?)
+
+- HIGH: revisit handling of 'buffered' traffic: 4 is a rather small buffer; (CHANNEL)
+        maybe reserve more bits in 'options' to allow for buffer size control?
+        Or: maybe even better, calculated required buffer size based on latency
+        and throughput (and available memory)
+
+- HIGH: if we receive BROKEN messages, cut down corresponding PATH (up to the
+        point of breakage) as well as connection/route (CORE)
+
+- OPTIMIZATION: proper connection evaluation during connection management:
+  + TUNNELS:
+    * consider quality of current connection set when deciding
+      how often to do maintenance
+    * interact with PEER to drive DHT GET/PUT operations based
+      on how much we like our connections
+
+
+- OPTIMIZATION: optimize stopping/restarting DHT search to situations
+  where we actually need it (i.e. not if we have a direct connection,
+  or if we already have plenty of good short ones, or maybe even
+  to take a break if we have some connections and have searched a lot (?)) (PEER)
diff --git a/src/service/cadet/cadet.conf.in b/src/service/cadet/cadet.conf.in
new file mode 100644
index 000000000..8f700951c
--- /dev/null
+++ b/src/service/cadet/cadet.conf.in
@@ -0,0 +1,53 @@
+[cadet]
+IMMEDIATE_START = YES
+START_ON_DEMAND = @START_ON_DEMAND@
+@JAVAPORT@PORT = 2096
+HOSTNAME = localhost
+BINARY = gnunet-service-cadet
+# PREFIX = valgrind --leak-check=yes
+ACCEPT_FROM = 127.0.0.1;
+ACCEPT_FROM6 = ::1;
+UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-cadet.sock
+UNIX_MATCH_UID = NO
+UNIX_MATCH_GID = YES
+
+
+# How often do we send KEEPALIVE messages on connections to keep them
+# from timing out?
+REFRESH_CONNECTION_TIME = 5 min
+
+# Percentage of packets CADET is artificially dropping. Used for testing only!
+# DROP_PERCENT =
+
+# How frequently do we usually announce our presence in the DHT?
+ID_ANNOUNCE_TIME = 1 h
+
+# FIXME: document
+CONNECT_TIMEOUT = 30 s
+
+# What is the replication level we give to the DHT when announcing our
+# existence?  Usually there is no need to change this.
+DHT_REPLICATION_LEVEL = 3
+
+# FIXME: not implemented
+# MAX_TUNNELS = 1000
+
+# FIXME: not implemented, replaced by MAX_ROUTES in NEW CADET!
+MAX_CONNECTIONS = 1000
+
+# How many routes do we participate in at most?  Should be smaller
+# than MAX_MSGS_QUEUE
+MAX_ROUTES = 5000
+
+# FIXME: not implemented
+MAX_MSGS_QUEUE = 10000
+
+# FIXME: not implemented
+MAX_PEERS = 1000
+
+# How often do we advance the ratchet even if there is not
+# any traffic?
+RATCHET_TIME = 1 h
+
+# How often do we advance the ratched if there is traffic?
+RATCHET_MESSAGES = 64
diff --git a/src/service/cadet/cadet.h b/src/service/cadet/cadet.h
new file mode 100644
index 000000000..e8caa7d35
--- /dev/null
+++ b/src/service/cadet/cadet.h
@@ -0,0 +1,578 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001 - 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @author Bartlomiej Polot
+ * @file cadet/cadet.h
+ */
+
+#ifndef CADET_H_
+#define CADET_H_
+
+#ifdef __cplusplus
+extern "C" {
+#if 0 /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#include <stdint.h>
+
+#if ! defined(GNUNET_CULL_LOGGING)
+#define CADET_TIMING_START                 \
+  struct GNUNET_TIME_Absolute __timestamp; \
+  __timestamp = GNUNET_TIME_absolute_get ()
+
+#define CADET_TIMING_END                                        \
+  struct GNUNET_TIME_Relative __duration;                       \
+  __duration = GNUNET_TIME_absolute_get_duration (__timestamp); \
+  LOG (GNUNET_ERROR_TYPE_INFO,                                  \
+       " %s duration %s\n",                                     \
+       __FUNCTION__,                                            \
+       GNUNET_STRINGS_relative_time_to_string (__duration, GNUNET_YES));
+#else
+#define CADET_TIMING_START
+#define CADET_TIMING_END
+#endif
+
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_core_service.h"
+#include "gnunet_cadet_service.h"
+#include "gnunet_protocols.h"
+#include "gnunet_cadet_service.h"
+
+/******************************************************************************/
+/**************************       CONSTANTS      ******************************/
+/******************************************************************************/
+
+/**
+ * Minimum value for channel IDs of local clients.
+ */
+#define GNUNET_CADET_LOCAL_CHANNEL_ID_CLI 0x80000000U
+
+/**
+ * FIXME.
+ */
+#define HIGH_PID 0xFF000000
+
+/**
+ * FIXME.
+ */
+#define LOW_PID 0x00FFFFFF
+
+
+/**
+ * Test if the two PIDs (of type `uint32_t`) are in the range where we
+ * have to worry about overflows.  This is the case when @a pid is
+ * large and @a max is small, useful when comparing @a pid smaller
+ * than @a max.
+ */
+#define PID_OVERFLOW(pid, max) (((pid) > HIGH_PID) && ((max) < LOW_PID))
+
+/******************************************************************************/
+/**************************        MESSAGES      ******************************/
+/******************************************************************************/
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
+ * Number uniquely identifying a channel of a client.
+ */
+struct GNUNET_CADET_ClientChannelNumber
+{
+  /**
+   * Values for channel numbering.
+   * Local channel numbers given by the service (incoming) are
+   * smaller than #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI.
+   * Local channel numbers given by the client (created) are
+   * larger than #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI.
+   */
+  uint32_t channel_of_client GNUNET_PACKED;
+};
+
+/**
+ * Opaque handle to a channel.
+ */
+struct GNUNET_CADET_Channel
+{
+
+  /**
+   * Other end of the channel.
+   */
+  struct GNUNET_PeerIdentity peer;
+
+  /**
+   * Handle to the cadet this channel belongs to
+   */
+  struct GNUNET_CADET_Handle *cadet;
+
+  /**
+   * Channel's port, if incoming.
+   */
+  struct GNUNET_CADET_Port *incoming_port;
+
+  /**
+   * Any data the caller wants to put in here, used for the
+   * various callbacks (@e disconnects, @e window_changes, handlers).
+   */
+  void *ctx;
+
+  /**
+   * Message Queue for the channel (which we are implementing).
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Task to allow mq to send more traffic.
+   */
+  struct GNUNET_SCHEDULER_Task *mq_cont;
+
+  /**
+   * Pending envelope with a message to be transmitted to the
+   * service as soon as we are allowed to.  Should only be
+   * non-NULL if @e allow_send is 0.
+   */
+  struct GNUNET_MQ_Envelope *pending_env;
+
+  /**
+   * Window change handler.
+   */
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
+
+  /**
+   * Disconnect handler.
+   */
+  GNUNET_CADET_DisconnectEventHandler disconnects;
+
+  /**
+   * Local ID of the channel, #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI bit is set if outbound.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * How many messages are we allowed to send to the service right now?
+   */
+  unsigned int allow_send;
+};
+
+/**
+ * Message for a client to create and destroy channels.
+ */
+struct GNUNET_CADET_PortMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN
+   * or #GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE
+   *
+   * Size: sizeof(struct GNUNET_CADET_ChannelMessage)
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Port to open/close.
+   */
+  struct GNUNET_HashCode port GNUNET_PACKED;
+};
+
+
+/**
+ * Message for a client to create channels.
+ */
+struct GNUNET_CADET_LocalChannelCreateMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE
+   *
+   * Size: sizeof(struct GNUNET_CADET_ChannelOpenMessageMessage)
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of a channel controlled by this client.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * Channel's peer
+   */
+  struct GNUNET_PeerIdentity peer;
+
+  /**
+   * Port of the channel.
+   */
+  struct GNUNET_HashCode port;
+
+  /**
+   * Options.
+   */
+  uint32_t opt GNUNET_PACKED;
+};
+
+
+/**
+ * Message for or to a client to destroy tunnel.
+ */
+struct GNUNET_CADET_LocalChannelDestroyMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of a channel controlled by this client.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+};
+
+
+/**
+ * Message for cadet data traffic.
+ */
+struct GNUNET_CADET_LocalData
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of the channel
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * Priority and preferences (an enum GNUNET_MQ_PriorityPreferences)
+   * of the message in NBO.
+   */
+  uint32_t pp GNUNET_PACKED;
+
+  /**
+   * Payload follows
+   */
+};
+
+
+/**
+ * Message to allow the client send more data to the service
+ * (always service -> client).
+ */
+struct GNUNET_CADET_LocalAck
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of the channel allowed to send more data.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+};
+
+
+/**
+ * Message to inform the client about channels in the service.
+ *
+ * TODO: split into two messages!
+ */
+struct GNUNET_CADET_LocalInfo
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL or
+   * #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of the channel allowed to send more data.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * ID of the destination of the channel (can be local peer).
+   */
+  struct GNUNET_PeerIdentity peer;
+};
+
+/**
+ * Message to drop another message of specific type. Used in test context
+ */
+struct GNUNET_CADET_RequestDropCadetMessage
+{
+
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_DROP_CADET_MESSAGE
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Type of the message this handler covers, in host byte order.
+   */
+  uint16_t type;
+
+  /**
+   * ID of the channel we want to drop a message for.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+};
+
+/**
+ * Message to inform the client about channels in the service.
+ */
+struct GNUNET_CADET_RequestPathInfoMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PATH
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t resered GNUNET_PACKED;
+
+  /**
+   * ID of the destination of the channel (can be local peer).
+   */
+  struct GNUNET_PeerIdentity peer;
+};
+
+
+/**
+ * Message to inform the client about channels in the service.
+ */
+struct GNUNET_CADET_ChannelInfoMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Root of the channel
+   */
+  struct GNUNET_PeerIdentity root;
+
+  /**
+   * Destination of the channel
+   */
+  struct GNUNET_PeerIdentity dest;
+
+  /* FIXME: expand! */
+};
+
+
+/**
+ * Message to as the service about information on a channel.
+ */
+struct GNUNET_CADET_RequestChannelInfoMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_CHANNEL.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Target of the channel.
+   */
+  struct GNUNET_PeerIdentity target;
+};
+
+
+/**
+ * Message to inform the client about one of the paths known to the service.
+ */
+struct GNUNET_CADET_LocalInfoPath
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PATH.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Offset of the peer that was requested.
+   */
+  uint32_t off GNUNET_PACKED;
+};
+
+
+/**
+ * Message to inform the client about one of the peers in the service.
+ */
+struct GNUNET_CADET_LocalInfoPeers
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Number of paths.
+   */
+  uint16_t paths GNUNET_PACKED;
+
+  /**
+   * Do we have a tunnel toward this peer?
+   */
+  int16_t tunnel GNUNET_PACKED;
+
+  /**
+   * Shortest known path.
+   */
+  uint32_t best_path_length GNUNET_PACKED;
+
+  /**
+   * ID of the peer (can be local peer).
+   */
+  struct GNUNET_PeerIdentity destination;
+};
+
+
+/**
+ * Message to inform the client about one of the tunnels in the service.
+ *
+ * TODO: split into two messages!
+ */
+struct GNUNET_CADET_LocalInfoTunnel
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL
+   * or #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Number of channels.
+   */
+  uint32_t channels GNUNET_PACKED;
+
+  /**
+   * ID of the destination of the tunnel (can be local peer).
+   */
+  struct GNUNET_PeerIdentity destination;
+
+  /**
+   * Number of connections.
+   */
+  uint32_t connections GNUNET_PACKED;
+
+  /**
+   * Encryption state.
+   */
+  uint16_t estate GNUNET_PACKED;
+
+  /**
+   * Connection state.
+   */
+  uint16_t cstate GNUNET_PACKED;
+
+  /* If TUNNEL (no 'S'): struct GNUNET_CADET_ConnectionTunnelIdentifier connection_ids[connections] */
+  /* If TUNNEL (no 'S'): uint32_t channel_ids[channels] */
+};
+
+
+GNUNET_NETWORK_STRUCT_END
+
+
+/**
+ * @brief Translate a fwd variable into a string representation, for logging.
+ *
+ * @param fwd Is FWD? (#GNUNET_YES or #GNUNET_NO)
+ *
+ * @return String representing FWD or BCK.
+ */
+char *
+GC_f2s (int fwd);
+
+
+/**
+ * Check if one pid is bigger than other, accounting for overflow.
+ *
+ * @param bigger Argument that should be bigger.
+ * @param smaller Argument that should be smaller.
+ *
+ * @return True if bigger (arg1) has a higher value than smaller (arg 2).
+ */
+int
+GC_is_pid_bigger (uint32_t bigger, uint32_t smaller);
+
+
+/**
+ * Get the higher ACK value out of two values, taking in account overflow.
+ *
+ * @param a First ACK value.
+ * @param b Second ACK value.
+ *
+ * @return Highest ACK value from the two.
+ */
+uint32_t
+GC_max_pid (uint32_t a, uint32_t b);
+
+
+/**
+ * Get the lower ACK value out of two values, taking in account overflow.
+ *
+ * @param a First ACK value.
+ * @param b Second ACK value.
+ *
+ * @return Lowest ACK value from the two.
+ */
+uint32_t
+GC_min_pid (uint32_t a, uint32_t b);
+
+
+/**
+ * Allocate a string with a hexdump of any binary data.
+ *
+ * @param bin Arbitrary binary data.
+ * @param len Length of @a bin in bytes.
+ * @param output Where to write the output (if *output be NULL it's allocated).
+ *
+ * @return The size of the output.
+ */
+size_t
+GC_bin2s (void *bin, unsigned int len, char **output);
+
+
+/**
+ * Convert a message type into a string to help debug
+ * Generated with:
+ * FIND:        "#define ([^ ]+)[ ]*([0-9]+)"
+ * REPLACE:     "    case \2: return "\1"; break;"
+ *
+ * @param m Message type.
+ *
+ * @return Human readable string description.
+ */
+const char *
+GC_m2s (uint16_t m);
+
+#if 0 /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/service/cadet/cadet_api.c b/src/service/cadet/cadet_api.c
new file mode 100644
index 000000000..0bfb01868
--- /dev/null
+++ b/src/service/cadet/cadet_api.c
@@ -0,0 +1,1072 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+#define LOG(kind, ...) GNUNET_log_from (kind, "cadet-api", __VA_ARGS__)
+
+/**
+ * Opaque handle to the service.
+ */
+struct GNUNET_CADET_Handle
+{
+  /**
+   * Message queue.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Ports open.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *ports;
+
+  /**
+   * Channels open.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
+
+  /**
+   * child of the next channel to create (to avoid reusing IDs often)
+   */
+  struct GNUNET_CADET_ClientChannelNumber next_ccn;
+
+  /**
+   * Configuration given by the client, in case of reconnection
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Task for trying to reconnect.
+   */
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
+
+  /**
+   * Time to the next reconnect in case one reconnect fails
+   */
+  struct GNUNET_TIME_Relative reconnect_time;
+};
+
+/**
+ * Opaque handle to a port.
+ */
+struct GNUNET_CADET_Port
+{
+  /**
+   * Port "number"
+   */
+  struct GNUNET_HashCode id;
+
+  /**
+   * Handle to the CADET session this port belongs to.
+   */
+  struct GNUNET_CADET_Handle *cadet;
+
+  /**
+   * Closure for @a handler.
+   */
+  void *cls;
+
+  /**
+   * Handler for incoming channels on this port
+   */
+  GNUNET_CADET_ConnectEventHandler connects;
+
+  /**
+   * Closure for @ref connects
+   */
+  void *connects_cls;
+
+  /**
+   * Window size change handler.
+   */
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
+
+  /**
+   * Handler called when an incoming channel is destroyed.
+   */
+  GNUNET_CADET_DisconnectEventHandler disconnects;
+
+  /**
+   * Payload handlers for incoming channels.
+   */
+  struct GNUNET_MQ_MessageHandler *handlers;
+};
+
+
+/**
+ * Find the Port struct for a hash.
+ *
+ * @param h CADET handle.
+ * @param hash HashCode for the port number.
+ * @return The port handle if known, NULL otherwise.
+ */
+static struct GNUNET_CADET_Port *
+find_port (const struct GNUNET_CADET_Handle *h,
+           const struct GNUNET_HashCode *hash)
+{
+  return GNUNET_CONTAINER_multihashmap_get (h->ports, hash);
+}
+
+
+/**
+ * Get the channel handler for the channel specified by id from the given handle
+ *
+ * @param h Cadet handle
+ * @param ccn ID of the wanted channel
+ * @return handle to the required channel or NULL if not found
+ */
+static struct GNUNET_CADET_Channel *
+find_channel (struct GNUNET_CADET_Handle *h,
+              struct GNUNET_CADET_ClientChannelNumber ccn)
+{
+  return GNUNET_CONTAINER_multihashmap32_get (h->channels,
+                                              ntohl (ccn.channel_of_client));
+}
+
+
+/**
+ * Create a new channel and insert it in the channel list of the cadet handle
+ *
+ * @param h Cadet handle
+ * @param ccnp pointer to desired ccn of the channel, NULL to assign one automatically.
+ * @return Handle to the created channel.
+ */
+static struct GNUNET_CADET_Channel *
+create_channel (struct GNUNET_CADET_Handle *h,
+                const struct GNUNET_CADET_ClientChannelNumber *ccnp)
+{
+  struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  ch = GNUNET_new (struct GNUNET_CADET_Channel);
+  ch->cadet = h;
+  if (NULL == ccnp)
+  {
+    while (NULL != find_channel (h, h->next_ccn))
+      h->next_ccn.channel_of_client =
+        htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI
+               | (1 + ntohl (h->next_ccn.channel_of_client)));
+    ccn = h->next_ccn;
+  }
+  else
+  {
+    ccn = *ccnp;
+  }
+  ch->ccn = ccn;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (
+                   h->channels,
+                   ntohl (ch->ccn.channel_of_client),
+                   ch,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  return ch;
+}
+
+
+/**
+ * Destroy the specified channel.
+ * - Destroys all peers, calling the disconnect callback on each if needed
+ * - Cancels all outgoing traffic for that channel, calling respective notifys
+ * - Calls cleaner if channel was inbound
+ * - Frees all memory used
+ *
+ * @param ch Pointer to the channel.
+ */
+static void
+destroy_channel (struct GNUNET_CADET_Channel *ch)
+{
+  struct GNUNET_CADET_Handle *h = ch->cadet;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying channel %X of %p\n",
+       htonl (ch->ccn.channel_of_client),
+       h);
+  GNUNET_assert (
+    GNUNET_YES ==
+    GNUNET_CONTAINER_multihashmap32_remove (h->channels,
+                                            ntohl (ch->ccn.channel_of_client),
+                                            ch));
+  if (NULL != ch->mq_cont)
+  {
+    GNUNET_SCHEDULER_cancel (ch->mq_cont);
+    ch->mq_cont = NULL;
+  }
+  /* signal channel destruction */
+  if (NULL != ch->disconnects)
+    ch->disconnects (ch->ctx, ch);
+  if (NULL != ch->pending_env)
+    GNUNET_MQ_discard (ch->pending_env);
+  GNUNET_MQ_destroy (ch->mq);
+  GNUNET_free (ch);
+}
+
+
+/**
+ * Reconnect to the service, retransmit all information to try to restore the
+ * original state.
+ *
+ * @param h handle to the cadet
+ */
+static void
+reconnect (struct GNUNET_CADET_Handle *h);
+
+
+/**
+ * Function called during #reconnect_cbk() to (re)open
+ * all ports that are still open.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param id port ID
+ * @param value a `struct GNUNET_CADET_Channel` to open
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+open_port_cb (void *cls, const struct GNUNET_HashCode *id, void *value)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+  struct GNUNET_CADET_Port *port = value;
+  struct GNUNET_CADET_PortMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  (void) id;
+  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
+  msg->port = port->id;
+  GNUNET_MQ_send (h->mq, env);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Reconnect callback: tries to reconnect again after a failed previous
+ * connection
+ *
+ * @param cls closure (cadet handle)
+ */
+static void
+reconnect_cbk (void *cls)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+
+  h->reconnect_task = NULL;
+  h->reconnect_time = GNUNET_TIME_STD_BACKOFF (h->reconnect_time);
+  reconnect (h);
+  GNUNET_CONTAINER_multihashmap_iterate (h->ports, &open_port_cb, h);
+}
+
+
+/**
+ * Notify the application about a change in the window size (if needed).
+ *
+ * @param ch Channel to notify about.
+ */
+static void
+notify_window_size (struct GNUNET_CADET_Channel *ch)
+{
+  if (NULL != ch->window_changes)
+    ch->window_changes (ch->ctx,
+                        ch, /* FIXME: remove 'ch'? */
+                        ch->allow_send);
+}
+
+
+/**
+ * Transmit the next message from our queue.
+ *
+ * @param cls Closure (channel whose mq to activate).
+ */
+static void
+cadet_mq_send_now (void *cls)
+{
+  struct GNUNET_CADET_Channel *ch = cls;
+  struct GNUNET_MQ_Envelope *env = ch->pending_env;
+
+  ch->mq_cont = NULL;
+  if (0 == ch->allow_send)
+  {
+    /* how did we get here? */
+    GNUNET_break (0);
+    return;
+  }
+  if (NULL == env)
+  {
+    /* how did we get here? */
+    GNUNET_break (0);
+    return;
+  }
+  ch->allow_send--;
+  ch->pending_env = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending message on channel %s to CADET, new window size is %u\n",
+              GNUNET_i2s (&ch->peer),
+              ch->allow_send);
+  GNUNET_MQ_send (ch->cadet->mq, env);
+  GNUNET_MQ_impl_send_continue (ch->mq);
+}
+
+
+/**
+ * Implement sending functionality of a message queue for
+ * us sending messages to a peer.
+ *
+ * Encapsulates the payload message in a #GNUNET_CADET_LocalData message
+ * in order to label the message with the channel ID and send the
+ * encapsulated message to the service.
+ *
+ * @param mq the message queue
+ * @param msg the message to send
+ * @param impl_state state of the implementation
+ */
+static void
+cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
+                    const struct GNUNET_MessageHeader *msg,
+                    void *impl_state)
+{
+  struct GNUNET_CADET_Channel *ch = impl_state;
+  struct GNUNET_CADET_Handle *h = ch->cadet;
+  uint16_t msize;
+  struct GNUNET_MQ_Envelope *orig_env;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalData *cadet_msg;
+  enum GNUNET_MQ_PriorityPreferences pp;
+
+  if (NULL == h->mq)
+  {
+    /* We're currently reconnecting, pretend this worked */
+    GNUNET_MQ_impl_send_continue (mq);
+    return;
+  }
+  orig_env = GNUNET_MQ_get_current_envelope (mq);
+  pp = GNUNET_MQ_env_get_options (orig_env);
+
+  /* check message size for sanity */
+  msize = ntohs (msg->size);
+  if (msize > GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE)
+  {
+    GNUNET_break (0);
+    GNUNET_MQ_impl_send_continue (mq);
+    return;
+  }
+  env = GNUNET_MQ_msg_nested_mh (cadet_msg,
+                                 GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                                 msg);
+  cadet_msg->ccn = ch->ccn;
+  cadet_msg->pp = htonl ((uint32_t) pp);
+  GNUNET_assert (NULL == ch->pending_env);
+  ch->pending_env = env;
+  if (0 < ch->allow_send)
+    ch->mq_cont = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now, ch);
+}
+
+
+/**
+ * Handle destruction of a message queue.  Implementations must not
+ * free @a mq, but should take care of @a impl_state.
+ *
+ * @param mq the message queue to destroy
+ * @param impl_state state of the implementation
+ */
+static void
+cadet_mq_destroy_impl (struct GNUNET_MQ_Handle *mq, void *impl_state)
+{
+  struct GNUNET_CADET_Channel *ch = impl_state;
+
+  GNUNET_assert (mq == ch->mq);
+  ch->mq = NULL;
+}
+
+
+/**
+ * We had an error processing a message we forwarded from a peer to
+ * the CADET service.  We should just complain about it but otherwise
+ * continue processing.
+ *
+ * @param cls closure with our `struct GNUNET_CADET_Channel`
+ * @param error error code
+ */
+static void
+cadet_mq_error_handler (void *cls, enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_Channel *ch = cls;
+
+  if (GNUNET_MQ_ERROR_NO_MATCH == error)
+  {
+    /* Got a message we did not understand, still try to continue! */
+    GNUNET_break_op (0);
+    GNUNET_CADET_receive_done (ch);
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "MQ error in communication with CADET: %d\n",
+                error);
+    if (NULL != ch->disconnects)
+      ch->disconnects (ch->ctx, ch);
+    GNUNET_CADET_channel_destroy (ch);
+  }
+}
+
+
+/**
+ * Implementation function that cancels the currently sent message.
+ * Should basically undo whatever #mq_send_impl() did.
+ *
+ * @param mq message queue
+ * @param impl_state state specific to the implementation
+ */
+static void
+cadet_mq_cancel_impl (struct GNUNET_MQ_Handle *mq, void *impl_state)
+{
+  struct GNUNET_CADET_Channel *ch = impl_state;
+
+  (void) mq;
+  GNUNET_assert (NULL != ch->pending_env);
+  GNUNET_MQ_discard (ch->pending_env);
+  ch->pending_env = NULL;
+  if (NULL != ch->mq_cont)
+  {
+    GNUNET_SCHEDULER_cancel (ch->mq_cont);
+    ch->mq_cont = NULL;
+  }
+}
+
+
+/**
+ * Process the new channel notification and add it to the channels in the handle
+ *
+ * @param cls   The cadet handle
+ * @param msg   A message with the details of the new incoming channel
+ */
+static void
+handle_channel_created (
+  void *cls,
+  const struct GNUNET_CADET_LocalChannelCreateMessage *msg)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+  struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_Port *port;
+  const struct GNUNET_HashCode *port_number;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  ccn = msg->ccn;
+  port_number = &msg->port;
+  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  port = find_port (h, port_number);
+  if (NULL == port)
+  {
+    /* We could have closed the port but the service didn't know about it yet
+     * This is not an error.
+     */
+    struct GNUNET_CADET_LocalChannelDestroyMessage *d_msg;
+    struct GNUNET_MQ_Envelope *env;
+
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "No handler for incoming channel %X (on port %s, recently closed?)\n",
+         ntohl (ccn.channel_of_client),
+         GNUNET_h2s (port_number));
+    env =
+      GNUNET_MQ_msg (d_msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+    d_msg->ccn = msg->ccn;
+    GNUNET_MQ_send (h->mq, env);
+    return;
+  }
+
+  ch = create_channel (h, &ccn);
+  ch->peer = msg->peer;
+  ch->incoming_port = port;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating incoming channel %X [%s] %p\n",
+       ntohl (ccn.channel_of_client),
+       GNUNET_h2s (port_number),
+       ch);
+
+  GNUNET_assert (NULL != port->connects);
+  ch->window_changes = port->window_changes;
+  ch->disconnects = port->disconnects;
+  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
+                                          &cadet_mq_destroy_impl,
+                                          &cadet_mq_cancel_impl,
+                                          ch,
+                                          port->handlers,
+                                          &cadet_mq_error_handler,
+                                          ch);
+  ch->ctx = port->connects (port->cls, ch, &msg->peer);
+  GNUNET_MQ_set_handlers_closure (ch->mq, ch->ctx);
+}
+
+
+/**
+ * Process the channel destroy notification and free associated resources
+ *
+ * @param cls   The cadet handle
+ * @param msg   A message with the details of the channel being destroyed
+ */
+static void
+handle_channel_destroy (
+  void *cls,
+  const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+  struct GNUNET_CADET_Channel *ch;
+
+  ch = find_channel (h, msg->ccn);
+  if (NULL == ch)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received channel destroy for unknown channel %X from CADET service (recently close?)\n",
+         ntohl (msg->ccn.channel_of_client));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received channel destroy for channel %X from CADET service\n",
+       ntohl (msg->ccn.channel_of_client));
+  destroy_channel (ch);
+}
+
+
+/**
+ * Check that message received from CADET service is well-formed.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param message the message we got
+ * @return #GNUNET_OK if the message is well-formed,
+ *         #GNUNET_SYSERR otherwise
+ */
+static int
+check_local_data (void *cls, const struct GNUNET_CADET_LocalData *message)
+{
+  uint16_t size;
+
+  (void) cls;
+  size = ntohs (message->header.size);
+  if (sizeof(*message) + sizeof(struct GNUNET_MessageHeader) > size)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process the incoming data packets, call appropriate handlers.
+ *
+ * @param cls     The cadet handle
+ * @param message A message encapsulating the data
+ */
+static void
+handle_local_data (void *cls, const struct GNUNET_CADET_LocalData *message)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+  const struct GNUNET_MessageHeader *payload;
+  struct GNUNET_CADET_Channel *ch;
+  uint16_t type;
+  int fwd;
+
+  ch = find_channel (h, message->ccn);
+  if (NULL == ch)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Unknown channel %X for incoming data (recently closed?)\n",
+         ntohl (message->ccn.channel_of_client));
+    return;
+  }
+
+  payload = (const struct GNUNET_MessageHeader *) &message[1];
+  type = ntohs (payload->type);
+  fwd = ntohl (ch->ccn.channel_of_client) <= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got a %s data on channel %s [%X] of type %u\n",
+       fwd ? "FWD" : "BWD",
+       GNUNET_i2s (&ch->peer),
+       ntohl (message->ccn.channel_of_client),
+       type);
+  GNUNET_MQ_inject_message (ch->mq, payload);
+}
+
+
+/**
+ * Process a local ACK message, enabling the client to send
+ * more data to the service.
+ *
+ * @param cls Cadet handle.
+ * @param message Message itself.
+ */
+static void
+handle_local_ack (void *cls, const struct GNUNET_CADET_LocalAck *message)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+  struct GNUNET_CADET_Channel *ch;
+
+  ch = find_channel (h, message->ccn);
+  if (NULL == ch)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "ACK on unknown channel %X\n",
+         ntohl (message->ccn.channel_of_client));
+    return;
+  }
+  ch->allow_send++;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got an ACK on mq channel %X (peer %s); new window size is %u!\n",
+       ntohl (ch->ccn.channel_of_client),
+       GNUNET_i2s (&ch->peer),
+       ch->allow_send);
+  if (NULL == ch->pending_env)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got an ACK on mq channel %X, allow send now %u!\n",
+         ntohl (ch->ccn.channel_of_client),
+         ch->allow_send);
+    notify_window_size (ch);
+    return;
+  }
+  if (NULL != ch->mq_cont)
+    return; /* already working on it! */
+  ch->mq_cont = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now, ch);
+}
+
+
+/**
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all channels that are still open.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param cid chanenl ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_channel_cb (void *cls, uint32_t cid, void *value)
+{
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Channel *ch = value;
+
+  (void) cls;
+  (void) cid;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Destroying channel due to GNUNET_CADET_disconnect()\n");
+  destroy_channel (ch);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure, a `struct GNUNET_CORE_Handle *`
+ * @param error error code
+ */
+static void
+handle_mq_error (void *cls, enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MQ ERROR: %u\n", error);
+  GNUNET_CONTAINER_multihashmap32_iterate (h->channels, &destroy_channel_cb, h);
+  GNUNET_MQ_destroy (h->mq);
+  h->mq = NULL;
+  GNUNET_assert (NULL == h->reconnect_task);
+  h->reconnect_task =
+    GNUNET_SCHEDULER_add_delayed (h->reconnect_time, &reconnect_cbk, h);
+}
+
+
+/**
+ * Reconnect to the service, retransmit all information to try to restore the
+ * original state.
+ *
+ * @param h handle to the cadet
+ */
+static void
+reconnect (struct GNUNET_CADET_Handle *h)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] =
+  { GNUNET_MQ_hd_fixed_size (channel_created,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
+                             struct GNUNET_CADET_LocalChannelCreateMessage,
+                             h),
+    GNUNET_MQ_hd_fixed_size (channel_destroy,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
+                             struct GNUNET_CADET_LocalChannelDestroyMessage,
+                             h),
+    GNUNET_MQ_hd_var_size (local_data,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                           struct GNUNET_CADET_LocalData,
+                           h),
+    GNUNET_MQ_hd_fixed_size (local_ack,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
+                             struct GNUNET_CADET_LocalAck,
+                             h),
+    GNUNET_MQ_handler_end () };
+
+  GNUNET_assert (NULL == h->mq);
+  h->mq =
+    GNUNET_CLIENT_connect (h->cfg, "cadet", handlers, &handle_mq_error, h);
+}
+
+
+/**
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all ports that are still open.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param id port ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_port_cb (void *cls, const struct GNUNET_HashCode *id, void *value)
+{
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Port *port = value;
+
+  (void) cls;
+  (void) id;
+  /* This is a warning, the app should have cleanly closed all open ports */
+  GNUNET_break (0);
+  GNUNET_CADET_close_port (port);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Disconnect from the cadet service. All channels will be destroyed. All channel
+ * disconnect callbacks will be called on any still connected peers, notifying
+ * about their disconnection. The registered inbound channel cleaner will be
+ * called should any inbound channels still exist.
+ *
+ * @param handle connection to cadet to disconnect
+ */
+void
+GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
+{
+  GNUNET_CONTAINER_multihashmap_iterate (handle->ports,
+                                         &destroy_port_cb,
+                                         handle);
+  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
+  handle->ports = NULL;
+  GNUNET_CONTAINER_multihashmap32_iterate (handle->channels,
+                                           &destroy_channel_cb,
+                                           handle);
+  GNUNET_CONTAINER_multihashmap32_destroy (handle->channels);
+  handle->channels = NULL;
+  if (NULL != handle->mq)
+  {
+    GNUNET_MQ_destroy (handle->mq);
+    handle->mq = NULL;
+  }
+  if (NULL != handle->reconnect_task)
+  {
+    GNUNET_SCHEDULER_cancel (handle->reconnect_task);
+    handle->reconnect_task = NULL;
+  }
+  GNUNET_free (handle);
+}
+
+
+void
+GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p)
+{
+  GNUNET_assert (
+    GNUNET_YES ==
+    GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports, &p->id, p));
+  if (NULL != p->cadet->mq)
+  {
+    struct GNUNET_CADET_PortMessage *msg;
+    struct GNUNET_MQ_Envelope *env;
+
+    env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
+    msg->port = p->id;
+    GNUNET_MQ_send (p->cadet->mq, env);
+  }
+  GNUNET_free (p->handlers);
+  GNUNET_free (p);
+}
+
+
+/**
+ * Destroy an existing channel.
+ *
+ * The existing end callback for the channel will NOT be called.
+ * Any pending outgoing messages will be sent but no incoming messages will be
+ * accepted and no data callbacks will be called.
+ *
+ * @param channel Channel handle, becomes invalid after this call.
+ */
+void
+GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
+{
+  struct GNUNET_CADET_Handle *h = channel->cadet;
+  struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  if (NULL != h->mq)
+  {
+    env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+    msg->ccn = channel->ccn;
+    GNUNET_MQ_send (h->mq, env);
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Destroying channel due to GNUNET_CADET_channel_destroy()\n");
+  channel->disconnects = NULL;
+  destroy_channel (channel);
+}
+
+
+const union GNUNET_CADET_ChannelInfo *
+GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
+                               enum GNUNET_CADET_ChannelInfoOption option,
+                               ...)
+{
+  switch (option)
+  {
+  case GNUNET_CADET_OPTION_PEER:
+    return (const union GNUNET_CADET_ChannelInfo *) &channel->peer;
+
+  default:
+    GNUNET_break (0);
+    return NULL;
+  }
+}
+
+
+/**
+ * Send an ack on the channel to confirm the processing of a message.
+ *
+ * @param ch Channel on which to send the ACK.
+ */
+void
+GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel)
+{
+  struct GNUNET_CADET_LocalAck *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending ACK on channel %X\n",
+       ntohl (channel->ccn.channel_of_client));
+  msg->ccn = channel->ccn;
+  GNUNET_MQ_send (channel->cadet->mq, env);
+}
+
+
+/**
+ * Connect to the MQ-based cadet service.
+ *
+ * @param cfg Configuration to use.
+ *
+ * @return Handle to the cadet service NULL on error.
+ */
+struct GNUNET_CADET_Handle *
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  struct GNUNET_CADET_Handle *h;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "GNUNET_CADET_connect()\n");
+  h = GNUNET_new (struct GNUNET_CADET_Handle);
+  h->cfg = cfg;
+  h->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_YES);
+  h->channels = GNUNET_CONTAINER_multihashmap32_create (4);
+  reconnect (h);
+  if (NULL == h->mq)
+  {
+    GNUNET_break (0);
+    GNUNET_CADET_disconnect (h);
+    return NULL;
+  }
+  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  return h;
+}
+
+
+/**
+ * Function to return link to AGPL source upon request.
+ *
+ * @param cls closure with the identification of the client
+ * @param msg AGPL request
+ */
+static void
+return_agpl (void *cls, const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  struct GNUNET_MQ_Handle *mq;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *res;
+  size_t slen;
+  const struct GNUNET_OS_ProjectData *pd = GNUNET_OS_project_data_get ();
+
+  (void) msg;
+  slen = strlen (pd->agpl_url) + 1;
+  env = GNUNET_MQ_msg_extra (res, GNUNET_MESSAGE_TYPE_RESPONSE_AGPL, slen);
+  memcpy (&res[1], GNUNET_AGPL_URL, slen);
+  mq = GNUNET_SERVICE_client_get_mq (client);
+  GNUNET_MQ_send (mq, env);
+  GNUNET_SERVICE_client_continue (client);
+}
+
+
+/**
+ * Open a port to receive incoming MQ-based channels.
+ *
+ * @param h CADET handle.
+ * @param port Hash identifying the port.
+ * @param connects Function called when an incoming channel is connected.
+ * @param connects_cls Closure for the @a connects handler.
+ * @param window_changes Function called when the transmit window size changes.
+ * @param disconnects Function called when a channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Port handle, NULL if port is in use
+ */
+struct GNUNET_CADET_Port *
+GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
+                        const struct GNUNET_HashCode *port,
+                        GNUNET_CADET_ConnectEventHandler connects,
+                        void *connects_cls,
+                        GNUNET_CADET_WindowSizeEventHandler window_changes,
+                        GNUNET_CADET_DisconnectEventHandler disconnects,
+                        const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  struct GNUNET_CADET_Port *p;
+  const struct GNUNET_OS_ProjectData *pd = GNUNET_OS_project_data_get ();
+
+  GNUNET_assert (NULL != connects);
+  GNUNET_assert (NULL != disconnects);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Listening to CADET port %s\n",
+              GNUNET_h2s (port));
+
+  p = GNUNET_new (struct GNUNET_CADET_Port);
+  p->cadet = h;
+  p->id = *port;
+  if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
+        h->ports,
+        &p->id,
+        p,
+        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  {
+    GNUNET_free (p);
+    return NULL;
+  }
+  p->connects = connects;
+  p->cls = connects_cls;
+  p->window_changes = window_changes;
+  p->disconnects = disconnects;
+  p->handlers = (NULL == pd->agpl_url)
+                ? GNUNET_MQ_copy_handlers (handlers)
+                : GNUNET_MQ_copy_handlers2 (handlers, &return_agpl, NULL);
+
+  GNUNET_assert (GNUNET_OK == open_port_cb (h, &p->id, p));
+  return p;
+}
+
+
+/**
+ * Create a new channel towards a remote peer.
+ *
+ * If the destination peer closes the channel after accepting it,
+ * @a disconnects will be called for this channel (unless
+ * #GNUNET_CADET_channel_destroy() was called on this end first).
+ *
+ * @param h CADET handle.
+ * @param channel_cls Closure for the channel. It's given to:
+ *                    - The disconnect handler @a disconnects
+ *                    - Each message type callback in @a handlers
+ * @param destination Peer identity the channel should go to.
+ * @param port Identification of the destination port.
+ * @param window_changes Function called when the transmit window size changes.
+ * @param disconnects Function called when the channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Handle to the channel.
+ */
+struct GNUNET_CADET_Channel *
+GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
+                             void *channel_cls,
+                             const struct GNUNET_PeerIdentity *destination,
+                             const struct GNUNET_HashCode *port,
+                             GNUNET_CADET_WindowSizeEventHandler window_changes,
+                             GNUNET_CADET_DisconnectEventHandler disconnects,
+                             const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  GNUNET_assert (NULL != disconnects);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating channel to peer %s at port %s\n",
+              GNUNET_i2s (destination),
+              GNUNET_h2s (port));
+  ch = create_channel (h, NULL);
+  ch->ctx = channel_cls;
+  ch->peer = *destination;
+  ch->window_changes = window_changes;
+  ch->disconnects = disconnects;
+
+  /* Create MQ for channel */
+  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
+                                          &cadet_mq_destroy_impl,
+                                          &cadet_mq_cancel_impl,
+                                          ch,
+                                          handlers,
+                                          &cadet_mq_error_handler,
+                                          ch);
+  GNUNET_MQ_set_handlers_closure (ch->mq, channel_cls);
+
+  /* Request channel creation to service */
+  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  msg->ccn = ch->ccn;
+  msg->port = *port;
+  msg->peer = *destination;
+  GNUNET_MQ_send (h->mq, env);
+  return ch;
+}
+
+
+/**
+ * Obtain the message queue for a connected peer.
+ *
+ * @param channel The channel handle from which to get the MQ.
+ *
+ * @return NULL if @a channel is not yet connected.
+ */
+struct GNUNET_MQ_Handle *
+GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel)
+{
+  return channel->mq;
+}
+
+
+/* end of cadet_api.c */
diff --git a/src/service/cadet/cadet_api_drop_message.c b/src/service/cadet/cadet_api_drop_message.c
new file mode 100644
index 000000000..e172418c4
--- /dev/null
+++ b/src/service/cadet/cadet_api_drop_message.c
@@ -0,0 +1,59 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017, 2019 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+*/
+/**
+ * @file cadet/cadet_api_drop_message.c
+ * @brief cadet api: client implementation of cadet service
+ * @author t3sserakt
+ */
+#include "platform.h"
+#include "cadet.h"
+
+
+/**
+ * Drop the next cadet message of a given type.
+ *
+ * @param mq message queue
+ * @param ccn client channel number.
+ * @param type of cadet message to be dropped.
+ */
+void
+GNUNET_CADET_drop_message (struct GNUNET_MQ_Handle *mq,
+                           struct GNUNET_CADET_ClientChannelNumber ccn,
+                           uint16_t type)
+{
+  struct GNUNET_CADET_RequestDropCadetMessage *message;
+  struct GNUNET_MQ_Envelope *env;
+
+  env = GNUNET_MQ_msg (message, GNUNET_MESSAGE_TYPE_CADET_DROP_CADET_MESSAGE);
+
+  message->ccn = ccn;
+  message->type = type;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Dropping message for channel of type %s (%d)\n", type ==
+              GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY ?
+              "GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY" : "UNKNOWN", type);
+
+  GNUNET_MQ_send (mq, env);
+
+}
+
+
+/* end of cadet_api_drop_message.c */
diff --git a/src/service/cadet/cadet_api_get_channel.c b/src/service/cadet/cadet_api_get_channel.c
new file mode 100644
index 000000000..05e2a8383
--- /dev/null
+++ b/src/service/cadet/cadet_api_get_channel.c
@@ -0,0 +1,260 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017, 2019 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api_get_channel.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Operation handle.
+ */
+struct GNUNET_CADET_ChannelMonitor
+{
+  /**
+   * Channel callback.
+   */
+  GNUNET_CADET_ChannelCB channel_cb;
+
+  /**
+   * Info callback closure for @c channel_cb.
+   */
+  void *channel_cb_cls;
+
+  /**
+   * Configuration we use.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Message queue to talk to CADET service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Task to reconnect.
+   */
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
+
+  /**
+   * Backoff for reconnect attempts.
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
+   * Peer we want information about.
+   */
+  struct GNUNET_PeerIdentity peer;
+};
+
+
+/**
+ * Check that message received from CADET service is well-formed.
+ *
+ * @param cls unused
+ * @param message the message we got
+ * @return #GNUNET_OK if the message is well-formed,
+ *         #GNUNET_SYSERR otherwise
+ */
+static int
+check_channel_info (void *cls,
+                    const struct GNUNET_CADET_ChannelInfoMessage *message)
+{
+  (void) cls;
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a local peer info reply, pass info to the user.
+ *
+ * @param cls Closure
+ * @param message Message itself.
+ */
+static void
+handle_channel_info (void *cls,
+                     const struct GNUNET_CADET_ChannelInfoMessage *message)
+{
+  struct GNUNET_CADET_ChannelMonitor *cm = cls;
+  struct GNUNET_CADET_ChannelInternals ci;
+
+  ci.root = message->root;
+  ci.dest = message->dest;
+  cm->channel_cb (cm->channel_cb_cls,
+                  &ci);
+  GNUNET_CADET_get_channel_cancel (cm);
+}
+
+
+/**
+ * Process a local peer info reply, pass info to the user.
+ *
+ * @param cls Closure
+ * @param message Message itself.
+ */
+static void
+handle_channel_info_end (void *cls,
+                         const struct GNUNET_MessageHeader *message)
+{
+  struct GNUNET_CADET_ChannelMonitor *cm = cls;
+
+  cm->channel_cb (cm->channel_cb_cls,
+                  NULL);
+  GNUNET_CADET_get_channel_cancel (cm);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_ChannelMonitor` operation
+ */
+static void
+reconnect (void *cls);
+
+
+/**
+ * Function called on connection trouble.  Reconnects.
+ *
+ * @param cls a `struct GNUNET_CADET_ChannelMonitor``
+ * @param error error code from MQ
+ */
+static void
+error_handler (void *cls,
+               enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_ChannelMonitor *cm = cls;
+
+  GNUNET_MQ_destroy (cm->mq);
+  cm->mq = NULL;
+  cm->backoff = GNUNET_TIME_randomized_backoff (cm->backoff,
+                                                GNUNET_TIME_UNIT_MINUTES);
+  cm->reconnect_task = GNUNET_SCHEDULER_add_delayed (cm->backoff,
+                                                     &reconnect,
+                                                     cm);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_ChannelMonitor` operation
+ */
+static void
+reconnect (void *cls)
+{
+  struct GNUNET_CADET_ChannelMonitor *cm = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (channel_info_end,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL_END,
+                             struct GNUNET_MessageHeader,
+                             cm),
+    GNUNET_MQ_hd_var_size (channel_info,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL,
+                           struct GNUNET_CADET_ChannelInfoMessage,
+                           cm),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_CADET_RequestChannelInfoMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  cm->reconnect_task = NULL;
+  cm->mq = GNUNET_CLIENT_connect (cm->cfg,
+                                  "cadet",
+                                  handlers,
+                                  &error_handler,
+                                  cm);
+  if (NULL == cm->mq)
+    return;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_CHANNEL);
+  msg->target = cm->peer;
+  GNUNET_MQ_send (cm->mq,
+                  env);
+}
+
+
+/**
+ * Request information about a specific channel of the running cadet peer.
+ *
+ * @param cfg configuration to use
+ * @param peer ID of the other end of the channel.
+ * @param callback Function to call with the requested data.
+ * @param callback_cls Closure for @c callback.
+ * @return NULL on error
+ */
+struct GNUNET_CADET_ChannelMonitor *
+GNUNET_CADET_get_channel (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                          struct GNUNET_PeerIdentity *peer,
+                          GNUNET_CADET_ChannelCB callback,
+                          void *callback_cls)
+{
+  struct GNUNET_CADET_ChannelMonitor *cm;
+
+  if (NULL == callback)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  cm = GNUNET_new (struct GNUNET_CADET_ChannelMonitor);
+  cm->channel_cb = callback;
+  cm->channel_cb_cls = callback_cls;
+  cm->cfg = cfg;
+  cm->peer = *peer;
+  reconnect (cm);
+  if (NULL == cm->mq)
+  {
+    GNUNET_free (cm);
+    return NULL;
+  }
+  return cm;
+}
+
+
+/**
+ * Cancel a channel monitor request. The callback will not be called (anymore).
+ *
+ * @param h Cadet handle.
+ * @return Closure that was given to #GNUNET_CADET_get_channel().
+ */
+void *
+GNUNET_CADET_get_channel_cancel (struct GNUNET_CADET_ChannelMonitor *cm)
+{
+  void *ret = cm->channel_cb_cls;
+
+  if (NULL != cm->mq)
+    GNUNET_MQ_destroy (cm->mq);
+  if (NULL != cm->reconnect_task)
+    GNUNET_SCHEDULER_cancel (cm->reconnect_task);
+  GNUNET_free (cm);
+  return ret;
+}
+
+
+/* end of cadet_api_get_channel.c */
diff --git a/src/service/cadet/cadet_api_get_path.c b/src/service/cadet/cadet_api_get_path.c
new file mode 100644
index 000000000..f32a5e20b
--- /dev/null
+++ b/src/service/cadet/cadet_api_get_path.c
@@ -0,0 +1,267 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api_get_path.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Operation handle.
+ */
+struct GNUNET_CADET_GetPath
+{
+  /**
+   * Monitor callback
+   */
+  GNUNET_CADET_PathCB path_cb;
+
+  /**
+   * Closure for @c path_cb.
+   */
+  void *path_cb_cls;
+
+  /**
+   * Message queue to talk to CADET service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Configuration we use.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Task to reconnect.
+   */
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
+
+  /**
+   * Backoff for reconnect attempts.
+   */
+  struct GNUNET_TIME_Relative backoff;
+
+  /**
+   * Peer we want information about.
+   */
+  struct GNUNET_PeerIdentity id;
+};
+
+
+/**
+ * Check that message received from CADET service is well-formed.
+ *
+ * @param cls unused
+ * @param message the message we got
+ * @return #GNUNET_OK if the message is well-formed,
+ *         #GNUNET_SYSERR otherwise
+ */
+static int
+check_get_path (void *cls,
+                const struct GNUNET_CADET_LocalInfoPath *message)
+{
+  size_t msize = sizeof(struct GNUNET_CADET_LocalInfoPath);
+  size_t esize;
+
+  (void) cls;
+  esize = ntohs (message->header.size);
+  if (esize < msize)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  if (0 != ((esize - msize) % sizeof(struct GNUNET_PeerIdentity)))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process a local peer info reply, pass info to the user.
+ *
+ * @param cls Closure
+ * @param message Message itself.
+ */
+static void
+handle_get_path (void *cls,
+                 const struct GNUNET_CADET_LocalInfoPath *message)
+{
+  struct GNUNET_CADET_GetPath *gp = cls;
+  struct GNUNET_CADET_PeerPathDetail ppd;
+
+  ppd.peer = gp->id;
+  ppd.path = (const struct GNUNET_PeerIdentity *) &message[1];
+  ppd.target_offset = ntohl (message->off);
+  ppd.path_length = (ntohs (message->header.size) - sizeof(*message))
+                    / sizeof(struct GNUNET_PeerIdentity);
+  gp->path_cb (gp->path_cb_cls,
+               &ppd);
+}
+
+
+/**
+ * Process a local peer info reply, pass info to the user.
+ *
+ * @param cls Closure
+ * @param message Message itself.
+ */
+static void
+handle_get_path_end (void *cls,
+                     const struct GNUNET_MessageHeader *message)
+{
+  struct GNUNET_CADET_GetPath *gp = cls;
+
+  (void) message;
+  gp->path_cb (gp->path_cb_cls,
+               NULL);
+  GNUNET_CADET_get_path_cancel (gp);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_GetPath` operation
+ */
+static void
+reconnect (void *cls);
+
+
+/**
+ * Function called on connection trouble.  Reconnects.
+ *
+ * @param cls a `struct GNUNET_CADET_GetPath`
+ * @param error error code from MQ
+ */
+static void
+error_handler (void *cls,
+               enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_GetPath *gp = cls;
+
+  GNUNET_MQ_destroy (gp->mq);
+  gp->mq = NULL;
+  gp->backoff = GNUNET_TIME_randomized_backoff (gp->backoff,
+                                                GNUNET_TIME_UNIT_MINUTES);
+  gp->reconnect_task = GNUNET_SCHEDULER_add_delayed (gp->backoff,
+                                                     &reconnect,
+                                                     gp);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_GetPath` operation
+ */
+static void
+reconnect (void *cls)
+{
+  struct GNUNET_CADET_GetPath *gp = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (get_path,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PATH,
+                           struct GNUNET_CADET_LocalInfoPath,
+                           gp),
+    GNUNET_MQ_hd_fixed_size (get_path_end,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PATH_END,
+                             struct GNUNET_MessageHeader,
+                             gp),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_CADET_RequestPathInfoMessage *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  gp->reconnect_task = NULL;
+  gp->mq = GNUNET_CLIENT_connect (gp->cfg,
+                                  "cadet",
+                                  handlers,
+                                  &error_handler,
+                                  gp);
+  if (NULL == gp->mq)
+    return;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PATH);
+  msg->peer = gp->id;
+  GNUNET_MQ_send (gp->mq,
+                  env);
+}
+
+
+struct GNUNET_CADET_GetPath *
+GNUNET_CADET_get_path (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                       const struct GNUNET_PeerIdentity *id,
+                       GNUNET_CADET_PathCB callback,
+                       void *callback_cls)
+{
+  struct GNUNET_CADET_GetPath *gp;
+
+  if (NULL == callback)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  gp = GNUNET_new (struct GNUNET_CADET_GetPath);
+  gp->path_cb = callback;
+  gp->path_cb_cls = callback_cls;
+  gp->cfg = cfg;
+  gp->id = *id;
+  reconnect (gp);
+  if (NULL == gp->mq)
+  {
+    GNUNET_free (gp);
+    return NULL;
+  }
+  return gp;
+}
+
+
+/**
+ * Cancel @a gp operation.
+ *
+ * @param gp operation to cancel
+ * @return closure from #GNUNET_CADET_get_path().
+ */
+void *
+GNUNET_CADET_get_path_cancel (struct GNUNET_CADET_GetPath *gp)
+{
+  void *ret = gp->path_cb_cls;
+
+  if (NULL != gp->mq)
+    GNUNET_MQ_destroy (gp->mq);
+  if (NULL != gp->reconnect_task)
+    GNUNET_SCHEDULER_cancel (gp->reconnect_task);
+  GNUNET_free (gp);
+  return ret;
+}
+
+
+/* end of cadet_api_get_path.c */
diff --git a/src/service/cadet/cadet_api_helper.c b/src/service/cadet/cadet_api_helper.c
new file mode 100644
index 000000000..52aec6cc9
--- /dev/null
+++ b/src/service/cadet/cadet_api_helper.c
@@ -0,0 +1,54 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api_helper.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Transitional function to convert an unsigned int port to a hash value.
+ * WARNING: local static value returned, NOT reentrant!
+ * WARNING: do not use this function for new code!
+ *
+ * @param port Numerical port (unsigned int format).
+ *
+ * @return A GNUNET_HashCode usable for the new CADET API.
+ */
+const struct GNUNET_HashCode *
+GC_u2h (uint32_t port)
+{
+  static struct GNUNET_HashCode hash;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "This is a transitional function, use proper crypto hashes as CADET ports\n");
+  GNUNET_CRYPTO_hash (&port,
+                      sizeof(port),
+                      &hash);
+  return &hash;
+}
diff --git a/src/service/cadet/cadet_api_list_peers.c b/src/service/cadet/cadet_api_list_peers.c
new file mode 100644
index 000000000..52f59cc74
--- /dev/null
+++ b/src/service/cadet/cadet_api_list_peers.c
@@ -0,0 +1,238 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017, 2019 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api_list_peers.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Operation handle.
+ */
+struct GNUNET_CADET_PeersLister
+{
+  /**
+   * Monitor callback
+   */
+  GNUNET_CADET_PeersCB peers_cb;
+
+  /**
+   * Info callback closure for @c info_cb.
+   */
+  void *peers_cb_cls;
+
+  /**
+   * Message queue to talk to CADET service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Configuration we use.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Task to reconnect.
+   */
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
+
+  /**
+   * Backoff for reconnect attempts.
+   */
+  struct GNUNET_TIME_Relative backoff;
+};
+
+
+/**
+ * Process a local reply about info on all tunnels, pass info to the user.
+ *
+ * @param cls a `struct GNUNET_CADET_PeersLister`
+ * @param info Message itself.
+ */
+static void
+handle_get_peers (void *cls,
+                  const struct GNUNET_CADET_LocalInfoPeers *info)
+{
+  struct GNUNET_CADET_PeersLister *pl = cls;
+  struct GNUNET_CADET_PeerListEntry ple;
+
+  ple.peer = info->destination;
+  ple.have_tunnel = (int) ntohs (info->tunnel);
+  ple.n_paths = (unsigned int) ntohs (info->paths);
+  ple.best_path_length = (unsigned int) ntohl (info->best_path_length);
+  pl->peers_cb (pl->peers_cb_cls,
+                &ple);
+}
+
+
+/**
+ * Process a end of list reply about info on all peers.
+ *
+ * @param cls a `struct GNUNET_CADET_PeersLister`
+ * @param msg Message itself.
+ */
+static void
+handle_get_peers_end (void *cls,
+                      const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_CADET_PeersLister *pl = cls;
+
+  (void) msg;
+
+  pl->peers_cb (pl->peers_cb_cls,
+                NULL);
+  GNUNET_CADET_list_peers_cancel (pl);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_PeersLister` operation
+ */
+static void
+reconnect (void *cls);
+
+
+/**
+ * Function called on connection trouble.  Reconnects.
+ *
+ * @param cls a `struct GNUNET_CADET_PeersLister`
+ * @param error error code from MQ
+ */
+static void
+error_handler (void *cls,
+               enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_PeersLister *pl = cls;
+
+  GNUNET_MQ_destroy (pl->mq);
+  pl->mq = NULL;
+  pl->backoff = GNUNET_TIME_randomized_backoff (pl->backoff,
+                                                GNUNET_TIME_UNIT_MINUTES);
+  pl->reconnect_task = GNUNET_SCHEDULER_add_delayed (pl->backoff,
+                                                     &reconnect,
+                                                     pl);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_PeersLister` operation
+ */
+static void
+reconnect (void *cls)
+{
+  struct GNUNET_CADET_PeersLister *pl = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (get_peers,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                             struct GNUNET_CADET_LocalInfoPeers,
+                             pl),
+    GNUNET_MQ_hd_fixed_size (get_peers_end,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS_END,
+                             struct GNUNET_MessageHeader,
+                             pl),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MessageHeader *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  pl->reconnect_task = NULL;
+  pl->mq = GNUNET_CLIENT_connect (pl->cfg,
+                                  "cadet",
+                                  handlers,
+                                  &error_handler,
+                                  pl);
+  if (NULL == pl->mq)
+    return;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PEERS);
+  GNUNET_MQ_send (pl->mq,
+                  env);
+}
+
+
+/**
+ * Request information about peers known to the running cadet service.
+ * The callback will be called for every peer known to the service.
+ * Only one info request (of any kind) can be active at once.
+ *
+ * @param cfg configuration to use
+ * @param callback Function to call with the requested data.
+ * @param callback_cls Closure for @c callback.
+ * @return NULL on error
+ */
+struct GNUNET_CADET_PeersLister *
+GNUNET_CADET_list_peers (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                         GNUNET_CADET_PeersCB callback,
+                         void *callback_cls)
+{
+  struct GNUNET_CADET_PeersLister *pl;
+
+  if (NULL == callback)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  pl = GNUNET_new (struct GNUNET_CADET_PeersLister);
+  pl->peers_cb = callback;
+  pl->peers_cb_cls = callback_cls;
+  pl->cfg = cfg;
+  reconnect (pl);
+  if (NULL == pl->mq)
+  {
+    GNUNET_free (pl);
+    return NULL;
+  }
+  return pl;
+}
+
+
+/**
+ * Cancel a peer info request. The callback will not be called (anymore).
+ *
+ * @param pl operation handle
+ * @return Closure given to GNUNET_CADET_get_peers().
+ */
+void *
+GNUNET_CADET_list_peers_cancel (struct GNUNET_CADET_PeersLister *pl)
+{
+  void *ret = pl->peers_cb_cls;
+
+  if (NULL != pl->mq)
+    GNUNET_MQ_destroy (pl->mq);
+  if (NULL != pl->reconnect_task)
+    GNUNET_SCHEDULER_cancel (pl->reconnect_task);
+  GNUNET_free (pl);
+  return ret;
+}
+
+
+/* end of cadet_api_list_peers.c */
diff --git a/src/service/cadet/cadet_api_list_tunnels.c b/src/service/cadet/cadet_api_list_tunnels.c
new file mode 100644
index 000000000..dce12835f
--- /dev/null
+++ b/src/service/cadet/cadet_api_list_tunnels.c
@@ -0,0 +1,239 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2017, 2019 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/cadet_api_list_tunnels.c
+ * @brief cadet api: client implementation of cadet service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_cadet_service.h"
+#include "cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Operation handle.
+ */
+struct GNUNET_CADET_ListTunnels
+{
+  /**
+   * Monitor callback
+   */
+  GNUNET_CADET_TunnelsCB tunnels_cb;
+
+  /**
+   * Info callback closure for @c tunnels_cb.
+   */
+  void *tunnels_cb_cls;
+
+  /**
+   * Message queue to talk to CADET service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Configuration we use.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Task to reconnect.
+   */
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
+
+  /**
+   * Backoff for reconnect attempts.
+   */
+  struct GNUNET_TIME_Relative backoff;
+};
+
+
+/**
+ * Process a local reply about info on all tunnels, pass info to the user.
+ *
+ * @param cls a `struct GNUNET_CADET_ListTunnels *`
+ * @param info Message itself.
+ */
+static void
+handle_get_tunnels (void *cls,
+                    const struct GNUNET_CADET_LocalInfoTunnel *info)
+{
+  struct GNUNET_CADET_ListTunnels *lt = cls;
+  struct GNUNET_CADET_TunnelDetails td;
+
+  td.peer = info->destination;
+  td.channels = ntohl (info->channels);
+  td.connections = ntohl (info->connections);
+  td.estate = ntohs (info->estate);
+  td.cstate = ntohs (info->cstate);
+  lt->tunnels_cb (lt->tunnels_cb_cls,
+                  &td);
+}
+
+
+/**
+ * Process a local reply about info on all tunnels, pass info to the user.
+ *
+ * @param cls a `struct GNUNET_CADET_ListTunnels *`
+ * @param msg Message itself.
+ */
+static void
+handle_get_tunnels_end (void *cls,
+                        const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_CADET_ListTunnels *lt = cls;
+
+  (void) msg;
+
+  lt->tunnels_cb (lt->tunnels_cb_cls,
+                  NULL);
+  GNUNET_CADET_list_tunnels_cancel (lt);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_ListTunnels` operation
+ */
+static void
+reconnect (void *cls);
+
+
+/**
+ * Function called on connection trouble.  Reconnects.
+ *
+ * @param cls a `struct GNUNET_CADET_ListTunnels`
+ * @param error error code from MQ
+ */
+static void
+error_handler (void *cls,
+               enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_CADET_ListTunnels *lt = cls;
+
+  GNUNET_MQ_destroy (lt->mq);
+  lt->mq = NULL;
+  lt->backoff = GNUNET_TIME_randomized_backoff (lt->backoff,
+                                                GNUNET_TIME_UNIT_MINUTES);
+  lt->reconnect_task = GNUNET_SCHEDULER_add_delayed (lt->backoff,
+                                                     &reconnect,
+                                                     lt);
+}
+
+
+/**
+ * Reconnect to the service and try again.
+ *
+ * @param cls a `struct GNUNET_CADET_ListTunnels` operation
+ */
+static void
+reconnect (void *cls)
+{
+  struct GNUNET_CADET_ListTunnels *lt = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (get_tunnels,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                             struct GNUNET_CADET_LocalInfoTunnel,
+                             lt),
+    GNUNET_MQ_hd_fixed_size (get_tunnels_end,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS_END,
+                             struct GNUNET_MessageHeader,
+                             lt),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MessageHeader *msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  lt->reconnect_task = NULL;
+  lt->mq = GNUNET_CLIENT_connect (lt->cfg,
+                                  "cadet",
+                                  handlers,
+                                  &error_handler,
+                                  lt);
+  if (NULL == lt->mq)
+    return;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_TUNNELS);
+  GNUNET_MQ_send (lt->mq,
+                  env);
+}
+
+
+/**
+ * Request information about tunnels of the running cadet peer.
+ * The callback will be called for every tunnel of the service.
+ * Only one info request (of any kind) can be active at once.
+ *
+ * @param cfg configuration to use
+ * @param callback Function to call with the requested data.
+ * @param callback_cls Closure for @c callback.
+ * @return NULL on error
+ */
+struct GNUNET_CADET_ListTunnels *
+GNUNET_CADET_list_tunnels (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                           GNUNET_CADET_TunnelsCB callback,
+                           void *callback_cls)
+{
+  struct GNUNET_CADET_ListTunnels *lt;
+
+  if (NULL == callback)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  lt = GNUNET_new (struct GNUNET_CADET_ListTunnels);
+  lt->tunnels_cb = callback;
+  lt->tunnels_cb_cls = callback_cls;
+  lt->cfg = cfg;
+  reconnect (lt);
+  if (NULL == lt->mq)
+  {
+    GNUNET_free (lt);
+    return NULL;
+  }
+  return lt;
+}
+
+
+/**
+ * Cancel a monitor request. The monitor callback will not be called.
+ *
+ * @param lt operation handle
+ * @return Closure given to GNUNET_CADET_list_tunnels().
+ */
+void *
+GNUNET_CADET_list_tunnels_cancel (struct GNUNET_CADET_ListTunnels *lt)
+{
+  void *ret = lt->tunnels_cb_cls;
+
+  if (NULL != lt->mq)
+    GNUNET_MQ_destroy (lt->mq);
+  if (NULL != lt->reconnect_task)
+    GNUNET_SCHEDULER_cancel (lt->reconnect_task);
+  GNUNET_free (lt);
+  return ret;
+}
+
+
+/* end of cadet_api_list_tunnels.c */
diff --git a/src/service/cadet/cadet_protocol.h b/src/service/cadet/cadet_protocol.h
new file mode 100644
index 000000000..117a7922e
--- /dev/null
+++ b/src/service/cadet/cadet_protocol.h
@@ -0,0 +1,541 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2007 - 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/cadet_protocol.h
+ * @brief P2P messages used by CADET
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+
+#ifndef CADET_PROTOCOL_H_
+#define CADET_PROTOCOL_H_
+
+/**
+ * At best, enable when debugging #5328!
+ */
+#define DEBUG_KX 0
+#if DEBUG_KX
+#warning NEVER run this in production! KX debugging is on!
+#endif
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "cadet.h"
+
+#ifdef __cplusplus
+
+struct GNUNET_CADET_TunnelMessage;
+extern "C"
+{
+#if 0
+/* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+/******************************************************************************/
+/********************      CADET NETWORK MESSAGES     **************************/
+/******************************************************************************/
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+
+/******************************************************************************/
+/*****************************   CONNECTION  **********************************/
+/******************************************************************************/
+
+
+/**
+ * Message for cadet connection creation.
+ */
+struct GNUNET_CADET_ConnectionCreateMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE
+   *
+   * Size: sizeof (struct GNUNET_CADET_ConnectionCreateMessage) +
+   *       path_length * sizeof (struct GNUNET_PeerIdentity)
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Connection options in network byte order.
+   * #GNUNET_CADET_OPTION_DEFAULT for buffered;
+   * #GNUNET_CADET_OPTION_NOBUFFER for unbuffered.
+   * Other flags are ignored and should not be set at this level.
+   */
+  uint32_t options GNUNET_PACKED;
+
+  /**
+   * This flag indicates the peer sending the connection create
+   * meassage likes to trigger a KX handshake.
+   */
+  int has_monotime;
+
+  /**
+   *  This monotonic time is set, if a peer likes to trigger a KX, but is not
+   *  the peer that should start the KX. (xrs,t3ss)
+   */
+  struct GNUNET_TIME_AbsoluteNBO monotime;
+
+  /**
+   *  We sign the monotime. The receiving peer can check the signature, to verify
+   *  the sending peer.
+   */
+  struct GNUNET_CRYPTO_EddsaSignature monotime_sig;
+
+  /**
+   * ID of the connection
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * path_length structs defining the *whole* path from the origin [0] to the
+   * final destination [path_length-1].
+   */
+  /* struct GNUNET_PeerIdentity peers[path_length]; */
+};
+
+
+/**
+ * Message for ack'ing a connection
+ */
+struct GNUNET_CADET_ConnectionCreateAckMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+};
+
+
+/**
+ * Message for notifying a disconnection in a path
+ */
+struct GNUNET_CADET_ConnectionBrokenMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * ID of the endpoint
+   */
+  struct GNUNET_PeerIdentity peer1;
+
+  /**
+   * ID of the endpoint
+   */
+  struct GNUNET_PeerIdentity peer2;
+};
+
+
+/**
+ * Message to destroy a connection.
+ */
+struct GNUNET_CADET_ConnectionDestroyMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+};
+
+
+/******************************************************************************/
+/*******************************   TUNNEL   ***********************************/
+/******************************************************************************/
+
+/**
+ * Unique identifier (counter) for an encrypted message in a channel.
+ * Used to match #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK
+ * and  #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL messages
+ * against the respective  #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED
+ * messages.
+ */
+struct CadetEncryptedMessageIdentifier
+{
+  /**
+   * This number is incremented by one per message. It may wrap around.
+   * In network byte order.
+   */
+  uint32_t pid GNUNET_PACKED;
+};
+
+
+/**
+ * Flags to be used in GNUNET_CADET_KX.
+ */
+enum GNUNET_CADET_KX_Flags
+{
+  /**
+   * Should the peer reply with its KX details?
+   */
+  GNUNET_CADET_KX_FLAG_NONE = 0,
+
+  /**
+   * The peer should reply with its KX details?
+   */
+  GNUNET_CADET_KX_FLAG_FORCE_REPLY = 1
+};
+
+
+/**
+ * Message for a Key eXchange for a tunnel.
+ */
+struct GNUNET_CADET_TunnelKeyExchangeMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX or
+   * #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH as part
+   * of `struct GNUNET_CADET_TunnelKeyExchangeAuthMessage`.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Flags for the key exchange in NBO, based on
+   * `enum GNUNET_CADET_KX_Flags`.
+   */
+  uint32_t flags GNUNET_PACKED;
+
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * Sender's ephemeral public ECC key encoded in a
+   * format suitable for network transmission, as created
+   * using 'gcry_sexp_sprint'.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey ephemeral_key;
+
+#if DEBUG_KX
+  /**
+   * Sender's ephemeral public ECC key encoded in a
+   * format suitable for network transmission, as created
+   * using 'gcry_sexp_sprint'.
+   */
+  struct GNUNET_CRYPTO_EcdhePrivateKey ephemeral_key_XXX; // for debugging KX-crypto!
+
+  /**
+   * Sender's ephemeral public ECC key encoded in a
+   * format suitable for network transmission, as created
+   * using 'gcry_sexp_sprint'.
+   */
+  struct GNUNET_CRYPTO_EddsaPrivateKey private_key_XXX; // for debugging KX-crypto!
+#endif
+
+  /**
+   * Sender's next ephemeral public ECC key encoded in a
+   * format suitable for network transmission, as created
+   * using 'gcry_sexp_sprint'.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey ratchet_key;
+};
+
+
+/**
+ * Message for a Key eXchange for a tunnel, with authentication.
+ * Used as a response to the initial KX as well as for rekeying.
+ */
+struct GNUNET_CADET_TunnelKeyExchangeAuthMessage
+{
+  /**
+   * Message header with key material.
+   */
+  struct GNUNET_CADET_TunnelKeyExchangeMessage kx;
+
+#if DEBUG_KX
+  /**
+   * Received ephemeral public ECC key encoded in a
+   * format suitable for network transmission, as created
+   * using 'gcry_sexp_sprint'.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey r_ephemeral_key_XXX; // for debugging KX-crypto!
+#endif
+
+  /**
+   * KDF-proof that sender could compute the 3-DH, used in lieu of a
+   * signature or payload data.
+   */
+  struct GNUNET_HashCode auth;
+};
+
+
+/**
+ * Encrypted axolotl header with numbers that identify which
+ * keys in which ratchet are to be used to decrypt the body.
+ */
+struct GNUNET_CADET_AxHeader
+{
+  /**
+   * Number of messages sent with the current ratchet key.
+   */
+  uint32_t Ns GNUNET_PACKED;
+
+  /**
+   * Number of messages sent with the previous ratchet key.
+   */
+  uint32_t PNs GNUNET_PACKED;
+
+  /**
+   * Current ratchet key.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey DHRs;
+};
+
+
+/**
+ * Axolotl-encrypted tunnel message with application payload.
+ */
+struct GNUNET_CADET_TunnelEncryptedMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Reserved, for alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * MAC of the encrypted message, used to verify message integrity.
+   * Everything after this value  will be encrypted with the header key
+   * and authenticated.
+   */
+  struct GNUNET_ShortHashCode hmac;
+
+  /**
+   * Axolotl-header that specifies which keys to use in which ratchet
+   * to decrypt the body that follows.
+   */
+  struct GNUNET_CADET_AxHeader ax_header;
+
+  /**
+   * Encrypted content follows.
+   */
+};
+
+
+/******************************************************************************/
+/*******************************   CHANNEL  ***********************************/
+/******************************************************************************/
+
+
+/**
+ * Message to create a Channel.
+ */
+struct GNUNET_CADET_ChannelOpenMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Channel options.
+   */
+  uint32_t opt GNUNET_PACKED;
+
+  /**
+   * Hash of destination port and listener.
+   */
+  struct GNUNET_HashCode h_port;
+
+  /**
+   * ID of the channel within the tunnel.
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+};
+
+
+/**
+ * Message to acknowledge opening a channel of type
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
+ */
+struct GNUNET_CADET_ChannelOpenAckMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the channel
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+
+  /**
+   * Port number of the channel, used to prove to the
+   * initiator that the receiver knows the port.
+   */
+  struct GNUNET_HashCode port;
+};
+
+
+/**
+ * Message to destroy a channel of type
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY.
+ */
+struct GNUNET_CADET_ChannelDestroyMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of the channel
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+};
+
+
+/**
+ * Number used to uniquely identify messages in a CADET Channel.
+ */
+struct ChannelMessageIdentifier
+{
+  /**
+   * Unique ID of the message, cycles around, in NBO.
+   */
+  uint32_t mid GNUNET_PACKED;
+};
+
+
+/**
+ * Message for cadet data traffic.
+ */
+struct GNUNET_CADET_ChannelAppDataMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Unique ID of the payload message.
+   */
+  struct ChannelMessageIdentifier mid;
+
+  /**
+   * ID of the channel
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+
+  /**
+   * Payload follows
+   */
+};
+
+
+/**
+ * Message to acknowledge end-to-end data.
+ */
+struct GNUNET_CADET_ChannelDataAckMessage
+{
+  /**
+   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * ID of the channel
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+
+  /**
+   * Bitfield of already-received newer messages.  Note that bit 0
+   * corresponds to @e mid + 1.
+   *
+   * pid +  0 @ LSB
+   * pid + 63 @ MSB
+   */
+  uint64_t futures GNUNET_PACKED;
+
+  /**
+   * Next message ID expected.
+   */
+  struct ChannelMessageIdentifier mid;
+};
+
+
+GNUNET_NETWORK_STRUCT_END
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+/* ifndef CADET_PROTOCOL_H */
+#endif
+/* end of cadet_protocol.h */
diff --git a/src/service/cadet/desirability_table.c b/src/service/cadet/desirability_table.c
new file mode 100644
index 000000000..b4db7c2dd
--- /dev/null
+++ b/src/service/cadet/desirability_table.c
@@ -0,0 +1,34 @@
+/* This file is in the public domain. */
+
+/**
+ * @brief Program to simulate results from #GCP_get_desirability_of_path()
+ * for various plausible inputs.
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include <stdio.h>
+
+int
+main ()
+{
+  for (unsigned int num_alts = 1; num_alts < 10; num_alts++)
+    for (unsigned int off = 0; off < 10; off++)
+      for (double delta = -(int) off; delta <= 5; delta += 0.25)
+      {
+        double weight_alts;
+
+        if (delta <= -1.0)
+          weight_alts = -1.0 * num_alts / delta;   /* discount alternative paths */
+        else if (delta >= 1.0)
+          weight_alts = 1.0 * num_alts * delta;   /* overcount alternative paths */
+        else
+          weight_alts = 1.0 * num_alts;   /* count alternative paths normally */
+
+        fprintf (stderr,
+                 "Paths: %u  Offset: %u  Delta: %5.2f  SCORE: %f\n",
+                 num_alts,
+                 off,
+                 delta,
+                 ((off + 1.0) / (weight_alts * weight_alts)));
+      }
+}
diff --git a/src/service/cadet/gnunet-service-cadet.c b/src/service/cadet/gnunet-service-cadet.c
new file mode 100644
index 000000000..620e43cc8
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet.c
@@ -0,0 +1,1376 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2013, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet.c
+ * @brief GNUnet CADET service with encryption
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * Dictionary:
+ * - peer: other cadet instance. If there is direct connection it's a neighbor.
+ * - path: series of directly connected peer from one peer to another.
+ * - connection: path which is being used in a tunnel.
+ * - tunnel: encrypted connection to a peer, neighbor or not.
+ * - channel: logical link between two clients, on the same or different peers.
+ *            have properties like reliability.
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "cadet.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_transport_application_service.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_core.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet_constants.h"
+
+
+#define LOG(level, ...) GNUNET_log (level, __VA_ARGS__)
+
+
+/**
+ * Struct containing information about a client of the service
+ */
+struct CadetClient
+{
+  /**
+   * Linked list next
+   */
+  struct CadetClient *next;
+
+  /**
+   * Linked list prev
+   */
+  struct CadetClient *prev;
+
+  /**
+   * Tunnels that belong to this client, indexed by local id,
+   * value is a `struct CadetChannel`.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
+
+  /**
+   * Handle to communicate with the client
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Ports that this client has declared interest in.
+   * Indexed by port, contains `struct OpenPort`
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *ports;
+
+  /**
+   * Channel ID to use for the next incoming channel for this client.
+   * Wraps around (in theory).
+   */
+  struct GNUNET_CADET_ClientChannelNumber next_ccn;
+
+  /**
+   * ID of the client, mainly for debug messages. Purely internal to this file.
+   */
+  unsigned int id;
+};
+
+
+/******************************************************************************/
+/***********************      GLOBAL VARIABLES     ****************************/
+/******************************************************************************/
+
+/****************************** Global variables ******************************/
+
+/**
+ * Handle to our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Handle to the statistics service.
+ */
+struct GNUNET_STATISTICS_Handle *stats;
+
+/**
+ * Handle to Transport service.
+ */
+struct GNUNET_TRANSPORT_ApplicationHandle *transport;
+
+/**
+ * Local peer own ID.
+ */
+struct GNUNET_PeerIdentity my_full_id;
+
+/**
+ * Own private key.
+ */
+struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
+
+/**
+ * Signal that shutdown is happening: prevent recovery measures.
+ */
+int shutting_down;
+
+/**
+ * DLL with all the clients, head.
+ */
+static struct CadetClient *clients_head;
+
+/**
+ * DLL with all the clients, tail.
+ */
+static struct CadetClient *clients_tail;
+
+/**
+ * Next ID to assign to a client.
+ */
+static unsigned int next_client_id;
+
+/**
+ * All ports clients of this peer have opened.  Maps from
+ * a hashed port to a `struct OpenPort`.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *open_ports;
+
+/**
+ * Map from ports to channels where the ports were closed at the
+ * time we got the inbound connection.
+ * Indexed by h_port, contains `struct CadetChannel`.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
+
+/**
+ * Map from PIDs to `struct CadetPeer` entries.
+ */
+struct GNUNET_CONTAINER_MultiPeerMap *peers;
+
+/**
+ * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
+ * hash codes to `struct CadetConnection` objects.
+ */
+struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+/**
+ * How many messages are needed to trigger an AXOLOTL ratchet advance.
+ */
+unsigned long long ratchet_messages;
+
+/**
+ * How long until we trigger a ratched advance due to time.
+ */
+struct GNUNET_TIME_Relative ratchet_time;
+
+/**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+struct GNUNET_TIME_Relative keepalive_period;
+
+/**
+ * Set to non-zero values to create random drops to test retransmissions.
+ */
+unsigned long long drop_percent;
+
+
+/**
+ * Send a message to a client.
+ *
+ * @param c client to get the message
+ * @param env envelope with the message
+ */
+void
+GSC_send_to_client (struct CadetClient *c,
+                    struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_MQ_send (c->mq,
+                  env);
+}
+
+
+/**
+ * Return identifier for a client as a string.
+ *
+ * @param c client to identify
+ * @return string for debugging
+ */
+const char *
+GSC_2s (struct CadetClient *c)
+{
+  static char buf[32];
+
+  GNUNET_snprintf (buf,
+                   sizeof(buf),
+                   "Client(%u)",
+                   c->id);
+  return buf;
+}
+
+
+/**
+ * Lookup channel of client @a c by @a ccn.
+ *
+ * @param c client to look in
+ * @param ccn channel ID to look up
+ * @return NULL if no such channel exists
+ */
+static struct CadetChannel *
+lookup_channel (struct CadetClient *c,
+                struct GNUNET_CADET_ClientChannelNumber ccn)
+{
+  return GNUNET_CONTAINER_multihashmap32_get (c->channels,
+                                              ntohl (ccn.channel_of_client));
+}
+
+
+/**
+ * Obtain the next LID to use for incoming connections to
+ * the given client.
+ *
+ * @param c client handle
+ */
+static struct GNUNET_CADET_ClientChannelNumber
+client_get_next_ccn (struct CadetClient *c)
+{
+  struct GNUNET_CADET_ClientChannelNumber ccn = c->next_ccn;
+
+  /* increment until we have a free one... */
+  while (NULL !=
+         lookup_channel (c,
+                         ccn))
+  {
+    ccn.channel_of_client
+      = htonl (1 + (ntohl (ccn.channel_of_client)));
+    if (ntohl (ccn.channel_of_client) >=
+        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+      ccn.channel_of_client = htonl (0);
+  }
+  c->next_ccn.channel_of_client
+    = htonl (1 + (ntohl (ccn.channel_of_client)));
+  return ccn;
+}
+
+
+/**
+ * Bind incoming channel to this client, and notify client about
+ * incoming connection.  Caller is responsible for notifying the other
+ * peer about our acceptance of the channel.
+ *
+ * @param c client to bind to
+ * @param ch channel to be bound
+ * @param dest peer that establishes the connection
+ * @param port port number
+ * @param options options
+ * @return local channel number assigned to the new client
+ */
+struct GNUNET_CADET_ClientChannelNumber
+GSC_bind (struct CadetClient *c,
+          struct CadetChannel *ch,
+          struct CadetPeer *dest,
+          const struct GNUNET_HashCode *port,
+          uint32_t options)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelCreateMessage *cm;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  ccn = client_get_next_ccn (c);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (
+                                                        ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Accepting incoming %s from %s on open port %s (%u), assigning ccn %X\n",
+       GCCH_2s (ch),
+       GCP_2s (dest),
+       GNUNET_h2s (port),
+       (uint32_t) ntohl (options),
+       (uint32_t) ntohl (ccn.channel_of_client));
+  /* notify local client about incoming connection! */
+  env = GNUNET_MQ_msg (cm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  cm->ccn = ccn;
+  cm->port = *port;
+  cm->opt = htonl (options);
+  cm->peer = *GCP_get_id (dest);
+  GSC_send_to_client (c,
+                      env);
+  return ccn;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_tunnels_now (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
+{
+  struct CadetPeer *cp = value;
+  struct CadetTunnel *t = GCP_get_tunnel (cp,
+                                          GNUNET_NO);
+
+  if (NULL != t)
+    GCT_destroy_tunnel_now (t);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_paths_now (void *cls,
+                   const struct GNUNET_PeerIdentity *pid,
+                   void *value)
+{
+  struct CadetPeer *cp = value;
+
+  GCP_drop_owned_paths (cp);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Shutdown everything once the clients have disconnected.
+ */
+static void
+shutdown_rest ()
+{
+  if (NULL != stats)
+  {
+    GNUNET_STATISTICS_destroy (stats,
+                               GNUNET_NO);
+    stats = NULL;
+  }
+  /* Destroy tunnels.  Note that all channels must be destroyed first! */
+  GCP_iterate_all (&destroy_tunnels_now,
+                   NULL);
+  /* All tunnels, channels, connections and CORE must be down before this point. */
+  GCP_iterate_all (&destroy_paths_now,
+                   NULL);
+  /* All paths, tunnels, channels, connections and CORE must be down before this point. */
+  GCP_destroy_all_peers ();
+  if (NULL != open_ports)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (open_ports);
+    open_ports = NULL;
+  }
+  if (NULL != loose_channels)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (loose_channels);
+    loose_channels = NULL;
+  }
+  if (NULL != peers)
+  {
+    GNUNET_CONTAINER_multipeermap_destroy (peers);
+    peers = NULL;
+  }
+  if (NULL != connections)
+  {
+    GNUNET_CONTAINER_multishortmap_destroy (connections);
+    connections = NULL;
+  }
+  if (NULL != transport)
+  {
+    GNUNET_TRANSPORT_application_done (transport);
+    transport = NULL;
+  }
+  GCD_shutdown ();
+  GCH_shutdown ();
+  GNUNET_free (my_private_key);
+  my_private_key = NULL;
+}
+
+
+/**
+ * Task run during shutdown.
+ *
+ * @param cls unused
+ */
+static void
+shutdown_task (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Shutting down\n");
+  shutting_down = GNUNET_YES;
+  GCO_shutdown ();
+  if (NULL == clients_head)
+    shutdown_rest ();
+}
+
+
+/**
+ * We had a remote connection @a value to port @a h_port before
+ * client @a cls opened port @a port.  Bind them now.
+ *
+ * @param cls the `struct CadetClient`
+ * @param port the hashed port
+ * @param value the `struct CadetChannel`
+ * @return #GNUNET_YES (iterate over all such channels)
+ */
+static int
+bind_loose_channel (void *cls,
+                    const struct GNUNET_HashCode *port,
+                    void *value)
+{
+  struct OpenPort *op = cls;
+  struct CadetChannel *ch = value;
+
+  GCCH_bind (ch,
+             op->c,
+             &op->port);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       &op->h_port,
+                                                       ch));
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handle port open request.  Creates a mapping from the
+ * port to the respective client and checks whether we have
+ * loose channels trying to bind to the port.  If so, those
+ * are bound.
+ *
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
+ */
+static void
+handle_port_open (void *cls,
+                  const struct GNUNET_CADET_PortMessage *pmsg)
+{
+  struct CadetClient *c = cls;
+  struct OpenPort *op;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Open port %s requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  if (NULL == c->ports)
+    c->ports = GNUNET_CONTAINER_multihashmap_create (4,
+                                                     GNUNET_NO);
+  op = GNUNET_new (struct OpenPort);
+  op->c = c;
+  op->port = pmsg->port;
+  GCCH_hash_port (&op->h_port,
+                  &pmsg->port,
+                  &my_full_id);
+  if (GNUNET_OK !=
+      GNUNET_CONTAINER_multihashmap_put (c->ports,
+                                         &op->port,
+                                         op,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  (void) GNUNET_CONTAINER_multihashmap_put (open_ports,
+                                            &op->h_port,
+                                            op,
+                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  GNUNET_CONTAINER_multihashmap_get_multiple (loose_channels,
+                                              &op->h_port,
+                                              &bind_loose_channel,
+                                              op);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for port close requests.  Marks this port as closed
+ * (unless of course we have another client with the same port
+ * open).  Note that existing channels accepted on the port are
+ * not affected.
+ *
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
+ */
+static void
+handle_port_close (void *cls,
+                   const struct GNUNET_CADET_PortMessage *pmsg)
+{
+  struct CadetClient *c = cls;
+  struct OpenPort *op;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s as requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  if (NULL == c->ports)
+  {
+    /* Client closed a port despite _never_ having opened one? */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  op = GNUNET_CONTAINER_multihashmap_get (c->ports,
+                                          &pmsg->port);
+  if (NULL == op)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                                       &op->port,
+                                                       op));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       &op->h_port,
+                                                       op));
+  GNUNET_free (op);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for requests for us creating a new channel to another peer and port.
+ *
+ * @param cls Identification of the client.
+ * @param tcm The actual message.
+ */
+static void
+handle_channel_create (void *cls,
+                       const struct GNUNET_CADET_LocalChannelCreateMessage *tcm)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  if (ntohl (tcm->ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    /* Channel ID not in allowed range. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,"Channel ID not in allowed range.");
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  ch = lookup_channel (c,
+                       tcm->ccn);
+  if (NULL != ch)
+  {
+    /* Channel ID already in use. Not allowed. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,"Channel ID already in use. Not allowed.");
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "New channel to %s at port %s requested by %s\n",
+       GNUNET_i2s (&tcm->peer),
+       GNUNET_h2s (&tcm->port),
+       GSC_2s (c));
+
+  /* Create channel */
+  ch = GCCH_channel_local_new (c,
+                               tcm->ccn,
+                               GCP_get (&tcm->peer,
+                                        GNUNET_YES),
+                               &tcm->port,
+                               ntohl (tcm->opt));
+  if (NULL == ch)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (
+                                                        tcm->ccn.
+                                                        channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for requests of destroying an existing channel.
+ *
+ * @param cls client identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_channel_destroy (void *cls,
+                        const struct
+                        GNUNET_CADET_LocalChannelDestroyMessage *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Client attempted to destroy unknown channel.
+       Can happen if the other side went down at the same time.*/
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s tried to destroy unknown channel %X\n",
+         GSC_2s (c),
+         (uint32_t) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is destroying %s\n",
+       GSC_2s (c),
+       GCCH_2s (ch));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (
+                                                           msg->ccn.
+                                                           channel_of_client),
+                                                         ch));
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Check for client traffic data message is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is OK, #GNUNET_SYSERR if not
+ */
+static int
+check_local_data (void *cls,
+                  const struct GNUNET_CADET_LocalData *msg)
+{
+  size_t payload_size;
+  size_t payload_claimed_size;
+  const char *buf;
+  struct GNUNET_MessageHeader pa;
+
+  /* FIXME: what is the format we shall allow for @a msg?
+     ONE payload item or multiple? Seems current cadet_api
+     at least in theory allows more than one. Next-gen
+     cadet_api will likely no more, so we could then
+     simplify this mess again. *//* Sanity check for message size */payload_size = ntohs (msg->header.size) - sizeof(*msg);
+  buf = (const char *) &msg[1];
+  while (payload_size >= sizeof(struct GNUNET_MessageHeader))
+  {
+    /* need to memcpy() for alignment */
+    GNUNET_memcpy (&pa,
+                   buf,
+                   sizeof(pa));
+    payload_claimed_size = ntohs (pa.size);
+    if ((payload_size < payload_claimed_size) ||
+        (payload_claimed_size < sizeof(struct GNUNET_MessageHeader)) ||
+        (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size))
+    {
+      GNUNET_break (0);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Local data of %u total size had sub-message %u at %u with %u bytes\n",
+           ntohs (msg->header.size),
+           ntohs (pa.type),
+           (unsigned int) (buf - (const char *) &msg[1]),
+           (unsigned int) payload_claimed_size);
+      return GNUNET_SYSERR;
+    }
+    payload_size -= payload_claimed_size;
+    buf += payload_claimed_size;
+  }
+  if (0 != payload_size)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handler for client payload traffic to be send on a channel to
+ * another peer.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_local_data (void *cls,
+                   const struct GNUNET_CADET_LocalData *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+  size_t payload_size;
+  const char *buf;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Dropping payload for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof(*msg);
+  GNUNET_STATISTICS_update (stats,
+                            "# payload received from clients",
+                            payload_size,
+                            GNUNET_NO);
+  buf = (const char *) &msg[1];
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received %u bytes payload from %s for %s\n",
+       (unsigned int) payload_size,
+       GSC_2s (c),
+       GCCH_2s (ch));
+  if (GNUNET_OK !=
+      GCCH_handle_local_data (ch,
+                              msg->ccn,
+                              buf,
+                              payload_size))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for client's ACKs for payload traffic.
+ *
+ * @param cls identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_local_ack (void *cls,
+                  const struct GNUNET_CADET_LocalAck *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Ignoring local ACK for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got a local ACK from %s for %s\n",
+       GSC_2s (c),
+       GCCH_2s (ch));
+  GCCH_handle_local_ack (ch,
+                         msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all peers to send a monitoring client info about each peer.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value Peer info.
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_peers_iterator (void *cls,
+                        const struct GNUNET_PeerIdentity *peer,
+                        void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoPeers *msg;
+
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  msg->destination = *peer;
+  msg->paths = htons (GCP_count_paths (p));
+  msg->tunnel = htons (NULL != GCP_get_tunnel (p,
+                                               GNUNET_NO));
+  msg->best_path_length = htonl (0);  // FIXME: get length of shortest known path!
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's INFO PEERS request.
+ *
+ * @param cls Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_get_peers (void *cls,
+                  const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_peers_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS_END);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all paths of a peer to build an InfoPeer message.
+ * Message contains blocks of peers, first not included.
+ *
+ * @param cls message queue for transmission
+ * @param path Path itself
+ * @param off offset of the peer on @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
+ */
+static int
+path_info_iterator (void *cls,
+                    struct CadetPeerPath *path,
+                    unsigned int off)
+{
+  struct GNUNET_MQ_Handle *mq = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoPath *resp;
+  struct GNUNET_PeerIdentity *id;
+  size_t path_size;
+  unsigned int path_length;
+
+  path_length = GCPP_get_length (path);
+  path_size = sizeof(struct GNUNET_PeerIdentity) * path_length;
+  if (sizeof(*resp) + path_size > UINT16_MAX)
+  {
+    /* try just giving the relevant path */
+    path_length = GNUNET_MIN ((UINT16_MAX - sizeof(*resp)) / sizeof(struct
+                                                                    GNUNET_PeerIdentity),
+                              off);
+    path_size = sizeof(struct GNUNET_PeerIdentity) * path_length;
+  }
+  if (sizeof(*resp) + path_size > UINT16_MAX)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Path of %u entries is too long for info message\n",
+         path_length);
+    return GNUNET_YES;
+  }
+  env = GNUNET_MQ_msg_extra (resp,
+                             path_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PATH);
+  id = (struct GNUNET_PeerIdentity *) &resp[1];
+
+  /* Don't copy first peer.  First peer is always the local one.  Last
+   * peer is always the destination (leave as 0, EOL).
+   */
+  for (unsigned int i = 0; i < path_length; i++)
+    id[i] = *GCP_get_id (GCPP_get_peer_at_offset (path,
+                                                  i));
+  resp->off = htonl (off);
+  GNUNET_MQ_send (mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PATH request.
+ *
+ * @param cls Identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_show_path (void *cls,
+                  const struct GNUNET_CADET_RequestPathInfoMessage *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *resp;
+
+  p = GCP_get (&msg->peer,
+               GNUNET_NO);
+  if (NULL != p)
+    GCP_iterate_indirect_paths (p,
+                                &path_info_iterator,
+                                c->mq);
+  env = GNUNET_MQ_msg (resp,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PATH_END);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all tunnels to send a monitoring client info about each tunnel.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value a `struct CadetPeer`
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_tunnels_iterator (void *cls,
+                          const struct GNUNET_PeerIdentity *peer,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoTunnel *msg;
+  struct CadetTunnel *t;
+
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL == t)
+    return GNUNET_YES;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  msg->destination = *peer;
+  msg->channels = htonl (GCT_count_channels (t));
+  msg->connections = htonl (GCT_count_any_connections (t));
+  msg->cstate = htons (0);
+  msg->estate = htons ((uint16_t) GCT_get_estate (t));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_TUNNELS request.
+ *
+ * @param cls client Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_info_tunnels (void *cls,
+                     const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_tunnels_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS_END);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_DROP_CADET_MESSAGE request.
+ *
+ * @param cls client Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_drop_message (void *cls,
+                     const struct GNUNET_CADET_RequestDropCadetMessage *message)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       message->ccn);
+
+  if (NULL != ch)
+    GCCH_assign_type_to_drop (ch, message);
+
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param client the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *client,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct CadetClient *c;
+
+  c = GNUNET_new (struct CadetClient);
+  c->client = client;
+  c->mq = mq;
+  c->id = next_client_id++; /* overflow not important: just for debug */
+  c->channels
+    = GNUNET_CONTAINER_multihashmap32_create (32);
+  GNUNET_CONTAINER_DLL_insert (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            +1,
+                            GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s connected\n",
+       GSC_2s (c));
+  return c;
+}
+
+
+/**
+ * A channel was destroyed by the other peer. Tell our client.
+ *
+ * @param c client that lost a channel
+ * @param ccn channel identification number for the client
+ * @param ch the channel object
+ */
+void
+GSC_handle_remote_channel_destroy (struct CadetClient *c,
+                                   struct GNUNET_CADET_ClientChannelNumber ccn,
+                                   struct CadetChannel *ch)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelDestroyMessage *tdm;
+
+  env = GNUNET_MQ_msg (tdm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+  tdm->ccn = ccn;
+  GSC_send_to_client (c,
+                      env);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (
+                                                           ccn.channel_of_client),
+                                                         ch));
+}
+
+
+void
+GSC_drop_loose_channel (const struct GNUNET_HashCode *h_port,
+                        struct CadetChannel *ch)
+{
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       h_port,
+                                                       ch));
+}
+
+
+/**
+ * Iterator for deleting each channel whose client endpoint disconnected.
+ *
+ * @param cls Closure (client that has disconnected).
+ * @param key The local channel id in host byte order
+ * @param value The value stored at the key (channel to destroy).
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+channel_destroy_iterator (void *cls,
+                          uint32_t key,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+  struct CadetChannel *ch = value;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying %s, due to %s disconnecting.\n",
+       GCCH_2s (ch),
+       GSC_2s (c));
+  ccn.channel_of_client = htonl (key);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         key,
+                                                         ch));
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              ccn);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Remove client's ports from the global hashmap on disconnect.
+ *
+ * @param cls the `struct CadetClient`
+ * @param port the port.
+ * @param value the `struct OpenPort` to remove
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+client_release_ports (void *cls,
+                      const struct GNUNET_HashCode *port,
+                      void *value)
+{
+  struct CadetClient *c = cls;
+  struct OpenPort *op = value;
+
+  GNUNET_assert (c == op->c);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s due to %s disconnect.\n",
+       GNUNET_h2s (port),
+       GSC_2s (c));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       &op->h_port,
+                                                       op));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                                       port,
+                                                       op));
+  GNUNET_free (op);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param client the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
+{
+  struct CadetClient *c = internal_cls;
+
+  GNUNET_assert (c->client == client);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is disconnecting.\n",
+       GSC_2s (c));
+  if (NULL != c->channels)
+  {
+    GNUNET_CONTAINER_multihashmap32_iterate (c->channels,
+                                             &channel_destroy_iterator,
+                                             c);
+    GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (c->channels));
+    GNUNET_CONTAINER_multihashmap32_destroy (c->channels);
+  }
+  if (NULL != c->ports)
+  {
+    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
+                                           &client_release_ports,
+                                           c);
+    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
+  }
+  GNUNET_CONTAINER_DLL_remove (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            -1,
+                            GNUNET_NO);
+  GNUNET_free (c);
+  if ((NULL == clients_head) &&
+      (GNUNET_YES == shutting_down))
+    shutdown_rest ();
+}
+
+
+/**
+ * Setup CADET internals.
+ *
+ * @param cls closure
+ * @param c configuration to use
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
+{
+  cfg = c;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "RATCHET_MESSAGES",
+                                             &ratchet_messages))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_MESSAGES",
+                               "needs to be a number");
+    ratchet_messages = 64;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "RATCHET_TIME",
+                                           &ratchet_time))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_TIME",
+                               "need delay value");
+    ratchet_time = GNUNET_TIME_UNIT_HOURS;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "REFRESH_CONNECTION_TIME",
+                                           &keepalive_period))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "REFRESH_CONNECTION_TIME",
+                               "need delay value");
+    keepalive_period = GNUNET_TIME_UNIT_MINUTES;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "DROP_PERCENT",
+                                             &drop_percent))
+  {
+    drop_percent = 0;
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+  }
+  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
+  if (NULL == my_private_key)
+  {
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key,
+                                      &my_full_id.public_key);
+  stats = GNUNET_STATISTICS_create ("cadet",
+                                    c);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  transport = GNUNET_TRANSPORT_application_init (c);
+  /* FIXME: optimize code to allow GNUNET_YES here! */
+  open_ports = GNUNET_CONTAINER_multihashmap_create (16,
+                                                     GNUNET_NO);
+  loose_channels = GNUNET_CONTAINER_multihashmap_create (16,
+                                                         GNUNET_NO);
+  peers = GNUNET_CONTAINER_multipeermap_create (16,
+                                                GNUNET_YES);
+  connections = GNUNET_CONTAINER_multishortmap_create (256,
+                                                       GNUNET_YES);
+  GCH_init (c);
+  GCD_init (c);
+  GCO_init (c);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "CADET started for peer %s\n",
+              GNUNET_i2s (&my_full_id));
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+  ("cadet",
+  GNUNET_SERVICE_OPTION_NONE,
+  &run,
+  &client_connect_cb,
+  &client_disconnect_cb,
+  NULL,
+  GNUNET_MQ_hd_fixed_size (port_open,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
+                           struct GNUNET_CADET_PortMessage,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (port_close,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
+                           struct GNUNET_CADET_PortMessage,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (channel_create,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
+                           struct GNUNET_CADET_LocalChannelCreateMessage,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (channel_destroy,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
+                           struct GNUNET_CADET_LocalChannelDestroyMessage,
+                           NULL),
+  GNUNET_MQ_hd_var_size (local_data,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                         struct GNUNET_CADET_LocalData,
+                         NULL),
+  GNUNET_MQ_hd_fixed_size (local_ack,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
+                           struct GNUNET_CADET_LocalAck,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (get_peers,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PEERS,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (show_path,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_PATH,
+                           struct GNUNET_CADET_RequestPathInfoMessage,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (info_tunnels,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_REQUEST_INFO_TUNNELS,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (drop_message,
+                           GNUNET_MESSAGE_TYPE_CADET_DROP_CADET_MESSAGE,
+                           struct GNUNET_CADET_RequestDropCadetMessage,
+                           NULL),
+  GNUNET_MQ_handler_end ());
+
+/* end of gnunet-service-cadet-new.c */
diff --git a/src/service/cadet/gnunet-service-cadet.h b/src/service/cadet/gnunet-service-cadet.h
new file mode 100644
index 000000000..8f2386d00
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet.h
@@ -0,0 +1,328 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet.h
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_H
+#define GNUNET_SERVICE_CADET_H
+
+#include "gnunet_util_lib.h"
+#include "cadet_protocol.h"
+
+/**
+ * A client to the CADET service.  Each client gets a unique handle.
+ */
+struct CadetClient;
+
+/**
+ * A peer in the GNUnet network.  Each peer we care about must have one globally
+ * unique such handle within this process.
+ */
+struct CadetPeer;
+
+/**
+ * Tunnel from us to another peer.  There can only be at most one
+ * tunnel per peer.
+ */
+struct CadetTunnel;
+
+/**
+ * Entry in the message queue of a `struct CadetTunnel`.
+ */
+struct CadetTunnelQueueEntry;
+
+/**
+ * A path of peer in the GNUnet network.  There must only be at most
+ * once such path.  Paths may share disjoint prefixes, but must all
+ * end at a unique suffix.  Paths must also not be proper subsets of
+ * other existing paths.
+ */
+struct CadetPeerPath;
+
+/**
+ * Entry in a peer path.
+ */
+struct CadetPeerPathEntry
+{
+  /**
+   * DLL of paths where the same @e peer is at the same offset.
+   */
+  struct CadetPeerPathEntry *next;
+
+  /**
+   * DLL of paths where the same @e peer is at the same offset.
+   */
+  struct CadetPeerPathEntry *prev;
+
+  /**
+   * The peer at this offset of the path.
+   */
+  struct CadetPeer *peer;
+
+  /**
+   * Path this entry belongs to.
+   */
+  struct CadetPeerPath *path;
+
+  /**
+   * Connection using this path, or NULL for none.
+   */
+  struct CadetConnection *cc;
+
+  /**
+   * Path's historic score up to this point.  Basically, how often did
+   * we succeed or fail to use the path up to this entry in a
+   * connection.  Positive values indicate good experiences, negative
+   * values bad experiences.  Code updating the score must guard
+   * against overflows.
+   */
+  int score;
+};
+
+/**
+ * Entry in list of connections used by tunnel, with metadata.
+ */
+struct CadetTConnection
+{
+  /**
+   * Next in DLL.
+   */
+  struct CadetTConnection *next;
+
+  /**
+   * Prev in DLL.
+   */
+  struct CadetTConnection *prev;
+
+  /**
+   * Connection handle.
+   */
+  struct CadetConnection *cc;
+
+  /**
+   * Tunnel this connection belongs to.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Creation time, to keep oldest connection alive.
+   */
+  struct GNUNET_TIME_Absolute created;
+
+  /**
+   * Connection throughput, to keep fastest connection alive.
+   */
+  uint32_t throughput;
+
+  /**
+   * Is the connection currently ready for transmission?
+   */
+  int is_ready;
+};
+
+
+/**
+ * Port opened by a client.
+ */
+struct OpenPort
+{
+  /**
+   * Client that opened the port.
+   */
+  struct CadetClient *c;
+
+  /**
+   * Port number.
+   */
+  struct GNUNET_HashCode port;
+
+  /**
+   * Port hashed with our PID (matches incoming OPEN messages).
+   */
+  struct GNUNET_HashCode h_port;
+};
+
+
+/**
+ * Active path through the network (used by a tunnel).  There may
+ * be at most one connection per path.
+ */
+struct CadetConnection;
+
+/**
+ * Description of a segment of a `struct CadetConnection` at the
+ * intermediate peers.  Routes are basically entries in a peer's
+ * routing table for forwarding traffic.  At both endpoints, the
+ * routes are terminated by a `struct CadetConnection`, which knows
+ * the complete `struct CadetPath` that is formed by the individual
+ * routes.
+ */
+struct CadetRoute;
+
+/**
+ * Logical end-to-end connection between clients.  There can be
+ * any number of channels between clients.
+ */
+struct CadetChannel;
+
+/**
+ * Handle to our configuration.
+ */
+extern const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Handle to the statistics service.
+ */
+extern struct GNUNET_STATISTICS_Handle *stats;
+
+/**
+ * Handle to Transport service.
+ */
+extern struct GNUNET_TRANSPORT_ApplicationHandle *transport;
+
+/**
+ * Local peer own ID.
+ */
+extern struct GNUNET_PeerIdentity my_full_id;
+
+/**
+ * Own private key.
+ */
+extern struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
+
+/**
+ * All ports clients of this peer have opened.  Maps from
+ * a hashed port to a `struct OpenPort`.
+ */
+extern struct GNUNET_CONTAINER_MultiHashMap *open_ports;
+
+/**
+ * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
+ * hash codes to `struct CadetConnection` objects.
+ */
+extern struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+/**
+ * Map from ports to channels where the ports were closed at the
+ * time we got the inbound connection.
+ * Indexed by h_port, contains `struct CadetChannel`.
+ */
+extern struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
+
+/**
+ * Map from PIDs to `struct CadetPeer` entries.
+ */
+extern struct GNUNET_CONTAINER_MultiPeerMap *peers;
+
+/**
+ * How many messages are needed to trigger an AXOLOTL ratchet advance.
+ */
+extern unsigned long long ratchet_messages;
+
+/**
+ * How long until we trigger a ratched advance due to time.
+ */
+extern struct GNUNET_TIME_Relative ratchet_time;
+
+/**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+extern struct GNUNET_TIME_Relative keepalive_period;
+
+/**
+ * Signal that shutdown is happening: prevent recovery measures.
+ */
+extern int shutting_down;
+
+/**
+ * Set to non-zero values to create random drops to test retransmissions.
+ */
+extern unsigned long long drop_percent;
+
+
+/**
+ * Send a message to a client.
+ *
+ * @param c client to get the message
+ * @param env envelope with the message
+ */
+void
+GSC_send_to_client (struct CadetClient *c,
+                    struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * A channel was destroyed by the other peer. Tell our client.
+ *
+ * @param c client that lost a channel
+ * @param ccn channel identification number for the client
+ * @param ch the channel object
+ */
+void
+GSC_handle_remote_channel_destroy (struct CadetClient *c,
+                                   struct GNUNET_CADET_ClientChannelNumber ccn,
+                                   struct CadetChannel *ch);
+
+/**
+ * A client that created a loose channel that was not bound to a port
+ * disconnected, drop it from the #loose_channels list.
+ *
+ * @param h_port the hashed port the channel was trying to bind to
+ * @param ch the channel that was lost
+ */
+void
+GSC_drop_loose_channel (const struct GNUNET_HashCode *h_port,
+                        struct CadetChannel *ch);
+
+
+/**
+ * Bind incoming channel to this client, and notify client
+ * about incoming connection.
+ *
+ * @param c client to bind to
+ * @param ch channel to be bound
+ * @param dest peer that establishes the connection
+ * @param port port number
+ * @param options options
+ * @return local channel number assigned to the new client
+ */
+struct GNUNET_CADET_ClientChannelNumber
+GSC_bind (struct CadetClient *c,
+          struct CadetChannel *ch,
+          struct CadetPeer *dest,
+          const struct GNUNET_HashCode *port,
+          uint32_t options);
+
+
+/**
+ * Return identifier for a client as a string.
+ *
+ * @param c client to identify
+ * @return string for debugging
+ */
+const char *
+GSC_2s (struct CadetClient *c);
+
+
+#endif
diff --git a/src/service/cadet/gnunet-service-cadet_channel.c b/src/service/cadet/gnunet-service-cadet_channel.c
new file mode 100644
index 000000000..78658c296
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_channel.c
@@ -0,0 +1,2008 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/gnunet-service-cadet_channel.c
+ * @brief logical links between CADET clients
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - Congestion/flow control:
+ *   + estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+ *     (and figure out how/where to use this!)
+ *   + figure out flow control without ACKs (unreliable traffic!)
+ * - revisit handling of 'unbuffered' traffic!
+ *   (need to push down through tunnel into connection selection)
+ * - revisit handling of 'buffered' traffic: 4 is a rather small buffer; maybe
+ *   reserve more bits in 'options' to allow for buffer size control?
+ */
+#include "platform.h"
+#include "cadet.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_paths.h"
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-chn", __VA_ARGS__)
+
+/**
+ * How long do we initially wait before retransmitting?
+ */
+#define CADET_INITIAL_RETRANSMIT_TIME \
+  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250)
+
+/**
+ * How long do we wait before dropping state about incoming
+ * connection to closed port?
+ */
+#define TIMEOUT_CLOSED_PORT \
+  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
+
+/**
+ * How long do we wait at least before retransmitting ever?
+ */
+#define MIN_RTT_DELAY \
+  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 75)
+
+/**
+ * Maximum message ID into the future we accept for out-of-order messages.
+ * If the message is more than this into the future, we drop it.  This is
+ * important both to detect values that are actually in the past, as well
+ * as to limit adversarially triggerable memory consumption.
+ *
+ * Note that right now we have "max_pending_messages = 4" hard-coded in
+ * the logic below, so a value of 4 would suffice here. But we plan to
+ * allow larger windows in the future...
+ */
+#define MAX_OUT_OF_ORDER_DISTANCE 1024
+
+
+/**
+ * All the states a channel can be in.
+ */
+enum CadetChannelState
+{
+  /**
+   * Uninitialized status, should never appear in operation.
+   */
+  CADET_CHANNEL_NEW,
+
+  /**
+   * Channel is to a port that is not open, we're waiting for the
+   * port to be opened.
+   */
+  CADET_CHANNEL_LOOSE,
+
+  /**
+   * CHANNEL_OPEN message sent, waiting for CHANNEL_OPEN_ACK.
+   */
+  CADET_CHANNEL_OPEN_SENT,
+
+  /**
+   * Connection confirmed, ready to carry traffic.
+   */
+  CADET_CHANNEL_READY
+};
+
+
+/**
+ * Info needed to retry a message in case it gets lost.
+ * Note that we DO use this structure also for unreliable
+ * messages.
+ */
+struct CadetReliableMessage
+{
+  /**
+   * Double linked list, FIFO style
+   */
+  struct CadetReliableMessage *next;
+
+  /**
+   * Double linked list, FIFO style
+   */
+  struct CadetReliableMessage *prev;
+
+  /**
+   * Which channel is this message in?
+   */
+  struct CadetChannel *ch;
+
+  /**
+   * Entry in the tunnels queue for this message, NULL if it has left
+   * the tunnel.  Used to cancel transmission in case we receive an
+   * ACK in time.
+   */
+  struct CadetTunnelQueueEntry *qe;
+
+  /**
+   * Data message we are trying to send.
+   */
+  struct GNUNET_CADET_ChannelAppDataMessage *data_message;
+
+  /**
+   * How soon should we retry if we fail to get an ACK?
+   * Messages in the queue are sorted by this value.
+   */
+  struct GNUNET_TIME_Absolute next_retry;
+
+  /**
+   * How long do we wait for an ACK after transmission?
+   * Use for the back-off calculation.
+   */
+  struct GNUNET_TIME_Relative retry_delay;
+
+  /**
+   * Time when we first successfully transmitted the message
+   * (that is, set @e num_transmissions to 1).
+   */
+  struct GNUNET_TIME_Absolute first_transmission_time;
+
+  /**
+   * Identifier of the connection that this message took when it
+   * was first transmitted.  Only useful if @e num_transmissions is 1.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier connection_taken;
+
+  /**
+   * How often was this message transmitted?  #GNUNET_SYSERR if there
+   * was an error transmitting the message, #GNUNET_NO if it was not
+   * yet transmitted ever, otherwise the number of (re) transmissions.
+   */
+  int num_transmissions;
+};
+
+
+/**
+ * List of received out-of-order data messages.
+ */
+struct CadetOutOfOrderMessage
+{
+  /**
+   * Double linked list, FIFO style
+   */
+  struct CadetOutOfOrderMessage *next;
+
+  /**
+   * Double linked list, FIFO style
+   */
+  struct CadetOutOfOrderMessage *prev;
+
+  /**
+   * ID of the message (messages up to this point needed
+   * before we give this one to the client).
+   */
+  struct ChannelMessageIdentifier mid;
+
+  /**
+   * The envelope with the payload of the out-of-order message
+   */
+  struct GNUNET_MQ_Envelope *env;
+};
+
+
+/**
+ * Client endpoint of a `struct CadetChannel`.  A channel may be a
+ * loopback channel, in which case it has two of these endpoints.
+ * Note that flow control also is required in both directions.
+ */
+struct CadetChannelClient
+{
+  /**
+   * Client handle.  Not by itself sufficient to designate
+   * the client endpoint, as the same client handle may
+   * be used for both the owner and the destination, and
+   * we thus also need the channel ID to identify the client.
+   */
+  struct CadetClient *c;
+
+  /**
+   * Head of DLL of messages received out of order or while client was unready.
+   */
+  struct CadetOutOfOrderMessage *head_recv;
+
+  /**
+   * Tail DLL of messages received out of order or while client was unready.
+   */
+  struct CadetOutOfOrderMessage *tail_recv;
+
+  /**
+   * Local tunnel number for this client.
+   * (if owner >= #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI,
+   *  otherwise < #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * Number of entries currently in @a head_recv DLL.
+   */
+  unsigned int num_recv;
+
+  /**
+   * Can we send data to the client?
+   */
+  int client_ready;
+};
+
+
+/**
+ * Struct containing all information regarding a channel to a remote client.
+ */
+struct CadetChannel
+{
+  /**
+   * Tunnel this channel is in.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Client owner of the tunnel, if any.
+   * (Used if this channel represends the initiating end of the tunnel.)
+   */
+  struct CadetChannelClient *owner;
+
+  /**
+   * Client destination of the tunnel, if any.
+   * (Used if this channel represents the listening end of the tunnel.)
+   */
+  struct CadetChannelClient *dest;
+
+  /**
+   * Last entry in the tunnel's queue relating to control messages
+   * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN or
+   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK).  Used to cancel
+   * transmission in case we receive updated information.
+   */
+  struct CadetTunnelQueueEntry *last_control_qe;
+
+  /**
+   * Head of DLL of messages sent and not yet ACK'd.
+   */
+  struct CadetReliableMessage *head_sent;
+
+  /**
+   * Tail of DLL of messages sent and not yet ACK'd.
+   */
+  struct CadetReliableMessage *tail_sent;
+
+  /**
+   * Task to resend/poll in case no ACK is received.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_control_task;
+
+  /**
+   * Task to resend/poll in case no ACK is received.
+   */
+  struct GNUNET_SCHEDULER_Task *retry_data_task;
+
+  /**
+   * Last time the channel was used
+   */
+  struct GNUNET_TIME_Absolute timestamp;
+
+  /**
+   * Destination port of the channel.
+   */
+  struct GNUNET_HashCode port;
+
+  /**
+   * Hash'ed port of the channel with initiator and destination PID.
+   */
+  struct GNUNET_HashCode h_port;
+
+  /**
+   * Counter for exponential backoff.
+   */
+  struct GNUNET_TIME_Relative retry_time;
+
+  /**
+   * Bitfield of already-received messages past @e mid_recv.
+   */
+  uint64_t mid_futures;
+
+  /**
+   * Next MID expected for incoming traffic.
+   */
+  struct ChannelMessageIdentifier mid_recv;
+
+  /**
+   * Next MID to use for outgoing traffic.
+   */
+  struct ChannelMessageIdentifier mid_send;
+
+  /**
+   * Total (reliable) messages pending ACK for this channel.
+   */
+  unsigned int pending_messages;
+
+  /**
+   * Maximum (reliable) messages pending ACK for this channel
+   * before we throttle the client.
+   */
+  unsigned int max_pending_messages;
+
+  /**
+   * Number identifying this channel in its tunnel.
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+
+  /**
+   * Channel state.
+   */
+  enum CadetChannelState state;
+
+  /**
+   * Count how many ACKs we skipped, used to prevent long
+   * sequences of ACK skipping.
+   */
+  unsigned int skip_ack_series;
+
+  /**
+   * Is the tunnel bufferless (minimum latency)?
+   */
+  int nobuffer;
+
+  /**
+   * Is the tunnel reliable?
+   */
+  int reliable;
+
+  /**
+   * Is the tunnel out-of-order?
+   */
+  int out_of_order;
+
+  /**
+   * Is this channel a loopback channel, where the destination is us again?
+   */
+  int is_loopback;
+
+  /**
+   * Flag to signal the destruction of the channel.  If this is set to
+   * #GNUNET_YES the channel will be destroyed once the queue is
+   * empty.
+   */
+  int destroy;
+
+  /**
+   * Type of message to be dropped. See GCT_send.
+   */
+  uint16_t type GNUNET_PACKED;
+
+};
+
+void
+GCCH_assign_type_to_drop (struct CadetChannel *ch, const struct
+                          GNUNET_CADET_RequestDropCadetMessage *message)
+{
+
+  ch->type = message->type;
+
+}
+
+
+/**
+ * Check if type of message is the one to drop.
+ * @param ch CadetChannel to check for message type to drop.
+ * @param message GNUNET_MessageHeader to compare the type with.
+ */
+int
+GCCH_is_type_to_drop (struct CadetChannel *ch, const struct
+                      GNUNET_MessageHeader *message)
+{
+
+  if (ch->type == message->type)
+  {
+    ch->type = 0;
+    return GNUNET_YES;
+  }
+  else
+    return GNUNET_NO;
+}
+
+
+/**
+ * Get the static string for identification of the channel.
+ *
+ * @param ch Channel.
+ *
+ * @return Static string with the channel IDs.
+ */
+const char *
+GCCH_2s (const struct CadetChannel *ch)
+{
+  static char buf[128];
+
+  GNUNET_snprintf (buf,
+                   sizeof(buf),
+                   "Channel %s:%s ctn:%X(%X/%X)",
+                   (GNUNET_YES == ch->is_loopback)
+                   ? "loopback"
+                   : GNUNET_i2s (GCP_get_id (GCT_get_destination (ch->t))),
+                   GNUNET_h2s (&ch->port),
+                   ch->ctn.cn,
+                   (NULL == ch->owner)
+                   ? 0
+                   : ntohl (ch->owner->ccn.channel_of_client),
+                   (NULL == ch->dest)
+                   ? 0
+                   : ntohl (ch->dest->ccn.channel_of_client));
+  return buf;
+}
+
+
+/**
+ * Hash the @a port and @a initiator and @a listener to
+ * calculate the "challenge" @a h_port we send to the other
+ * peer on #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN.
+ *
+ * @param[out] h_port set to the hash of @a port, @a initiator and @a listener
+ * @param port cadet port, as seen by CADET clients
+ * @param listener peer that is listining on @a port
+ */
+void
+GCCH_hash_port (struct GNUNET_HashCode *h_port,
+                const struct GNUNET_HashCode *port,
+                const struct GNUNET_PeerIdentity *listener)
+{
+  struct GNUNET_HashContext *hc;
+
+  hc = GNUNET_CRYPTO_hash_context_start ();
+  GNUNET_CRYPTO_hash_context_read (hc, port, sizeof(*port));
+  GNUNET_CRYPTO_hash_context_read (hc, listener, sizeof(*listener));
+  GNUNET_CRYPTO_hash_context_finish (hc, h_port);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Calculated port hash %s\n",
+       GNUNET_h2s (h_port));
+}
+
+
+/**
+ * Get the channel's public ID.
+ *
+ * @param ch Channel.
+ *
+ * @return ID used to identify the channel with the remote peer.
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch)
+{
+  return ch->ctn;
+}
+
+
+/**
+ * Release memory associated with @a ccc
+ *
+ * @param ccc data structure to clean up
+ */
+static void
+free_channel_client (struct CadetChannelClient *ccc)
+{
+  struct CadetOutOfOrderMessage *com;
+
+  while (NULL != (com = ccc->head_recv))
+  {
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+  }
+  GNUNET_free (ccc);
+}
+
+
+/**
+ * Destroy the given channel.
+ *
+ * @param ch channel to destroy
+ */
+static void
+channel_destroy (struct CadetChannel *ch)
+{
+  struct CadetReliableMessage *crm;
+
+  while (NULL != (crm = ch->head_sent))
+  {
+    GNUNET_assert (ch == crm->ch);
+    if (NULL != crm->qe)
+    {
+      GCT_send_cancel (crm->qe);
+      crm->qe = NULL;
+    }
+    GNUNET_CONTAINER_DLL_remove (ch->head_sent, ch->tail_sent, crm);
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
+  }
+  if (CADET_CHANNEL_LOOSE == ch->state)
+  {
+    GSC_drop_loose_channel (&ch->h_port, ch);
+  }
+  if (NULL != ch->owner)
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
+  }
+  if (NULL != ch->dest)
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  if (NULL != ch->last_control_qe)
+  {
+    GCT_send_cancel (ch->last_control_qe);
+    ch->last_control_qe = NULL;
+  }
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if (NULL != ch->retry_control_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  if (GNUNET_NO == ch->is_loopback)
+  {
+    GCT_remove_channel (ch->t, ch, ch->ctn);
+    ch->t = NULL;
+  }
+  GNUNET_free (ch);
+}
+
+
+/**
+ * Send a channel create message.
+ *
+ * @param cls Channel for which to send.
+ */
+static void
+send_channel_open (void *cls);
+
+
+/**
+ * Function called once the tunnel confirms that we sent the
+ * create message.  Delays for a bit until we retry.
+ *
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
+ */
+static void
+channel_open_sent_cb (void *cls,
+                      const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
+  ch->retry_time = GNUNET_TIME_STD_BACKOFF (ch->retry_time);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sent CADET_CHANNEL_OPEN on %s, retrying in %s\n",
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (ch->retry_time, GNUNET_YES));
+  ch->retry_control_task =
+    GNUNET_SCHEDULER_add_delayed (ch->retry_time, &send_channel_open, ch);
+}
+
+
+/**
+ * Send a channel open message.
+ *
+ * @param cls Channel for which to send.
+ */
+static void
+send_channel_open (void *cls)
+{
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelOpenMessage msgcc;
+
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN message for %s\n",
+       GCCH_2s (ch));
+  msgcc.header.size = htons (sizeof(msgcc));
+  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
+  // TODO This will be removed in a major release, because this will be a protocol breaking change. We set the deprecated "reliable" bit here that was removed.
+  msgcc.opt = 2;
+  msgcc.h_port = ch->h_port;
+  msgcc.ctn = ch->ctn;
+  ch->state = CADET_CHANNEL_OPEN_SENT;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe =
+    GCT_send (ch->t, &msgcc.header, &channel_open_sent_cb, ch, &msgcc.ctn);
+  GNUNET_assert (NULL == ch->retry_control_task);
+}
+
+
+/**
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
+ */
+void
+GCCH_tunnel_up (struct CadetChannel *ch)
+{
+  GNUNET_assert (NULL == ch->retry_control_task);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Tunnel up, sending CHANNEL_OPEN on %s now\n",
+       GCCH_2s (ch));
+  ch->retry_control_task = GNUNET_SCHEDULER_add_now (&send_channel_open, ch);
+}
+
+
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber ccn,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options)
+{
+  struct CadetChannel *ch;
+  struct CadetChannelClient *ccco;
+
+  ccco = GNUNET_new (struct CadetChannelClient);
+  ccco->c = owner;
+  ccco->ccn = ccn;
+  ccco->client_ready = GNUNET_YES;
+
+  ch = GNUNET_new (struct CadetChannel);
+  ch->mid_recv.mid = htonl (1);  /* The OPEN_ACK counts as message 0! */
+  ch->nobuffer = GNUNET_NO;
+  ch->reliable = GNUNET_YES;
+  ch->out_of_order = GNUNET_NO;
+  ch->max_pending_messages =
+    (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  ch->owner = ccco;
+  ch->port = *port;
+  GCCH_hash_port (&ch->h_port, port, GCP_get_id (destination));
+  if (0 == GNUNET_memcmp (&my_full_id, GCP_get_id (destination)))
+  {
+    struct OpenPort *op;
+
+    ch->is_loopback = GNUNET_YES;
+    op = GNUNET_CONTAINER_multihashmap_get (open_ports, &ch->h_port);
+    if (NULL == op)
+    {
+      /* port closed, wait for it to possibly open */
+      ch->state = CADET_CHANNEL_LOOSE;
+      (void) GNUNET_CONTAINER_multihashmap_put (
+        loose_channels,
+        &ch->h_port,
+        ch,
+        GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Created loose incoming loopback channel to port %s\n",
+           GNUNET_h2s (&ch->port));
+    }
+    else
+    {
+      GCCH_bind (ch, op->c, &op->port);
+    }
+  }
+  else
+  {
+    ch->t = GCP_get_tunnel (destination, GNUNET_YES);
+    ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+    ch->ctn = GCT_add_channel (ch->t, ch);
+  }
+  GNUNET_STATISTICS_update (stats, "# channels", 1, GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Created channel to port %s at peer %s for %s using %s\n",
+       GNUNET_h2s (port),
+       GCP_2s (destination),
+       GSC_2s (owner),
+       (GNUNET_YES == ch->is_loopback) ? "loopback" : GCT_2s (ch->t));
+  return ch;
+}
+
+
+/**
+ * We had an incoming channel to a port that is closed.
+ * It has not been opened for a while, drop it.
+ *
+ * @param cls the channel to drop
+ */
+static void
+timeout_closed_cb (void *cls)
+{
+  struct CadetChannel *ch = cls;
+
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing incoming channel to port %s from peer %s due to timeout\n",
+       GNUNET_h2s (&ch->port),
+       GCP_2s (GCT_get_destination (ch->t)));
+  channel_destroy (ch);
+}
+
+
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber ctn,
+                           const struct GNUNET_HashCode *h_port,
+                           uint32_t options)
+{
+  struct CadetChannel *ch;
+  struct OpenPort *op;
+
+  ch = GNUNET_new (struct CadetChannel);
+  ch->h_port = *h_port;
+  ch->t = t;
+  ch->ctn = ctn;
+  ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+  ch->nobuffer = GNUNET_NO;
+  ch->reliable = GNUNET_YES;
+  ch->out_of_order = GNUNET_NO;
+  ch->max_pending_messages =
+    (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  GNUNET_STATISTICS_update (stats, "# channels", 1, GNUNET_NO);
+
+  op = GNUNET_CONTAINER_multihashmap_get (open_ports, h_port);
+  if (NULL == op)
+  {
+    /* port closed, wait for it to possibly open */
+    ch->state = CADET_CHANNEL_LOOSE;
+    (void) GNUNET_CONTAINER_multihashmap_put (
+      loose_channels,
+      &ch->h_port,
+      ch,
+      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+    GNUNET_assert (NULL == ch->retry_control_task);
+    ch->retry_control_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT_CLOSED_PORT,
+                                                           &timeout_closed_cb,
+                                                           ch);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Created loose incoming channel to port %s from peer %s\n",
+         GNUNET_h2s (&ch->port),
+         GCP_2s (GCT_get_destination (ch->t)));
+  }
+  else
+  {
+    GCCH_bind (ch, op->c, &op->port);
+  }
+  GNUNET_STATISTICS_update (stats, "# channels", 1, GNUNET_NO);
+  return ch;
+}
+
+
+/**
+ * Function called once the tunnel confirms that we sent the
+ * ACK message.  Just remembers it was sent, we do not expect
+ * ACKs for ACKs ;-).
+ *
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
+ */
+static void
+send_ack_cb (void *cls,
+             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
+}
+
+
+/**
+ * Compute and send the current #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK to the other peer.
+ *
+ * @param ch channel to send the #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK for
+ */
+static void
+send_channel_data_ack (struct CadetChannel *ch)
+{
+  struct GNUNET_CADET_ChannelDataAckMessage msg;
+
+  if (GNUNET_NO == ch->reliable)
+    return; /* no ACKs */
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
+  msg.header.size = htons (sizeof(msg));
+  msg.ctn = ch->ctn;
+  msg.mid.mid = htonl (ntohl (ch->mid_recv.mid));
+  msg.futures = GNUNET_htonll (ch->mid_futures);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending DATA_ACK %u:%llX via %s\n",
+       (unsigned int) ntohl (msg.mid.mid),
+       (unsigned long long) ch->mid_futures,
+       GCCH_2s (ch));
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t, &msg.header, &send_ack_cb, ch,
+                                  &msg.ctn);
+}
+
+
+/**
+ * Send our initial #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK to the client confirming that the
+ * connection is up.
+ *
+ * @param cls the `struct CadetChannel`
+ */
+static void
+send_open_ack (void *cls)
+{
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelOpenAckMessage msg;
+
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
+  msg.header.size = htons (sizeof(msg));
+  msg.reserved = htonl (0);
+  msg.ctn = ch->ctn;
+  msg.port = ch->port;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t, &msg.header, &send_ack_cb, ch,
+                                  &msg.ctn);
+}
+
+
+/**
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
+ *
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
+ */
+void
+GCCH_handle_duplicate_open (
+  struct CadetChannel *ch,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
+{
+  if (NULL == ch->dest)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: port is closed\n",
+         GCCH_2s (ch));
+    return;
+  }
+  if (NULL != ch->retry_control_task)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: control message is pending\n",
+         GCCH_2s (ch));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retransmitting CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  ch->retry_control_task = GNUNET_SCHEDULER_add_now (&send_open_ack, ch);
+}
+
+
+/**
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK to the client to solicit more messages.
+ *
+ * @param ch channel the ack is for
+ * @param to_owner #GNUNET_YES to send to owner,
+ *                 #GNUNET_NO to send to dest
+ */
+static void
+send_ack_to_client (struct CadetChannel *ch, int to_owner)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalAck *ack;
+  struct CadetChannelClient *ccc;
+
+  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
+  if (NULL == ccc)
+  {
+    /* This can happen if we are just getting ACKs after
+       our local client already disconnected. */
+    GNUNET_assert (GNUNET_YES == ch->destroy);
+    return;
+  }
+  env = GNUNET_MQ_msg (ack, GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
+  ack->ccn = ccc->ccn;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X (%u/%u pending)\n",
+       GSC_2s (ccc->c),
+       (GNUNET_YES == to_owner) ? "owner" : "dest",
+       ntohl (ack->ccn.channel_of_client),
+       ch->pending_messages,
+       ch->max_pending_messages);
+  GSC_send_to_client (ccc->c, env);
+}
+
+
+void
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c,
+           const struct GNUNET_HashCode *port)
+{
+  uint32_t options;
+  struct CadetChannelClient *cccd;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Binding %s from %s to port %s of %s\n",
+       GCCH_2s (ch),
+       GCT_2s (ch->t),
+       GNUNET_h2s (&ch->port),
+       GSC_2s (c));
+  if (NULL != ch->retry_control_task)
+  {
+    /* there might be a timeout task here */
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  options = 0;
+  cccd = GNUNET_new (struct CadetChannelClient);
+  GNUNET_assert (NULL == ch->dest);
+  ch->dest = cccd;
+  ch->port = *port;
+  cccd->c = c;
+  cccd->client_ready = GNUNET_YES;
+  cccd->ccn = GSC_bind (c,
+                        ch,
+                        (GNUNET_YES == ch->is_loopback)
+                        ? GCP_get (&my_full_id, GNUNET_YES)
+                        : GCT_get_destination (ch->t),
+                        port,
+                        options);
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  ch->mid_recv.mid = htonl (1);  /* The OPEN counts as message 0! */
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    ch->state = CADET_CHANNEL_OPEN_SENT;
+    GCCH_handle_channel_open_ack (ch, NULL, port);
+  }
+  else
+  {
+    /* notify other peer that we accepted the connection */
+    ch->state = CADET_CHANNEL_READY;
+    ch->retry_control_task = GNUNET_SCHEDULER_add_now (&send_open_ack, ch);
+  }
+  /* give client it's initial supply of ACKs */
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  for (unsigned int i = 0; i < ch->max_pending_messages; i++)
+    send_ack_to_client (ch, GNUNET_NO);
+}
+
+
+/**
+ * One of our clients has disconnected, tell the other one that we
+ * are finished. Done asynchronously to avoid concurrent modification
+ * issues if this is the same client.
+ *
+ * @param cls the `struct CadetChannel` where one of the ends is now dead
+ */
+static void
+signal_remote_destroy_cb (void *cls)
+{
+  struct CadetChannel *ch = cls;
+  struct CadetChannelClient *ccc;
+
+  /* Find which end is left... */
+  ch->retry_control_task = NULL;
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  GSC_handle_remote_channel_destroy (ccc->c, ccc->ccn, ch);
+  channel_destroy (ch);
+}
+
+
+/**
+ * Destroy locally created channel.  Called by the local client, so no
+ * need to tell the client.
+ *
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
+ */
+void
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s asks for destruction of %s\n",
+       GSC_2s (c),
+       GCCH_2s (ch));
+  GNUNET_assert (NULL != c);
+  if ((NULL != ch->owner) && (c == ch->owner->c) &&
+      (ccn.channel_of_client == ch->owner->ccn.channel_of_client))
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
+  }
+  else if ((NULL != ch->dest) && (c == ch->dest->c) &&
+           (ccn.channel_of_client == ch->dest->ccn.channel_of_client))
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
+
+  if (GNUNET_YES == ch->destroy)
+  {
+    /* other end already destroyed, with the local client gone, no need
+       to finish transmissions, just destroy immediately. */
+    channel_destroy (ch);
+    return;
+  }
+  if ((NULL != ch->head_sent) && ((NULL != ch->owner) || (NULL != ch->dest)))
+  {
+    /* Wait for other end to destroy us as well,
+       and otherwise allow send queue to be transmitted first */
+    ch->destroy = GNUNET_YES;
+    return;
+  }
+  if ((GNUNET_YES == ch->is_loopback) &&
+      ((NULL != ch->owner) || (NULL != ch->dest)))
+  {
+    if (NULL != ch->retry_control_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task =
+      GNUNET_SCHEDULER_add_now (&signal_remote_destroy_cb, ch);
+    return;
+  }
+  if (GNUNET_NO == ch->is_loopback)
+  {
+    /* If the we ever sent the CHANNEL_CREATE, we need to send a destroy message. */
+    switch (ch->state)
+    {
+    case CADET_CHANNEL_NEW:
+      /* We gave up on a channel that we created as a client to a remote
+         target, but that never went anywhere. Nothing to do here. */
+      break;
+
+    case CADET_CHANNEL_LOOSE:
+      break;
+
+    default:
+      GCT_send_channel_destroy (ch->t, ch->ctn);
+    }
+  }
+  /* Nothing left to do, just finish destruction */
+  channel_destroy (ch);
+}
+
+
+void
+GCCH_handle_channel_open_ack (
+  struct CadetChannel *ch,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+  const struct GNUNET_HashCode *port)
+{
+  switch (ch->state)
+  {
+  case CADET_CHANNEL_NEW:
+    /* this should be impossible */
+    GNUNET_break (0);
+    break;
+
+  case CADET_CHANNEL_LOOSE:
+    /* This makes no sense. */
+    GNUNET_break_op (0);
+    break;
+
+  case CADET_CHANNEL_OPEN_SENT:
+    if (NULL == ch->owner)
+    {
+      /* We're not the owner, wrong direction! */
+      GNUNET_break_op (0);
+      return;
+    }
+    if (0 != GNUNET_memcmp (&ch->port, port))
+    {
+      /* Other peer failed to provide the right port,
+         refuse connection. */
+      GNUNET_break_op (0);
+      return;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CHANNEL_OPEN_ACK for waiting %s, entering READY state\n",
+         GCCH_2s (ch));
+    if (NULL != ch->retry_control_task)   /* can be NULL if ch->is_loopback */
+    {
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+      ch->retry_control_task = NULL;
+    }
+    ch->state = CADET_CHANNEL_READY;
+    /* On first connect, send client as many ACKs as we allow messages
+       to be buffered! */
+    for (unsigned int i = 0; i < ch->max_pending_messages; i++)
+      send_ack_to_client (ch, GNUNET_YES);
+    break;
+
+  case CADET_CHANNEL_READY:
+    /* duplicate ACK, maybe we retried the CREATE. Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received duplicate channel OPEN_ACK for %s\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats, "# duplicate CREATE_ACKs", 1, GNUNET_NO);
+    break;
+  }
+}
+
+
+/**
+ * Test if element @a e1 comes before element @a e2.
+ *
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param m1 a message of to sort
+ * @param m2 another message to sort
+ * @return #GNUNET_YES if @a e1 < @a e2, otherwise #GNUNET_NO
+ */
+static int
+is_before (void *cls,
+           struct CadetOutOfOrderMessage *m1,
+           struct CadetOutOfOrderMessage *m2)
+{
+  int *duplicate = cls;
+  uint32_t v1 = ntohl (m1->mid.mid);
+  uint32_t v2 = ntohl (m2->mid.mid);
+  uint32_t delta;
+
+  delta = v2 - v1;
+  if (0 == delta)
+    *duplicate = GNUNET_YES;
+  if (delta > (uint32_t) INT_MAX)
+  {
+    /* in overflow range, we can safely assume we wrapped around */
+    return GNUNET_NO;
+  }
+  else
+  {
+    /* result is small, thus v2 > v1, thus m1 < m2 */
+    return GNUNET_YES;
+  }
+}
+
+
+void
+GCCH_handle_channel_plaintext_data (
+  struct CadetChannel *ch,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+  const struct GNUNET_CADET_ChannelAppDataMessage *msg)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalData *ld;
+  struct CadetChannelClient *ccc;
+  size_t payload_size;
+  struct CadetOutOfOrderMessage *com;
+  int duplicate;
+  uint32_t mid_min;
+  uint32_t mid_max;
+  uint32_t mid_msg;
+  uint32_t delta;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  if ((NULL == ch->owner) && (NULL == ch->dest))
+  {
+    /* This client is gone, but we still have messages to send to
+       the other end (which is why @a ch is not yet dead).  However,
+       we cannot pass messages to our client anymore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping incoming payload on %s as this end is already closed\n",
+         GCCH_2s (ch));
+    /* send back DESTROY notification to stop further retransmissions! */
+    if (GNUNET_YES == ch->destroy)
+      GCT_send_channel_destroy (ch->t, ch->ctn);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof(*msg);
+  env = GNUNET_MQ_msg_extra (ld,
+                             payload_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+  ld->ccn = (NULL == ch->dest) ? ch->owner->ccn : ch->dest->ccn;
+  GNUNET_memcpy (&ld[1], &msg[1], payload_size);
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if (GNUNET_YES == ccc->client_ready)
+  {
+    /*
+     * We ad-hoc send the message if
+     * - The channel is out-of-order
+     * - The channel is reliable and MID matches next expected MID
+     * - The channel is unreliable and MID is before lowest seen MID
+     */if ((GNUNET_YES == ch->out_of_order) ||
+        ((msg->mid.mid == ch->mid_recv.mid) && (GNUNET_YES == ch->reliable)) ||
+        ((GNUNET_NO == ch->reliable) &&
+         (ntohl (msg->mid.mid) >= ntohl (ch->mid_recv.mid)) &&
+         ((NULL == ccc->head_recv) ||
+          (ntohl (msg->mid.mid) < ntohl (ccc->head_recv->mid.mid)))))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Giving %u bytes of payload with MID %u from %s to client %s\n",
+           (unsigned int) payload_size,
+           ntohl (msg->mid.mid),
+           GCCH_2s (ch),
+           GSC_2s (ccc->c));
+      ccc->client_ready = GNUNET_NO;
+      GSC_send_to_client (ccc->c, env);
+      if (GNUNET_NO == ch->out_of_order)
+        ch->mid_recv.mid = htonl (1 + ntohl (msg->mid.mid));
+      else
+        ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
+      ch->mid_futures >>= 1;
+      if ((GNUNET_YES == ch->out_of_order) && (GNUNET_NO == ch->reliable))
+      {
+        /* possibly shift by more if we skipped messages */
+        uint64_t delta = htonl (msg->mid.mid) - 1 - ntohl (ch->mid_recv.mid);
+
+        if (delta > 63)
+          ch->mid_futures = 0;
+        else
+          ch->mid_futures >>= delta;
+        ch->mid_recv.mid = htonl (1 + ntohl (msg->mid.mid));
+      }
+      send_channel_data_ack (ch);
+      return;
+    }
+  }
+
+  if (GNUNET_YES == ch->reliable)
+  {
+    /* check if message ought to be dropped because it is ancient/too distant/duplicate */
+    mid_min = ntohl (ch->mid_recv.mid);
+    mid_max = mid_min + ch->max_pending_messages;
+    mid_msg = ntohl (msg->mid.mid);
+    if (((uint32_t) (mid_msg - mid_min) > ch->max_pending_messages) ||
+        ((uint32_t) (mid_max - mid_msg) > ch->max_pending_messages))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "%s at %u drops ancient or far-future message %u\n",
+           GCCH_2s (ch),
+           (unsigned int) mid_min,
+           ntohl (msg->mid.mid));
+
+      GNUNET_STATISTICS_update (stats,
+                                "# duplicate DATA (ancient or future)",
+                                1,
+                                GNUNET_NO);
+      GNUNET_MQ_discard (env);
+      send_channel_data_ack (ch);
+      return;
+    }
+    /* mark bit for future ACKs */
+    delta = mid_msg - mid_min - 1;   /* overflow/underflow are OK here */
+    if (delta < 64)
+    {
+      if (0 != (ch->mid_futures & (1LLU << delta)))
+      {
+        /* Duplicate within the queue, drop also */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+             (unsigned int) payload_size,
+             GCCH_2s (ch),
+             ntohl (msg->mid.mid));
+        GNUNET_STATISTICS_update (stats, "# duplicate DATA", 1, GNUNET_NO);
+        GNUNET_MQ_discard (env);
+        send_channel_data_ack (ch);
+        return;
+      }
+      ch->mid_futures |= (1LLU << delta);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Marked bit %llX for mid %u (base: %u); now: %llX\n",
+           (1LLU << delta),
+           mid_msg,
+           mid_min,
+           (unsigned long long) ch->mid_futures);
+    }
+  }
+  else /* ! ch->reliable */
+  {
+    struct CadetOutOfOrderMessage *next_msg;
+
+    /**
+     * We always send if possible in this case.
+     * It is guaranteed that the queued MID < received MID
+     **/
+    if ((NULL != ccc->head_recv) && (GNUNET_YES == ccc->client_ready))
+    {
+      next_msg = ccc->head_recv;
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Giving queued MID %u from %s to client %s\n",
+           ntohl (next_msg->mid.mid),
+           GCCH_2s (ch),
+           GSC_2s (ccc->c));
+      ccc->client_ready = GNUNET_NO;
+      GSC_send_to_client (ccc->c, next_msg->env);
+      ch->mid_recv.mid = htonl (1 + ntohl (next_msg->mid.mid));
+      ch->mid_futures >>= 1;
+      send_channel_data_ack (ch);
+      GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, next_msg);
+      ccc->num_recv--;
+      /* Do not process duplicate MID */
+      if (msg->mid.mid == next_msg->mid.mid)     /* Duplicate */
+      {
+        /* Duplicate within the queue, drop */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Message on %s (mid %u) dropped, duplicate\n",
+             GCCH_2s (ch),
+             ntohl (msg->mid.mid));
+        GNUNET_free (next_msg);
+        GNUNET_MQ_discard (env);
+        return;
+      }
+      GNUNET_free (next_msg);
+    }
+
+    if (ntohl (msg->mid.mid) < ntohl (ch->mid_recv.mid)) /* Old */
+    {
+      /* Duplicate within the queue, drop */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Message on %s (mid %u) dropped, old.\n",
+           GCCH_2s (ch),
+           ntohl (msg->mid.mid));
+      GNUNET_MQ_discard (env);
+      return;
+    }
+
+    /* Channel is unreliable, so we do not ACK. But we also cannot
+       allow buffering everything, so check if we have space... */
+    if (ccc->num_recv >= ch->max_pending_messages)
+    {
+      struct CadetOutOfOrderMessage *drop;
+
+      /* Yep, need to drop. Drop the oldest message in
+         the buffer. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Queue full due slow client on %s, dropping oldest message\n",
+           GCCH_2s (ch));
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to slow client",
+                                1,
+                                GNUNET_NO);
+      drop = ccc->head_recv;
+      GNUNET_assert (NULL != drop);
+      GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, drop);
+      ccc->num_recv--;
+      GNUNET_MQ_discard (drop->env);
+      GNUNET_free (drop);
+    }
+  }
+
+  /* Insert message into sorted out-of-order queue */
+  com = GNUNET_new (struct CadetOutOfOrderMessage);
+  com->mid = msg->mid;
+  com->env = env;
+  duplicate = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
+                                      is_before,
+                                      &duplicate,
+                                      ccc->head_recv,
+                                      ccc->tail_recv,
+                                      com);
+  ccc->num_recv++;
+  if (GNUNET_YES == duplicate)
+  {
+    /* Duplicate within the queue, drop also (this is not covered by
+       the case above if "delta" >= 64, which could be the case if
+       max_pending_messages is also >= 64 or if our client is unready
+       and we are seeing retransmissions of the message our client is
+       blocked on. */LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+         (unsigned int) payload_size,
+         GCCH_2s (ch),
+         ntohl (msg->mid.mid));
+    GNUNET_STATISTICS_update (stats, "# duplicate DATA", 1, GNUNET_NO);
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+    send_channel_data_ack (ch);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queued %s payload of %u bytes on %s-%X(%p) (mid %u, need %u first)\n",
+       (GNUNET_YES == ccc->client_ready) ? "out-of-order" : "client-not-ready",
+       (unsigned int) payload_size,
+       GCCH_2s (ch),
+       ntohl (ccc->ccn.channel_of_client),
+       ccc,
+       ntohl (msg->mid.mid),
+       ntohl (ch->mid_recv.mid));
+  /* NOTE: this ACK we _could_ skip, as the packet is out-of-order and
+     the sender may already be transmitting the previous one.  Needs
+     experimental evaluation to see if/when this ACK helps or
+     hurts. (We might even want another option.) */
+  send_channel_data_ack (ch);
+}
+
+
+/**
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
+ */
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
+ * We need to retry a transmission, the last one took too long to
+ * be acknowledged.
+ *
+ * @param cls the `struct CadetChannel` where we need to retransmit
+ */
+static void
+retry_transmission (void *cls)
+{
+  struct CadetChannel *ch = cls;
+  struct CadetReliableMessage *crm = ch->head_sent;
+
+  ch->retry_data_task = NULL;
+  GNUNET_assert (NULL == crm->qe);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retrying transmission on %s of message %u\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid));
+  crm->qe = GCT_send (ch->t, &crm->data_message->header, &data_sent_cb, crm,
+                      &crm->data_message->ctn);
+  GNUNET_assert (NULL == ch->retry_data_task);
+}
+
+
+/**
+ * We got an PLAINTEXT_DATA_ACK for a message in our queue, remove it from
+ * the queue and tell our client that it can send more.
+ *
+ * @param ch the channel that got the PLAINTEXT_DATA_ACK
+ * @param cti identifier of the connection that delivered the message
+ * @param crm the message that got acknowledged
+ */
+static void
+handle_matching_ack (struct CadetChannel *ch,
+                     const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                     struct CadetReliableMessage *crm)
+{
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent, ch->tail_sent, crm);
+  ch->pending_messages--;
+  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       ch->pending_messages);
+  if (NULL != crm->qe)
+  {
+    GCT_send_cancel (crm->qe);
+    crm->qe = NULL;
+  }
+  if ((1 == crm->num_transmissions) && (NULL != cti))
+  {
+    GCC_ack_observed (cti);
+    if (0 == GNUNET_memcmp (cti, &crm->connection_taken))
+    {
+      GCC_latency_observed (cti,
+                            GNUNET_TIME_absolute_get_duration (
+                              crm->first_transmission_time));
+    }
+  }
+  GNUNET_free (crm->data_message);
+  GNUNET_free (crm);
+  send_ack_to_client (ch, (NULL == ch->owner) ? GNUNET_NO : GNUNET_YES);
+}
+
+
+/**
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
+ *
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
+ */
+void
+GCCH_handle_channel_plaintext_data_ack (
+  struct CadetChannel *ch,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+  const struct GNUNET_CADET_ChannelDataAckMessage *ack)
+{
+  struct CadetReliableMessage *crm;
+  struct CadetReliableMessage *crmn;
+  int found;
+  uint32_t mid_base;
+  uint64_t mid_mask;
+  unsigned int delta;
+
+  GNUNET_break (GNUNET_NO == ch->is_loopback);
+  if (GNUNET_NO == ch->reliable)
+  {
+    /* not expecting ACKs on unreliable channel, odd */
+    GNUNET_break_op (0);
+    return;
+  }
+  /* mid_base is the MID of the next message that the
+     other peer expects (i.e. that is missing!), everything
+     LOWER (but excluding mid_base itself) was received. */
+  mid_base = ntohl (ack->mid.mid);
+  mid_mask = GNUNET_htonll (ack->futures);
+  found = GNUNET_NO;
+  for (crm = ch->head_sent; NULL != crm; crm = crmn)
+  {
+    crmn = crm->next;
+    delta = (unsigned int) (ntohl (crm->data_message->mid.mid) - mid_base);
+    if (delta >= UINT_MAX - ch->max_pending_messages)
+    {
+      /* overflow, means crm was a bit in the past, so this ACK counts for it. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with base %u satisfying past message %u on %s\n",
+           (unsigned int) mid_base,
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch, cti, crm);
+      found = GNUNET_YES;
+      continue;
+    }
+    delta--;
+    if (delta >= 64)
+      continue;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Testing bit %llX for mid %u (base: %u)\n",
+         (1LLU << delta),
+         ntohl (crm->data_message->mid.mid),
+         mid_base);
+    if (0 != (mid_mask & (1LLU << delta)))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with mask for %u on %s\n",
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch, cti, crm);
+      found = GNUNET_YES;
+    }
+  }
+  if (GNUNET_NO == found)
+  {
+    /* ACK for message we already dropped, might have been a
+       duplicate ACK? Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate DATA_ACK on %s, ignoring\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats, "# duplicate DATA_ACKs", 1, GNUNET_NO);
+    return;
+  }
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if ((NULL != ch->head_sent) && (NULL == ch->head_sent->qe))
+    ch->retry_data_task = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                                   &retry_transmission,
+                                                   ch);
+}
+
+
+void
+GCCH_handle_remote_destroy (
+  struct CadetChannel *ch,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
+{
+  struct CadetChannelClient *ccc;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received remote channel DESTROY for %s\n",
+       GCCH_2s (ch));
+  if (GNUNET_YES == ch->destroy)
+  {
+    /* Local client already gone, this is instant-death. */
+    channel_destroy (ch);
+    return;
+  }
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if ((NULL != ccc) && (NULL != ccc->head_recv))
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Lost end of transmission due to remote shutdown on %s\n",
+         GCCH_2s (ch));
+    /* FIXME: change API to notify client about truncated transmission! */
+  }
+  ch->destroy = GNUNET_YES;
+  if (NULL != ccc)
+    GSC_handle_remote_channel_destroy (ccc->c, ccc->ccn, ch);
+  channel_destroy (ch);
+}
+
+
+/**
+ * Test if element @a e1 comes before element @a e2.
+ *
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param crm1 an element of to sort
+ * @param crm2 another element to sort
+ * @return #GNUNET_YES if @a e1 < @a e2, otherwise #GNUNET_NO
+ */
+static int
+cmp_crm_by_next_retry (void *cls,
+                       struct CadetReliableMessage *crm1,
+                       struct CadetReliableMessage *crm2)
+{
+  if (crm1->next_retry.abs_value_us < crm2->next_retry.abs_value_us)
+    return GNUNET_YES;
+  return GNUNET_NO;
+}
+
+
+/**
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
+ */
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetReliableMessage *crm = cls;
+  struct CadetChannel *ch = crm->ch;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  GNUNET_assert (NULL != crm->qe);
+  crm->qe = NULL;
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent, ch->tail_sent, crm);
+  if (GNUNET_NO == ch->reliable)
+  {
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
+    ch->pending_messages--;
+    send_ack_to_client (ch, (NULL == ch->owner) ? GNUNET_NO : GNUNET_YES);
+    return;
+  }
+  if (NULL == cid)
+  {
+    /* There was an error sending. */
+    crm->num_transmissions = GNUNET_SYSERR;
+  }
+  else if (GNUNET_SYSERR != crm->num_transmissions)
+  {
+    /* Increment transmission counter, and possibly store @a cid
+       if this was the first transmission. */
+    crm->num_transmissions++;
+    if (1 == crm->num_transmissions)
+    {
+      crm->first_transmission_time = GNUNET_TIME_absolute_get ();
+      crm->connection_taken = *cid;
+      GCC_ack_expected (cid);
+    }
+  }
+  if ((0 == crm->retry_delay.rel_value_us) && (NULL != cid))
+  {
+    struct CadetConnection *cc = GCC_lookup (cid);
+
+    if (NULL != cc)
+      crm->retry_delay = GCC_get_metrics (cc)->aged_latency;
+    else
+      crm->retry_delay = ch->retry_time;
+  }
+  crm->retry_delay = GNUNET_TIME_STD_BACKOFF (crm->retry_delay);
+  crm->retry_delay = GNUNET_TIME_relative_max (crm->retry_delay, MIN_RTT_DELAY);
+  crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
+
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetReliableMessage,
+                                      cmp_crm_by_next_retry,
+                                      NULL,
+                                      ch->head_sent,
+                                      ch->tail_sent,
+                                      crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message %u sent, next transmission on %s in %s\n",
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (
+         GNUNET_TIME_absolute_get_remaining (
+           ch->head_sent->next_retry),
+         GNUNET_YES));
+  if (NULL == ch->head_sent->qe)
+  {
+    if (NULL != ch->retry_data_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                                   &retry_transmission,
+                                                   ch);
+  }
+}
+
+
+/**
+ * Handle data given by a client.
+ *
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
+ *
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
+ */
+int
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len)
+{
+  struct CadetReliableMessage *crm;
+
+  if (ch->pending_messages >= ch->max_pending_messages)
+  {
+    GNUNET_break (0);  /* Fails: #5370 */
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_YES == ch->destroy)
+  {
+    /* we are going down, drop messages */
+    return GNUNET_OK;
+  }
+  ch->pending_messages++;
+
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    struct CadetChannelClient *receiver;
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_LocalData *ld;
+    int ack_to_owner;
+
+    env =
+      GNUNET_MQ_msg_extra (ld, buf_len, GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+    if ((NULL != ch->owner) &&
+        (sender_ccn.channel_of_client == ch->owner->ccn.channel_of_client))
+    {
+      receiver = ch->dest;
+      ack_to_owner = GNUNET_YES;
+    }
+    else if ((NULL != ch->dest) &&
+             (sender_ccn.channel_of_client == ch->dest->ccn.channel_of_client))
+    {
+      receiver = ch->owner;
+      ack_to_owner = GNUNET_NO;
+    }
+    else
+    {
+      GNUNET_free (env);
+      GNUNET_break (0);
+      return GNUNET_SYSERR;
+    }
+    GNUNET_assert (NULL != receiver);
+    ld->ccn = receiver->ccn;
+    GNUNET_memcpy (&ld[1], buf, buf_len);
+    if (GNUNET_YES == receiver->client_ready)
+    {
+      ch->pending_messages--;
+      GSC_send_to_client (receiver->c, env);
+      send_ack_to_client (ch, ack_to_owner);
+    }
+    else
+    {
+      struct CadetOutOfOrderMessage *oom;
+
+      oom = GNUNET_new (struct CadetOutOfOrderMessage);
+      oom->env = env;
+      GNUNET_CONTAINER_DLL_insert_tail (receiver->head_recv,
+                                        receiver->tail_recv,
+                                        oom);
+      receiver->num_recv++;
+    }
+    return GNUNET_OK;
+  }
+
+  /* Everything is correct, send the message. */
+  crm = GNUNET_malloc (sizeof(*crm));
+  crm->ch = ch;
+  crm->data_message = GNUNET_malloc (
+    sizeof(struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
+  crm->data_message->header.size =
+    htons (sizeof(struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
+  crm->data_message->header.type =
+    htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
+  ch->mid_send.mid = htonl (ntohl (ch->mid_send.mid) + 1);
+  crm->data_message->mid = ch->mid_send;
+  crm->data_message->ctn = ch->ctn;
+  GNUNET_memcpy (&crm->data_message[1], buf, buf_len);
+  GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent, ch->tail_sent, crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message %u from local client to %s with %lu bytes\n",
+       ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       (unsigned long) buf_len);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  crm->qe = GCT_send (ch->t, &crm->data_message->header, &data_sent_cb, crm,
+                      &crm->data_message->ctn);
+  GNUNET_assert (NULL == ch->retry_data_task);
+  return GNUNET_OK;
+}
+
+
+void
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn)
+{
+  struct CadetChannelClient *ccc;
+  struct CadetOutOfOrderMessage *com;
+
+  if ((NULL != ch->owner) &&
+      (ch->owner->ccn.channel_of_client == client_ccn.channel_of_client))
+    ccc = ch->owner;
+  else if ((NULL != ch->dest) &&
+           (ch->dest->ccn.channel_of_client == client_ccn.channel_of_client))
+    ccc = ch->dest;
+  else
+    GNUNET_assert (0);
+  ccc->client_ready = GNUNET_YES;
+  com = ccc->head_recv;
+  if (NULL == com)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data, but none pending on %s-%X(%p)!\n",
+         GSC_2s (ccc->c),
+         ntohl (client_ccn.channel_of_client),
+         GCCH_2s (ch),
+         ntohl (ccc->ccn.channel_of_client),
+         ccc);
+    return;   /* none pending */
+  }
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    int to_owner;
+
+    /* Messages are always in-order, just send */
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, com);
+    ccc->num_recv--;
+    GSC_send_to_client (ccc->c, com->env);
+    /* Notify sender that we can receive more */
+    if ((NULL != ch->owner) &&
+        (ccc->ccn.channel_of_client == ch->owner->ccn.channel_of_client))
+    {
+      to_owner = GNUNET_NO;
+    }
+    else
+    {
+      GNUNET_assert ((NULL != ch->dest) && (ccc->ccn.channel_of_client ==
+                                            ch->dest->ccn.channel_of_client));
+      to_owner = GNUNET_YES;
+    }
+    send_ack_to_client (ch, to_owner);
+    GNUNET_free (com);
+    return;
+  }
+
+  if ((com->mid.mid != ch->mid_recv.mid) && (GNUNET_NO == ch->out_of_order) &&
+      (GNUNET_YES == ch->reliable))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data (but next one is out-of-order %u vs. %u)!\n",
+         GSC_2s (ccc->c),
+         ntohl (ccc->ccn.channel_of_client),
+         ntohl (com->mid.mid),
+         ntohl (ch->mid_recv.mid));
+    return;   /* missing next one in-order */
+  }
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got LOCAL_ACK, giving payload message %u to %s-%X on %s\n",
+       ntohl (com->mid.mid),
+       GSC_2s (ccc->c),
+       ntohl (ccc->ccn.channel_of_client),
+       GCCH_2s (ch));
+
+  /* all good, pass next message to client */
+  GNUNET_CONTAINER_DLL_remove (ccc->head_recv, ccc->tail_recv, com);
+  ccc->num_recv--;
+  /* FIXME: if unreliable, this is not aggressive
+     enough, as it would be OK to have lost some! */
+
+  ch->mid_recv.mid = htonl (1 + ntohl (com->mid.mid));
+  ch->mid_futures >>= 1; /* equivalent to division by 2 */
+  ccc->client_ready = GNUNET_NO;
+  GSC_send_to_client (ccc->c, com->env);
+  GNUNET_free (com);
+  send_channel_data_ack (ch);
+  if (NULL != ccc->head_recv)
+    return;
+  if (GNUNET_NO == ch->destroy)
+    return;
+  GCT_send_channel_destroy (ch->t, ch->ctn);
+  channel_destroy (ch);
+}
+
+
+#define LOG2(level, ...) \
+  GNUNET_log_from_nocheck (level, "cadet-chn", __VA_ARGS__)
+
+
+/**
+ * Log channel info.
+ *
+ * @param ch Channel.
+ * @param level Debug level to use.
+ */
+void
+GCCH_debug (struct CadetChannel *ch, enum GNUNET_ErrorType level)
+{
+#if ! defined(GNUNET_CULL_LOGGING)
+  int do_log;
+
+  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
+                                       "cadet-chn",
+                                       __FILE__,
+                                       __FUNCTION__,
+                                       __LINE__);
+  if (0 == do_log)
+    return;
+
+  if (NULL == ch)
+  {
+    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
+    return;
+  }
+  LOG2 (level, "CHN %s:%X (%p)\n", GCT_2s (ch->t), ch->ctn.cn, ch);
+  if (NULL != ch->owner)
+  {
+    LOG2 (level,
+          "CHN origin %s ready %s local-id: %u\n",
+          GSC_2s (ch->owner->c),
+          ch->owner->client_ready ? "YES" : "NO",
+          ntohl (ch->owner->ccn.channel_of_client));
+  }
+  if (NULL != ch->dest)
+  {
+    LOG2 (level,
+          "CHN destination %s ready %s local-id: %u\n",
+          GSC_2s (ch->dest->c),
+          ch->dest->client_ready ? "YES" : "NO",
+          ntohl (ch->dest->ccn.channel_of_client));
+  }
+  LOG2 (level,
+        "CHN  Message IDs recv: %d (%llX), send: %d\n",
+        ntohl (ch->mid_recv.mid),
+        (unsigned long long) ch->mid_futures,
+        ntohl (ch->mid_send.mid));
+#endif
+}
+
+
+/* end of gnunet-service-cadet_channel.c */
diff --git a/src/service/cadet/gnunet-service-cadet_channel.h b/src/service/cadet/gnunet-service-cadet_channel.h
new file mode 100644
index 000000000..b09de883f
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_channel.h
@@ -0,0 +1,306 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_channel.h
+ * @brief GNUnet CADET service with encryption
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_CHANNEL_H
+#define GNUNET_SERVICE_CADET_CHANNEL_H
+
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_peer.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * A channel is a bidirectional connection between two CADET
+ * clients.  Communication can be reliable, unreliable, in-order
+ * or out-of-order.  One client is the "local" client, this
+ * one initiated the connection.   The other client is the
+ * "incoming" client, this one listened on a port to accept
+ * the connection from the "local" client.
+ */
+struct CadetChannel;
+
+
+/**
+ * Hash the @a port and @a initiator and @a listener to
+ * calculate the "challenge" @a h_port we send to the other
+ * peer on #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN.
+ *
+ * @param[out] h_port set to the hash of @a port, @a initiator and @a listener
+ * @param port cadet port, as seen by CADET clients
+ * @param listener peer that is listining on @a port
+ */
+void
+GCCH_hash_port (struct GNUNET_HashCode *h_port,
+                const struct GNUNET_HashCode *port,
+                const struct GNUNET_PeerIdentity *listener);
+
+/**
+ * Check if type of message is the one to drop.
+ * @param ch CadetChannel to check for message type to drop.
+ * @param message GNUNET_MessageHeader to compare the type with.
+ */
+int
+GCCH_is_type_to_drop (struct CadetChannel *ch, const struct
+                      GNUNET_MessageHeader *message);
+
+/**
+ * Assign type of message to drop.
+ * @param ch CadetChannel to assign type to drop.
+ * @param message GNUNET_CADET_RequestDropCadetMessage to get the type from.
+ */
+void
+GCCH_assign_type_to_drop (struct CadetChannel *ch, const struct
+                          GNUNET_CADET_RequestDropCadetMessage *message);
+
+/**
+ * Get the static string for identification of the channel.
+ *
+ * @param ch Channel.
+ *
+ * @return Static string with the channel IDs.
+ */
+const char *
+GCCH_2s (const struct CadetChannel *ch);
+
+
+/**
+ * Log channel info.
+ *
+ * @param ch Channel.
+ * @param level Debug level to use.
+ */
+void
+GCCH_debug (struct CadetChannel *ch,
+            enum GNUNET_ErrorType level);
+
+
+/**
+ * Get the channel's public ID.
+ *
+ * @param ch Channel.
+ *
+ * @return ID used to identify the channel with the remote peer.
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch);
+
+
+/**
+ * Create a new channel.
+ *
+ * @param owner local client owning the channel
+ * @param owner_id local chid of this channel at the @a owner
+ * @param destination peer to which we should build the channel
+ * @param port desired port at @a destination
+ * @param options options for the channel
+ * @return handle to the new channel
+ */
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber owner_id,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options);
+
+
+/**
+ * A client is bound to the port that we have a channel
+ * open to.  Send the acknowledgement for the connection
+ * request and establish the link with the client.
+ *
+ * @param ch open incoming channel
+ * @param c client listening on the respective @a port
+ * @param port port number @a c is listening on
+ */
+void
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c,
+           const struct GNUNET_HashCode *port);
+
+
+/**
+ * Destroy locally created channel.  Called by the
+ * local client, so no need to tell the client.
+ *
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
+ */
+void
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn);
+
+
+/**
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
+ */
+void
+GCCH_tunnel_up (struct CadetChannel *ch);
+
+
+/**
+ * Create a new channel based on a request coming in over the network.
+ *
+ * @param t tunnel to the remote peer
+ * @param chid identifier of this channel in the tunnel
+ * @param h_port hash of desired local port
+ * @param options options for the channel
+ * @return handle to the new channel
+ */
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber chid,
+                           const struct GNUNET_HashCode *h_port,
+                           uint32_t options);
+
+
+/**
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
+ *
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
+ */
+void
+GCCH_handle_duplicate_open (struct CadetChannel *ch,
+                            const struct
+                            GNUNET_CADET_ConnectionTunnelIdentifier *cti);
+
+
+/**
+ * We got payload data for a channel.  Pass it on to the client
+ * and send an ACK to the other end (once flow control allows it!)
+ *
+ * @param ch channel that got data
+ * @param cti identifier of the connection that delivered the message
+ * @param msg message that was received
+ */
+void
+GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
+                                    const struct
+                                    GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                    const struct
+                                    GNUNET_CADET_ChannelAppDataMessage *msg);
+
+
+/**
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
+ *
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
+ */
+void
+GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
+                                        const struct
+                                        GNUNET_CADET_ConnectionTunnelIdentifier
+                                        *cti,
+                                        const struct
+                                        GNUNET_CADET_ChannelDataAckMessage *ack);
+
+
+/**
+ * We got an acknowledgement for the creation of the channel
+ * (the port is open on the other side).  Verify that the
+ * other end really has the right port, and begin transmissions.
+ *
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *        NULL if the ACK was inferred because we got payload or are on loopback
+ * @param port port number (needed to verify receiver knows the port)
+ */
+void
+GCCH_handle_channel_open_ack (struct CadetChannel *ch,
+                              const struct
+                              GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                              const struct GNUNET_HashCode *port);
+
+
+/**
+ * Destroy channel, based on the other peer closing the
+ * connection.  Also needs to remove this channel from
+ * the tunnel.
+ *
+ * FIXME: need to make it possible to defer destruction until we have
+ * received all messages up to the destroy, and right now the destroy
+ * message (and this API) fails to give is the information we need!
+ *
+ * FIXME: also need to know if the other peer got a destroy from
+ * us before!
+ *
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *            NULL if we are simulating receiving a destroy due to shutdown
+ */
+void
+GCCH_handle_remote_destroy (struct CadetChannel *ch,
+                            const struct
+                            GNUNET_CADET_ConnectionTunnelIdentifier *cti);
+
+
+/**
+ * Handle data given by a client.
+ *
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
+ *
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
+ */
+int
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len);
+
+
+/**
+ * Handle ACK from client on local channel.  Means the client is ready
+ * for more data, see if we have any for it.
+ *
+ * @param ch channel to destroy
+ * @param client_ccn ccn of the client sending the ack
+ */
+void
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn);
+
+#endif
diff --git a/src/service/cadet/gnunet-service-cadet_connection.c b/src/service/cadet/gnunet-service-cadet_connection.c
new file mode 100644
index 000000000..f01c1388e
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_connection.c
@@ -0,0 +1,1079 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_connection.c
+ * @brief management of CORE-level end-to-end connections; establishes
+ *        end-to-end routes and transmits messages along the route
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_signatures.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet_cadet_service.h"
+#include "gnunet_statistics_service.h"
+#include "cadet_protocol.h"
+
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-con", __VA_ARGS__)
+
+
+/**
+ * How long do we wait initially before retransmitting the KX?
+ * TODO: replace by 2 RTT if/once we have connection-level RTT data!
+ */
+#define INITIAL_CONNECTION_CREATE_RETRY_DELAY \
+  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 200)
+
+
+/**
+ * All the states a connection can be in.
+ */
+enum CadetConnectionState
+{
+  /**
+   * Uninitialized status, we have not yet even gotten the message queue.
+   */
+  CADET_CONNECTION_NEW,
+
+  /**
+   * Connection create message in queue, awaiting transmission by CORE.
+   */
+  CADET_CONNECTION_SENDING_CREATE,
+
+  /**
+   * Connection create message sent, waiting for ACK.
+   */
+  CADET_CONNECTION_SENT,
+
+  /**
+   * We are an inbound connection, and received a CREATE. Need to
+   * send an CREATE_ACK back.
+   */
+  CADET_CONNECTION_CREATE_RECEIVED,
+
+  /**
+   * Connection confirmed, ready to carry traffic.
+   */
+  CADET_CONNECTION_READY
+};
+
+/**
+ * Low-level connection to a destination.
+ */
+struct CadetConnection
+{
+  /**
+   * ID of the connection.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * To which peer does this connection go?
+   */
+  struct CadetPeer *destination;
+
+  /**
+   * Which tunnel is using this connection?
+   */
+  struct CadetTConnection *ct;
+
+  /**
+   * Path we are using to our destination.
+   */
+  struct CadetPeerPath *path;
+
+  /**
+   * Pending message, NULL if we are ready to transmit.
+   */
+  struct GNUNET_MQ_Envelope *env;
+
+  /**
+   * Handle for calling #GCP_request_mq_cancel() once we are finished.
+   */
+  struct GCP_MessageQueueManager *mq_man;
+
+  /**
+   * Task for connection maintenance.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Queue entry for keepalive messages.
+   */
+  struct CadetTunnelQueueEntry *keepalive_qe;
+
+  /**
+   * Function to call once we are ready to transmit.
+   */
+  GCC_ReadyCallback ready_cb;
+
+  /**
+   * Closure for @e ready_cb.
+   */
+  void *ready_cb_cls;
+
+  /**
+   * How long do we wait before we try again with a CREATE message?
+   */
+  struct GNUNET_TIME_Relative retry_delay;
+
+  /**
+   * Earliest time for re-trying CREATE
+   */
+  struct GNUNET_TIME_Absolute create_at;
+
+  /**
+   * Earliest time for re-trying CREATE_ACK
+   */
+  struct GNUNET_TIME_Absolute create_ack_at;
+
+  /**
+   * Performance metrics for this connection.
+   */
+  struct CadetConnectionMetrics metrics;
+
+  /**
+   * State of the connection.
+   */
+  enum CadetConnectionState state;
+
+  /**
+   * How many latency observations did we make for this connection?
+   */
+  unsigned int latency_datapoints;
+
+  /**
+   * Offset of our @e destination in @e path.
+   */
+  unsigned int off;
+
+  /**
+   * Are we ready to transmit via @e mq_man right now?
+   */
+  int mqm_ready;
+};
+
+
+/**
+ * Lookup a connection by its identifier.
+ *
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
+ */
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  return GNUNET_CONTAINER_multishortmap_get (connections,
+                                             &cid->connection_of_tunnel);
+}
+
+
+/**
+ * Update the connection state. Also triggers the necessary
+ * MQM notifications.
+ *
+ * @param cc connection to update the state for
+ * @param new_state new state for @a cc
+ * @param new_mqm_ready new `mqm_ready` state for @a cc
+ */
+static void
+update_state (struct CadetConnection *cc,
+              enum CadetConnectionState new_state,
+              int new_mqm_ready)
+{
+  int old_ready;
+  int new_ready;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Trying to update connection state for %s having old state %d to new %d and mqm_ready old %d to mqm_ready new %d\n",
+       GCT_2s (cc->ct->t),
+       cc->state,
+       new_state,
+       cc->mqm_ready,
+       new_mqm_ready);
+
+  if ((new_state == cc->state) && (new_mqm_ready == cc->mqm_ready))
+    return; /* no change, nothing to do */
+  old_ready =
+    ((CADET_CONNECTION_READY == cc->state) && (GNUNET_YES == cc->mqm_ready));
+  new_ready =
+    ((CADET_CONNECTION_READY == new_state) && (GNUNET_YES == new_mqm_ready));
+  cc->state = new_state;
+  cc->mqm_ready = new_mqm_ready;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Updating connection state for %s having old_ready %d and new_rady %d\n",
+       GCT_2s (cc->ct->t),
+       old_ready,
+       new_ready);
+
+  if (old_ready != new_ready)
+    cc->ready_cb (cc->ready_cb_cls, new_ready);
+}
+
+
+/**
+ * Destroy a connection, part of the internal implementation.  Called
+ * only from #GCC_destroy_from_core() or #GCC_destroy_from_tunnel().
+ *
+ * @param cc connection to destroy
+ */
+static void
+GCC_destroy (struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Destroying %s\n", GCC_2s (cc));
+  if (NULL != cc->mq_man)
+  {
+    GCP_request_mq_cancel (cc->mq_man, NULL);
+    cc->mq_man = NULL;
+  }
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
+  if (NULL != cc->keepalive_qe)
+  {
+    GCT_send_cancel (cc->keepalive_qe);
+    cc->keepalive_qe = NULL;
+  }
+  GCPP_del_connection (cc->path, cc->off, cc);
+  for (unsigned int i = 0; i < cc->off; i++)
+    GCP_remove_connection (GCPP_get_peer_at_offset (cc->path, i), cc);
+  GNUNET_assert (
+    GNUNET_YES ==
+    GNUNET_CONTAINER_multishortmap_remove (connections,
+                                           &GCC_get_id (cc)
+                                           ->connection_of_tunnel,
+                                           cc));
+  GNUNET_free (cc);
+}
+
+
+/**
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
+ *
+ * @param cc connection to destroy
+ */
+void
+GCC_destroy_without_core (struct CadetConnection *cc)
+{
+  if (NULL != cc->ct)
+  {
+    GCT_connection_lost (cc->ct);
+    cc->ct = NULL;
+  }
+  GCC_destroy (cc);
+}
+
+
+/**
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
+ *
+ * @param cc connection to destroy
+ */
+void
+GCC_destroy_without_tunnel (struct CadetConnection *cc)
+{
+  cc->ct = NULL;
+  if ((CADET_CONNECTION_SENDING_CREATE != cc->state) && (NULL != cc->mq_man))
+  {
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_ConnectionDestroyMessage *destroy_msg;
+
+    /* Need to notify next hop that we are down. */
+    env =
+      GNUNET_MQ_msg (destroy_msg, GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
+    destroy_msg->cid = cc->cid;
+    GCP_request_mq_cancel (cc->mq_man, env);
+    cc->mq_man = NULL;
+  }
+  GCC_destroy (cc);
+}
+
+
+/**
+ * Return the tunnel associated with this connection.
+ *
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
+ */
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc)
+{
+  return cc->ct;
+}
+
+
+/**
+ * Obtain performance @a metrics from @a cc.
+ *
+ * @param cc connection to query
+ * @return the metrics
+ */
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc)
+{
+  return &cc->metrics;
+}
+
+
+/**
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ */
+static void
+send_keepalive (void *cls);
+
+
+/**
+ * Keepalive was transmitted.  Remember this, and possibly
+ * schedule the next one.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
+ */
+static void
+keepalive_done (void *cls,
+                const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetConnection *cc = cls;
+
+  cc->keepalive_qe = NULL;
+  if ((GNUNET_YES == cc->mqm_ready) && (NULL == cc->task))
+    cc->task =
+      GNUNET_SCHEDULER_add_delayed (keepalive_period, &send_keepalive, cc);
+}
+
+
+/**
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ */
+static void
+send_keepalive (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_MessageHeader msg;
+
+  cc->task = NULL;
+  if (CADET_TUNNEL_KEY_OK != GCT_get_estate (cc->ct->t))
+  {
+    /* Tunnel not yet ready, wait with keepalives... */
+    cc->task =
+      GNUNET_SCHEDULER_add_delayed (keepalive_period, &send_keepalive, cc);
+    return;
+  }
+  GNUNET_assert (NULL != cc->ct);
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (NULL == cc->keepalive_qe);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Sending KEEPALIVE on behalf of %s via %s\n",
+       GCC_2s (cc),
+       GCT_2s (cc->ct->t));
+  GNUNET_STATISTICS_update (stats, "# keepalives sent", 1, GNUNET_NO);
+  msg.size = htons (sizeof(msg));
+  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
+
+  cc->keepalive_qe = GCT_send (cc->ct->t, &msg, &keepalive_done, cc, NULL);
+}
+
+
+/**
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
+ *
+ * @param cid connection identifier where we expect an ACK
+ */
+void
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetConnection *cc;
+
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection already down? */
+  cc->metrics.num_acked_transmissions++;
+}
+
+
+/**
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cid.
+ *
+ * @param cid connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
+ */
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  struct CadetConnection *cc;
+
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection already down? */
+  cc->metrics.num_successes++;
+}
+
+
+void
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                      struct GNUNET_TIME_Relative latency)
+{
+  struct CadetConnection *cc;
+  double weight;
+  double result;
+
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection already down? */
+  GNUNET_STATISTICS_update (stats, "# latencies observed", 1, GNUNET_NO);
+  cc->latency_datapoints++;
+  if (cc->latency_datapoints >= 7)
+    weight = 7.0;
+  else
+    weight = cc->latency_datapoints;
+  /* Compute weighted average, giving at MOST weight 7 to the
+     existing values, or less if that value is based on fewer than 7
+     measurements. */
+  result = (weight * cc->metrics.aged_latency.rel_value_us)
+           + 1.0 * latency.rel_value_us;
+  result /= (weight + 1.0);
+  cc->metrics.aged_latency.rel_value_us = (uint64_t) result;
+}
+
+
+void
+GCC_handle_connection_create_ack (struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CADET_CONNECTION_CREATE_ACK for %s in state %d (%s)\n",
+       GCC_2s (cc),
+       cc->state,
+       (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+  if (CADET_CONNECTION_READY == cc->state)
+    return; /* Duplicate ACK, ignore */
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
+  cc->metrics.age = GNUNET_TIME_absolute_get ();
+  update_state (cc, CADET_CONNECTION_READY, cc->mqm_ready);
+  if ((NULL == cc->keepalive_qe) && (GNUNET_YES == cc->mqm_ready) &&
+      (NULL == cc->task))
+    cc->task =
+      GNUNET_SCHEDULER_add_delayed (keepalive_period, &send_keepalive, cc);
+}
+
+
+/**
+ * Handle KX message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KX message with ephermal %s on CC %s in state %d\n",
+       GNUNET_e2s (&msg->ephemeral_key),
+       GNUNET_sh2s (&cc->cid.connection_of_tunnel),
+       cc->state);
+  if (CADET_CONNECTION_SENT == cc->state)
+  {
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
+  }
+  GCT_handle_kx (cc->ct, msg);
+}
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KX AUTH message with ephermal %s on CC %s in state %d\n",
+       GNUNET_e2s (&msg->kx.ephemeral_key),
+       GNUNET_sh2s (&cc->cid.connection_of_tunnel),
+       cc->state);
+  if (CADET_CONNECTION_SENT == cc->state)
+  {
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
+  }
+  GCT_handle_kx_auth (cc->ct, msg);
+}
+
+
+/**
+ * Handle encrypted message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
+ */
+void
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+{
+  if (CADET_CONNECTION_SENT == cc->state)
+  {
+    /* We didn't get the CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Faking connection ACK for %s due to ENCRYPTED payload\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
+  }
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  GCT_handle_encrypted (cc->ct, msg);
+}
+
+
+/**
+ * Set the signature for a monotime value on a GNUNET_CADET_ConnectionCreateMessage.
+ *
+ * @param msg The GNUNET_CADET_ConnectionCreateMessage.
+ */
+void
+set_monotime_sig (struct GNUNET_CADET_ConnectionCreateMessage *msg)
+{
+
+  struct CadetConnectionCreatePS cp = { .purpose.purpose = htonl (
+                                          GNUNET_SIGNATURE_PURPOSE_CADET_CONNECTION_INITIATOR),
+                                        .purpose.size = htonl (sizeof(cp)),
+                                        .monotonic_time = msg->monotime};
+
+  GNUNET_CRYPTO_eddsa_sign (my_private_key, &cp,
+                            &msg->monotime_sig);
+
+}
+
+
+/**
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE message to the
+ * first hop.
+ *
+ * @param cls the `struct CadetConnection` to initiate
+ */
+static void
+send_create (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateMessage *create_msg;
+  struct GNUNET_PeerIdentity *pids;
+  struct GNUNET_MQ_Envelope *env;
+  struct CadetTunnel *t;
+
+  cc->task = NULL;
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  env =
+    GNUNET_MQ_msg_extra (create_msg,
+                         (2 + cc->off) * sizeof(struct GNUNET_PeerIdentity),
+                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
+  // TODO This will be removed in a major release, because this will be a protocol breaking change. We set the deprecated 'reliable' bit here that was removed.
+  create_msg->options = 2;
+  create_msg->cid = cc->cid;
+
+  // check for tunnel state and set signed monotime (xrs,t3ss)
+  t = GCP_get_tunnel (cc->destination, GNUNET_YES);
+  if ((NULL != t) && (GCT_get_estate (t) == CADET_TUNNEL_KEY_UNINITIALIZED) &&
+      (GCT_alice_or_betty (GCP_get_id (cc->destination)) == GNUNET_NO))
+  {
+    create_msg->has_monotime = GNUNET_YES;
+    create_msg->monotime = GNUNET_TIME_absolute_hton (
+      GNUNET_TIME_absolute_get_monotonic (cfg));
+    set_monotime_sig (create_msg);
+  }
+
+  pids = (struct GNUNET_PeerIdentity *) &create_msg[1];
+  pids[0] = my_full_id;
+  for (unsigned int i = 0; i <= cc->off; i++)
+    pids[i + 1] = *GCP_get_id (GCPP_get_peer_at_offset (cc->path, i));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_CONNECTION_CREATE message for %s with %u hops\n",
+       GCC_2s (cc),
+       cc->off + 2);
+  cc->env = env;
+  cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
+  cc->create_at = GNUNET_TIME_relative_to_absolute (cc->retry_delay);
+  update_state (cc, CADET_CONNECTION_SENT, GNUNET_NO);
+  GCP_send (cc->mq_man, env);
+}
+
+
+/**
+ * Send a CREATE_ACK message towards the origin.
+ *
+ * @param cls the `struct CadetConnection` to initiate
+ */
+static void
+send_create_ack (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateAckMessage *ack_msg;
+  struct GNUNET_MQ_Envelope *env;
+
+  cc->task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CONNECTION_CREATE_ACK message for %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  env =
+    GNUNET_MQ_msg (ack_msg, GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK);
+  ack_msg->cid = cc->cid;
+  cc->env = env;
+  cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
+  cc->create_ack_at = GNUNET_TIME_relative_to_absolute (cc->retry_delay);
+  if (CADET_CONNECTION_CREATE_RECEIVED == cc->state)
+    update_state (cc, CADET_CONNECTION_READY, GNUNET_NO);
+  if (CADET_CONNECTION_READY == cc->state)
+    cc->task =
+      GNUNET_SCHEDULER_add_delayed (keepalive_period, &send_keepalive, cc);
+  GCP_send (cc->mq_man, env);
+}
+
+
+/**
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
+ *
+ * @param cc connection that got the duplicate CREATE
+ */
+void
+GCC_handle_duplicate_create (struct CadetConnection *cc)
+{
+  if (GNUNET_YES == cc->mqm_ready)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate CREATE for %s, scheduling another ACK (%s)\n",
+         GCC_2s (cc),
+         (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+    /* Revert back to the state of having only received the 'CREATE',
+       and immediately proceed to send the CREATE_ACK. */
+    update_state (cc, CADET_CONNECTION_CREATE_RECEIVED, cc->mqm_ready);
+    if (NULL != cc->task)
+      GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task =
+      GNUNET_SCHEDULER_add_at (cc->create_ack_at, &send_create_ack, cc);
+  }
+  else
+  {
+    /* We are currently sending something else back, which
+       can only be an ACK or payload, either of which would
+       do. So actually no need to do anything. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate CREATE for %s. MQ is busy, not queueing another ACK\n",
+         GCC_2s (cc));
+  }
+}
+
+
+/**
+ * There has been a change in the message queue existence for our
+ * peer at the first hop.  Adjust accordingly.
+ *
+ * @param cls the `struct CadetConnection`
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
+ */
+static void
+manage_first_hop_mq (void *cls, int available)
+{
+  struct CadetConnection *cc = cls;
+
+  if (GNUNET_YES != available)
+  {
+    /* Connection is down, for now... */
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "Core MQ for %s went down\n", GCC_2s (cc));
+    update_state (cc, CADET_CONNECTION_NEW, GNUNET_NO);
+    cc->retry_delay = INITIAL_CONNECTION_CREATE_RETRY_DELAY;
+    if (NULL != cc->task)
+    {
+      GNUNET_SCHEDULER_cancel (cc->task);
+      cc->task = NULL;
+    }
+    return;
+  }
+
+  update_state (cc, cc->state, GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Core MQ for %s became available in state %d\n",
+       GCC_2s (cc),
+       cc->state);
+  switch (cc->state)
+  {
+  case CADET_CONNECTION_NEW:
+    /* Transmit immediately */
+    cc->task = GNUNET_SCHEDULER_add_at (cc->create_at, &send_create, cc);
+    break;
+
+  case CADET_CONNECTION_SENDING_CREATE:
+    /* Should not be possible to be called in this state. */
+    GNUNET_assert (0);
+    break;
+
+  case CADET_CONNECTION_SENT:
+    /* Retry a bit later... */
+    cc->task = GNUNET_SCHEDULER_add_at (cc->create_at, &send_create, cc);
+    break;
+
+  case CADET_CONNECTION_CREATE_RECEIVED:
+    /* We got the 'CREATE' (incoming connection), should send the CREATE_ACK */
+    cc->metrics.age = GNUNET_TIME_absolute_get ();
+    cc->task =
+      GNUNET_SCHEDULER_add_at (cc->create_ack_at, &send_create_ack, cc);
+    break;
+
+  case CADET_CONNECTION_READY:
+    if ((NULL == cc->keepalive_qe) && (GNUNET_YES == cc->mqm_ready) &&
+        (NULL == cc->task))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Scheduling keepalive for %s in %s\n",
+           GCC_2s (cc),
+           GNUNET_STRINGS_relative_time_to_string (keepalive_period,
+                                                   GNUNET_YES));
+      cc->task =
+        GNUNET_SCHEDULER_add_delayed (keepalive_period, &send_keepalive, cc);
+    }
+    break;
+  }
+}
+
+
+/**
+ * Create a connection to @a destination via @a path and notify @a cb
+ * whenever we are ready for more data.  Shared logic independent of
+ * who is initiating the connection.
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param ct which tunnel uses this connection
+ * @param init_state initial state for the connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
+ */
+static struct CadetConnection *
+connection_create (struct CadetPeer *destination,
+                   struct CadetPeerPath *path,
+                   unsigned int off,
+                   struct CadetTConnection *ct,
+                   const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                   enum CadetConnectionState init_state,
+                   GCC_ReadyCallback ready_cb,
+                   void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  struct CadetPeer *first_hop;
+
+  cc = GNUNET_new (struct CadetConnection);
+  cc->state = init_state;
+  cc->ct = ct;
+  cc->destination = destination; /* xrs,t3ss,lurchi*/
+  cc->cid = *cid;
+  cc->retry_delay =
+    GNUNET_TIME_relative_multiply (INITIAL_CONNECTION_CREATE_RETRY_DELAY, off);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multishortmap_put (
+                   connections,
+                   &GCC_get_id (cc)->connection_of_tunnel,
+                   cc,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  cc->ready_cb = ready_cb;
+  cc->ready_cb_cls = ready_cb_cls;
+  cc->path = path;
+  cc->off = off;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating %s using path %s (offset: %u)\n",
+       GCC_2s (cc),
+       GCPP_2s (path),
+       off);
+  GCPP_add_connection (path, off, cc);
+  for (unsigned int i = 0; i < off; i++)
+    GCP_add_connection (GCPP_get_peer_at_offset (path, i), cc);
+  first_hop = GCPP_get_peer_at_offset (path, 0);
+  cc->mq_man = GCP_request_mq (first_hop, &manage_first_hop_mq, cc);
+  return cc;
+}
+
+
+/**
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
+ */
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  unsigned int off;
+
+  off = GCPP_find_peer (path, destination);
+  GNUNET_assert (UINT_MAX != off);
+  cc = GCPP_get_connection (path, destination, off);
+  if (NULL != cc)
+  {
+    int cmp;
+
+    cmp = GNUNET_memcmp (cid, &cc->cid);
+    if (0 == cmp)
+    {
+      /* Two peers picked the SAME random connection identifier at the
+         same time for the same path? Must be malicious.  Drop
+         connection (existing and inbound), even if it is the only
+         one. */
+      GNUNET_break_op (0);
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
+      return NULL;
+    }
+    if (0 < cmp)
+    {
+      /* drop existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, dropping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
+    }
+    else
+    {
+      /* keep existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, keeping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      return NULL;
+    }
+  }
+
+  return connection_create (destination,
+                            path,
+                            off,
+                            ct,
+                            cid,
+                            CADET_CONNECTION_CREATE_RECEIVED,
+                            ready_cb,
+                            ready_cb_cls);
+}
+
+
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls)
+{
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE, &cid, sizeof(cid));
+  return connection_create (destination,
+                            path,
+                            off,
+                            ct,
+                            &cid,
+                            CADET_CONNECTION_NEW,
+                            ready_cb,
+                            ready_cb_cls);
+}
+
+
+/**
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
+ *
+ * @param cc connection identification
+ * @param env envelope with message to transmit; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
+ */
+void
+GCC_transmit (struct CadetConnection *cc, struct GNUNET_MQ_Envelope *env)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Scheduling message for transmission on %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (CADET_CONNECTION_READY == cc->state);
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  cc->mqm_ready = GNUNET_NO;
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
+  GCP_send (cc->mq_man, env);
+}
+
+
+/**
+ * Obtain the path used by this connection.
+ *
+ * @param cc connection
+ * @param[out] off set to the length of the path we use
+ * @return path to @a cc
+ */
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc, unsigned int *off)
+{
+  *off = cc->off;
+  return cc->path;
+}
+
+
+/**
+ * Obtain unique ID for the connection.
+ *
+ * @param cc connection.
+ * @return unique number of the connection
+ */
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc)
+{
+  return &cc->cid;
+}
+
+
+/**
+ * Get a (static) string for a connection.
+ *
+ * @param cc Connection.
+ */
+const char *
+GCC_2s (const struct CadetConnection *cc)
+{
+  static char buf[128];
+
+  if (NULL == cc)
+    return "Connection(NULL)";
+
+  if (NULL != cc->ct)
+  {
+    GNUNET_snprintf (buf,
+                     sizeof(buf),
+                     "Connection %s (%s)",
+                     GNUNET_sh2s (&cc->cid.connection_of_tunnel),
+                     GCT_2s (cc->ct->t));
+    return buf;
+  }
+  GNUNET_snprintf (buf,
+                   sizeof(buf),
+                   "Connection %s",
+                   GNUNET_sh2s (&cc->cid.connection_of_tunnel));
+  return buf;
+}
+
+
+#define LOG2(level, ...) \
+  GNUNET_log_from_nocheck (level, "cadet-con", __VA_ARGS__)
+
+
+/**
+ * Log connection info.
+ *
+ * @param cc connection
+ * @param level Debug level to use.
+ */
+void
+GCC_debug (struct CadetConnection *cc, enum GNUNET_ErrorType level)
+{
+#if ! defined(GNUNET_CULL_LOGGING)
+  int do_log;
+
+  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
+                                       "cadet-con",
+                                       __FILE__,
+                                       __FUNCTION__,
+                                       __LINE__);
+  if (0 == do_log)
+    return;
+  if (NULL == cc)
+  {
+    LOG2 (level, "Connection (NULL)\n");
+    return;
+  }
+  LOG2 (level,
+        "%s to %s via path %s in state %d is %s\n",
+        GCC_2s (cc),
+        GCP_2s (cc->destination),
+        GCPP_2s (cc->path),
+        cc->state,
+        (GNUNET_YES == cc->mqm_ready) ? "ready" : "busy");
+#endif
+}
+
+
+/* end of gnunet-service-cadet_connection.c */
diff --git a/src/service/cadet/gnunet-service-cadet_connection.h b/src/service/cadet/gnunet-service-cadet_connection.h
new file mode 100644
index 000000000..54c845b13
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_connection.h
@@ -0,0 +1,363 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_connection.h
+ * @brief A connection is a live end-to-end messaging mechanism
+ *       where the peers are identified by a path and know how
+ *       to forward along the route using a connection identifier
+ *       for routing the data.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_CONNECTION_H
+#define GNUNET_SERVICE_CADET_CONNECTION_H
+
+#include "gnunet_util_lib.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_peer.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * Function called to notify tunnel about change in our readiness.
+ *
+ * @param cls closure
+ * @param is_ready #GNUNET_YES if the connection is now ready for transmission,
+ *                 #GNUNET_NO if the connection is no longer ready for transmission
+ */
+typedef void
+(*GCC_ReadyCallback)(void *cls,
+                     int is_ready);
+
+
+/**
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
+ *
+ * @param cc connection to destroy
+ */
+void
+GCC_destroy_without_core (struct CadetConnection *cc);
+
+
+/**
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
+ *
+ * @param cc connection to destroy
+ */
+void
+GCC_destroy_without_tunnel (struct CadetConnection *cc);
+
+
+/**
+ * Lookup a connection by its identifier.
+ *
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
+ */
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
+ */
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls);
+
+
+/**
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
+ */
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls);
+
+
+/**
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
+ *
+ * @param cc connection identification
+ * @param env envelope with message to transmit;
+ *            the #GNUNET_MQ_notify_send() must not have yet been used
+ *            for the envelope.  Also, the message better match the
+ *            connection identifier of this connection...
+ */
+void
+GCC_transmit (struct CadetConnection *cc,
+              struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * A #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK was received for
+ * this connection, implying that the end-to-end connection is up.
+ * Process it.
+ *
+ * @param cc the connection that got the ACK.
+ */
+void
+GCC_handle_connection_create_ack (struct CadetConnection *cc);
+
+
+/**
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
+ *
+ * @param cc connection that got the duplicate CREATE
+ */
+void
+GCC_handle_duplicate_create (struct CadetConnection *cc);
+
+
+/**
+ * Handle KX message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct
+                    GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
+
+/**
+ * Purpose for the signature of a monotime.
+ */
+struct CadetConnectionCreatePS
+{
+
+  /**
+   * Purpose is #GNUNET_SIGNATURE_PURPOSE_CADET_CONNECTION_INITIATOR
+   */
+  struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
+
+  /**
+   * Time at the initiator when generating the signature.
+   *
+   * Note that the receiver MUST IGNORE the absolute time, and only interpret
+   * the value as a mononic time and reject "older" values than the last one
+   * observed.  This is necessary as we do not want to require synchronized
+   * clocks and may not have a bidirectional communication channel.
+   *
+   * Even with this, there is no real guarantee against replay achieved here,
+   * unless the latest timestamp is persisted.  Persistence should be
+   * provided via PEERSTORE if possible.
+   */
+  struct GNUNET_TIME_AbsoluteNBO monotonic_time;
+
+};
+
+/**
+ * Performance metrics for a connection.
+ */
+struct CadetConnectionMetrics
+{
+  /**
+   * Our current best estimate of the latency, based on a weighted
+   * average of at least @a latency_datapoints values.
+   */
+  struct GNUNET_TIME_Relative aged_latency;
+
+  /**
+   * When was this connection first established? (by us sending or
+   * receiving the CREATE_ACK for the first time)
+   */
+  struct GNUNET_TIME_Absolute age;
+
+  /**
+   * When was this connection last used? (by us sending or
+   * receiving a PAYLOAD message on it)
+   */
+  struct GNUNET_TIME_Absolute last_use;
+
+  /**
+   * How many packets that ought to generate an ACK did we send via
+   * this connection?
+   */
+  unsigned long long num_acked_transmissions;
+
+  /**
+   * Number of packets that were sent via this connection did actually
+   * receive an ACK?  (Note: ACKs may be transmitted and lost via
+   * other connections, so this value should only be interpreted
+   * relative to @e num_acked_transmissions and in relation to other
+   * connections.)
+   */
+  unsigned long long num_successes;
+};
+
+
+/**
+ * Obtain performance @a metrics from @a cc.
+ *
+ * @param cc connection to query
+ * @return the metrics
+ */
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc);
+
+
+/**
+ * Handle encrypted message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
+ */
+void
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
+
+
+/**
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
+ *
+ * @param cid connection identifier where we expect an ACK
+ */
+void
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cti.
+ *
+ * @param cid connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
+ */
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
+ * We observed some the given @a latency on the connection
+ * identified by @a cti.  (The same connection was taken
+ * in both directions.)
+ *
+ * @param cti connection identifier where we measured latency
+ * @param latency the observed latency
+ */
+void
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                      struct GNUNET_TIME_Relative latency);
+
+
+/**
+ * Return the tunnel associated with this connection.
+ *
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
+ */
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc);
+
+
+/**
+ * Obtain the path used by this connection.
+ *
+ * @param cc connection
+ * @param[out] off set to offset in this path where the connection @a cc ends
+ * @return path to @a cc
+ */
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc,
+              unsigned int *off);
+
+
+/**
+ * Obtain unique ID for the connection.
+ *
+ * @param cc connection.
+ * @return unique number of the connection
+ */
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc);
+
+
+/**
+ * Get a (static) string for a connection.
+ *
+ * @param cc Connection.
+ */
+const char *
+GCC_2s (const struct CadetConnection *cc);
+
+
+/**
+ * Log connection info.
+ *
+ * @param cc connection
+ * @param level Debug level to use.
+ */
+void
+GCC_debug (struct CadetConnection *cc,
+           enum GNUNET_ErrorType level);
+
+
+#endif
diff --git a/src/service/cadet/gnunet-service-cadet_core.c b/src/service/cadet/gnunet-service-cadet_core.c
new file mode 100644
index 000000000..68f7405c3
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_core.c
@@ -0,0 +1,1337 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_core.c
+ * @brief cadet service; interaction with CORE service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * All functions in this file should use the prefix GCO (Gnunet Cadet cOre (bottom))
+ *
+ * TODO:
+ * - Optimization: given BROKEN messages, destroy paths (?)
+ */
+#include "platform.h"
+#include "gnunet-service-cadet_core.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet_core_service.h"
+#include "gnunet_statistics_service.h"
+#include "cadet_protocol.h"
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-cor", __VA_ARGS__)
+
+/**
+ * Information we keep per direction for a route.
+ */
+struct RouteDirection;
+
+/**
+ * Set of CadetRoutes that have exactly the same number of messages
+ * in their buffer.  Used so we can efficiently find all of those
+ * routes that have the current maximum of messages in the buffer (in
+ * case we have to purge).
+ */
+struct Rung
+{
+  /**
+   * Rung of RouteDirections with one more buffer entry each.
+   */
+  struct Rung *next;
+
+  /**
+   * Rung of RouteDirections with one less buffer entry each.
+   */
+  struct Rung *prev;
+
+  /**
+   * DLL of route directions with a number of buffer entries matching this rung.
+   */
+  struct RouteDirection *rd_head;
+
+  /**
+   * DLL of route directions with a number of buffer entries matching this rung.
+   */
+  struct RouteDirection *rd_tail;
+
+  /**
+   * Total number of route directions in this rung.
+   */
+  unsigned int num_routes;
+
+  /**
+   * Number of messages route directions at this rung have
+   * in their buffer.
+   */
+  unsigned int rung_off;
+};
+
+
+/**
+ * Information we keep per direction for a route.
+ */
+struct RouteDirection
+{
+  /**
+   * DLL of other route directions within the same `struct Rung`.
+   */
+  struct RouteDirection *prev;
+
+  /**
+   * DLL of other route directions within the same `struct Rung`.
+   */
+  struct RouteDirection *next;
+
+  /**
+   * Rung of this route direction (matches length of the buffer DLL).
+   */
+  struct Rung *rung;
+
+  /**
+   * Head of DLL of envelopes we have in the buffer for this direction.
+   */
+  struct GNUNET_MQ_Envelope *env_head;
+
+  /**
+   * Tail of DLL of envelopes we have in the buffer for this direction.
+   */
+  struct GNUNET_MQ_Envelope *env_tail;
+
+  /**
+   * Target peer.
+   */
+  struct CadetPeer *hop;
+
+  /**
+   * Route this direction is part of.
+   */
+  struct CadetRoute *my_route;
+
+  /**
+   * Message queue manager for @e hop.
+   */
+  struct GCP_MessageQueueManager *mqm;
+
+  /**
+   * Is @e mqm currently ready for transmission?
+   */
+  int is_ready;
+};
+
+
+struct CadetRoute
+{
+  /**
+   * Information about the next hop on this route.
+   */
+  struct RouteDirection next;
+
+  /**
+   * Information about the previous hop on this route.
+   */
+  struct RouteDirection prev;
+
+  /**
+   * Unique identifier for the connection that uses this route.
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
+
+  /**
+   * When was this route last in use?
+   */
+  struct GNUNET_TIME_Absolute last_use;
+
+  /**
+   * Position of this route in the #route_heap.
+   */
+  struct GNUNET_CONTAINER_HeapNode *hn;
+};
+
+
+/**
+ * Handle to the CORE service.
+ */
+static struct GNUNET_CORE_Handle *core;
+
+/**
+ * Routes on which this peer is an intermediate.
+ */
+static struct GNUNET_CONTAINER_MultiShortmap *routes;
+
+/**
+ * Heap of routes, MIN-sorted by last activity.
+ */
+static struct GNUNET_CONTAINER_Heap *route_heap;
+
+/**
+ * Rung zero (always pointed to by #rung_head).
+ */
+static struct Rung rung_zero;
+
+/**
+ * DLL of rungs, with the head always point to a rung of
+ * route directions with no messages in the queue.
+ */
+static struct Rung *rung_head = &rung_zero;
+
+/**
+ * Tail of the #rung_head DLL.
+ */
+static struct Rung *rung_tail = &rung_zero;
+
+/**
+ * Maximum number of concurrent routes this peer will support.
+ */
+static unsigned long long max_routes;
+
+/**
+ * Maximum number of envelopes we will buffer at this peer.
+ */
+static unsigned long long max_buffers;
+
+/**
+ * Current number of envelopes we have buffered at this peer.
+ */
+static unsigned long long cur_buffers;
+
+/**
+ * Task to timeout routes.
+ */
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+/**
+ * Get the route corresponding to a hash.
+ *
+ * @param cid hash generated from the connection identifier
+ */
+static struct CadetRoute *
+get_route (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+{
+  return GNUNET_CONTAINER_multishortmap_get (routes,
+                                             &cid->connection_of_tunnel);
+}
+
+
+/**
+ * Lower the rung in which @a dir is by 1.
+ *
+ * @param dir direction to lower in rung.
+ */
+static void
+lower_rung (struct RouteDirection *dir)
+{
+  struct Rung *rung = dir->rung;
+  struct Rung *prev;
+
+  GNUNET_CONTAINER_DLL_remove (rung->rd_head, rung->rd_tail, dir);
+  prev = rung->prev;
+  GNUNET_assert (NULL != prev);
+  if (prev->rung_off != rung->rung_off - 1)
+  {
+    prev = GNUNET_new (struct Rung);
+    prev->rung_off = rung->rung_off - 1;
+    GNUNET_CONTAINER_DLL_insert_after (rung_head, rung_tail, rung->prev, prev);
+  }
+  GNUNET_assert (NULL != prev);
+  GNUNET_CONTAINER_DLL_insert (prev->rd_head, prev->rd_tail, dir);
+  dir->rung = prev;
+}
+
+
+/**
+ * Discard the buffer @a env from the route direction @a dir and
+ * move @a dir down a rung.
+ *
+ * @param dir direction that contains the @a env in the buffer
+ * @param env envelope to discard
+ */
+static void
+discard_buffer (struct RouteDirection *dir, struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_MQ_dll_remove (&dir->env_head, &dir->env_tail, env);
+  cur_buffers--;
+  GNUNET_MQ_discard (env);
+  lower_rung (dir);
+  GNUNET_STATISTICS_set (stats, "# buffer use", cur_buffers, GNUNET_NO);
+}
+
+
+/**
+ * Discard all messages from the highest rung, to make space.
+ */
+static void
+discard_all_from_rung_tail ()
+{
+  struct Rung *tail = rung_tail;
+  struct RouteDirection *dir;
+
+  while (NULL != (dir = tail->rd_head))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Queue full due new message on connection %s, dropping old message\n",
+         GNUNET_sh2s (&dir->my_route->cid.connection_of_tunnel));
+    GNUNET_STATISTICS_update (stats,
+                              "# messages dropped due to full buffer",
+                              1,
+                              GNUNET_NO);
+    discard_buffer (dir, dir->env_head);
+  }
+  GNUNET_CONTAINER_DLL_remove (rung_head, rung_tail, tail);
+  GNUNET_free (tail);
+}
+
+
+/**
+ * We message @a msg from @a prev.  Find its route by @a cid and
+ * forward to the next hop.  Drop and signal broken route if we do not
+ * have a route.
+ *
+ * @param prev previous hop (sender)
+ * @param cid connection identifier, tells us which route to use
+ * @param msg the message to forward
+ */
+static void
+route_message (struct CadetPeer *prev,
+               const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+               const struct GNUNET_MessageHeader *msg,
+               const enum GNUNET_MQ_PriorityPreferences priority)
+{
+  struct CadetRoute *route;
+  struct RouteDirection *dir;
+  struct Rung *rung;
+  struct Rung *nxt;
+  struct GNUNET_MQ_Envelope *env;
+
+  route = get_route (cid);
+  if (NULL == route)
+  {
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_ConnectionBrokenMessage *bm;
+
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Failed to route message of type %u from %s on connection %s: no route\n",
+         ntohs (msg->type),
+         GCP_2s (prev),
+         GNUNET_sh2s (&cid->connection_of_tunnel));
+    switch (ntohs (msg->type))
+    {
+    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
+    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
+      /* No need to respond to these! */
+      return;
+    }
+    env = GNUNET_MQ_msg (bm, GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
+    bm->cid = *cid;
+    bm->peer1 = my_full_id;
+    GCP_send_ooo (prev, env);
+    return;
+  }
+  route->last_use = GNUNET_TIME_absolute_get ();
+  GNUNET_CONTAINER_heap_update_cost (route->hn, route->last_use.abs_value_us);
+  dir = (prev == route->prev.hop) ? &route->next : &route->prev;
+  if (GNUNET_YES == dir->is_ready)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Routing message of type %u from %s to %s on connection %s\n",
+         ntohs (msg->type),
+         GCP_2s (prev),
+         GNUNET_i2s (GCP_get_id (dir->hop)),
+         GNUNET_sh2s (&cid->connection_of_tunnel));
+    dir->is_ready = GNUNET_NO;
+    GCP_send (dir->mqm, GNUNET_MQ_msg_copy (msg));
+    return;
+  }
+  /* Check if low latency is required and if the previous message was
+     unreliable; if so, make sure we only queue one message per
+     direction (no buffering). */
+  if ((0 != (priority & GNUNET_MQ_PREF_LOW_LATENCY)) &&
+      (NULL != dir->env_head) &&
+      (0 ==
+       (GNUNET_MQ_env_get_options (dir->env_head) & GNUNET_MQ_PREF_UNRELIABLE)))
+    discard_buffer (dir, dir->env_head);
+  /* Check for duplicates */
+  for (const struct GNUNET_MQ_Envelope *env = dir->env_head; NULL != env;
+       env = GNUNET_MQ_env_next (env))
+  {
+    const struct GNUNET_MessageHeader *hdr = GNUNET_MQ_env_get_msg (env);
+
+    if ((hdr->size == msg->size) && (0 == memcmp (hdr, msg, ntohs (msg->size))))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Received duplicate of message already in buffer, dropping\n");
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to duplicate in buffer",
+                                1,
+                                GNUNET_NO);
+      return;
+    }
+  }
+
+  rung = dir->rung;
+  if (cur_buffers == max_buffers)
+  {
+    /* Need to make room. */
+    if (NULL != rung->next)
+    {
+      /* Easy case, drop messages from route directions in highest rung */
+      discard_all_from_rung_tail ();
+    }
+    else
+    {
+      /* We are in the highest rung, drop our own! */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Queue full due new message on connection %s, dropping old message\n",
+           GNUNET_sh2s (&dir->my_route->cid.connection_of_tunnel));
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to full buffer",
+                                1,
+                                GNUNET_NO);
+      discard_buffer (dir, dir->env_head);
+      rung = dir->rung;
+    }
+  }
+  /* remove 'dir' from current rung */
+  GNUNET_CONTAINER_DLL_remove (rung->rd_head, rung->rd_tail, dir);
+  /* make 'nxt' point to the next higher rung, create if necessary */
+  nxt = rung->next;
+  if ((NULL == nxt) || (rung->rung_off + 1 != nxt->rung_off))
+  {
+    nxt = GNUNET_new (struct Rung);
+    nxt->rung_off = rung->rung_off + 1;
+    GNUNET_CONTAINER_DLL_insert_after (rung_head, rung_tail, rung, nxt);
+  }
+  /* insert 'dir' into next higher rung */
+  GNUNET_CONTAINER_DLL_insert (nxt->rd_head, nxt->rd_tail, dir);
+  dir->rung = nxt;
+
+  /* add message into 'dir' buffer */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queueing new message of type %u from %s to %s on connection %s\n",
+       ntohs (msg->type),
+       GCP_2s (prev),
+       GNUNET_i2s (GCP_get_id (dir->hop)),
+       GNUNET_sh2s (&cid->connection_of_tunnel));
+  env = GNUNET_MQ_msg_copy (msg);
+  GNUNET_MQ_env_set_options (env, priority);
+  if ((0 != (priority & GNUNET_MQ_PREF_LOW_LATENCY)) &&
+      (0 != (priority & GNUNET_MQ_PREF_OUT_OF_ORDER)) &&
+      (NULL != dir->env_head) &&
+      (0 == (GNUNET_MQ_env_get_options (dir->env_head)
+             & GNUNET_MQ_PREF_LOW_LATENCY)))
+    GNUNET_MQ_dll_insert_head (&dir->env_head, &dir->env_tail, env);
+  else
+    GNUNET_MQ_dll_insert_tail (&dir->env_head, &dir->env_tail, env);
+  cur_buffers++;
+  GNUNET_STATISTICS_set (stats, "# buffer use", cur_buffers, GNUNET_NO);
+  /* Clean up 'rung' if now empty (and not head) */
+  if ((NULL == rung->rd_head) && (rung != rung_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (rung_head, rung_tail, rung);
+    GNUNET_free (rung);
+  }
+}
+
+
+/**
+ * Check if the create_connection message has the appropriate size.
+ *
+ * @param cls Closure (unused).
+ * @param msg Message to check.
+ *
+ * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
+ */
+static int
+check_connection_create (void *cls,
+                         const struct GNUNET_CADET_ConnectionCreateMessage *msg)
+{
+  uint16_t size = ntohs (msg->header.size) - sizeof(*msg);
+
+  if (0 != (size % sizeof(struct GNUNET_PeerIdentity)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_NO;
+  }
+  return GNUNET_YES;
+}
+
+
+/**
+ * Free internal data of a route direction.
+ *
+ * @param dir direction to destroy (do NOT free memory of 'dir' itself)
+ */
+static void
+destroy_direction (struct RouteDirection *dir)
+{
+  struct GNUNET_MQ_Envelope *env;
+
+  while (NULL != (env = dir->env_head))
+  {
+    GNUNET_STATISTICS_update (stats,
+                              "# messages dropped due to route destruction",
+                              1,
+                              GNUNET_NO);
+    discard_buffer (dir, env);
+  }
+  if (NULL != dir->mqm)
+  {
+    GCP_request_mq_cancel (dir->mqm, NULL);
+    dir->mqm = NULL;
+  }
+  GNUNET_CONTAINER_DLL_remove (rung_head->rd_head, rung_head->rd_tail, dir);
+}
+
+
+/**
+ * Destroy our state for @a route.
+ *
+ * @param route route to destroy
+ */
+static void
+destroy_route (struct CadetRoute *route)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying route from %s to %s of connection %s\n",
+       GNUNET_i2s (GCP_get_id (route->prev.hop)),
+       GNUNET_i2s2 (GCP_get_id (route->next.hop)),
+       GNUNET_sh2s (&route->cid.connection_of_tunnel));
+  GNUNET_assert (route == GNUNET_CONTAINER_heap_remove_node (route->hn));
+  GNUNET_assert (
+    GNUNET_YES ==
+    GNUNET_CONTAINER_multishortmap_remove (routes,
+                                           &route->cid.connection_of_tunnel,
+                                           route));
+  GNUNET_STATISTICS_set (stats,
+                         "# routes",
+                         GNUNET_CONTAINER_multishortmap_size (routes),
+                         GNUNET_NO);
+  destroy_direction (&route->prev);
+  destroy_direction (&route->next);
+  GNUNET_free (route);
+}
+
+
+/**
+ * Send message that a route is broken between @a peer1 and @a peer2.
+ *
+ * @param target where to send the message
+ * @param cid connection identifier to use
+ * @param peer1 one of the peers where a link is broken
+ * @param peer2 another one of the peers where a link is broken
+ */
+static void
+send_broken (struct RouteDirection *target,
+             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+             const struct GNUNET_PeerIdentity *peer1,
+             const struct GNUNET_PeerIdentity *peer2)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_ConnectionBrokenMessage *bm;
+
+  if (NULL == target->mqm)
+    return; /* Can't send notification, connection is down! */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Notifying %s about BROKEN route at %s-%s of connection %s\n",
+       GCP_2s (target->hop),
+       GNUNET_i2s (peer1),
+       GNUNET_i2s2 (peer2),
+       GNUNET_sh2s (&cid->connection_of_tunnel));
+
+  env = GNUNET_MQ_msg (bm, GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
+  bm->cid = *cid;
+  if (NULL != peer1)
+    bm->peer1 = *peer1;
+  if (NULL != peer2)
+    bm->peer2 = *peer2;
+  GCP_request_mq_cancel (target->mqm, env);
+  target->mqm = NULL;
+}
+
+
+/**
+ * Function called to check if any routes have timed out, and if
+ * so, to clean them up.  Finally, schedules itself again at the
+ * earliest time where there might be more work.
+ *
+ * @param cls NULL
+ */
+static void
+timeout_cb (void *cls)
+{
+  struct CadetRoute *r;
+  struct GNUNET_TIME_Relative linger;
+  struct GNUNET_TIME_Absolute exp;
+
+  timeout_task = NULL;
+  linger = GNUNET_TIME_relative_multiply (keepalive_period, 3);
+  while (NULL != (r = GNUNET_CONTAINER_heap_peek (route_heap)))
+  {
+    exp = GNUNET_TIME_absolute_add (r->last_use, linger);
+    if (0 != GNUNET_TIME_absolute_get_remaining (exp).rel_value_us)
+    {
+      /* Route not yet timed out, wait until it does. */
+      timeout_task = GNUNET_SCHEDULER_add_at (exp, &timeout_cb, NULL);
+      return;
+    }
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Sending BROKEN due to timeout (%s was last use, %s linger)\n",
+                GNUNET_STRINGS_absolute_time_to_string (r->last_use),
+                GNUNET_STRINGS_relative_time_to_string (linger, GNUNET_YES));
+    send_broken (&r->prev, &r->cid, NULL, NULL);
+    send_broken (&r->next, &r->cid, NULL, NULL);
+    destroy_route (r);
+  }
+  /* No more routes left, so no need for a #timeout_task */
+}
+
+
+/**
+ * Function called when the message queue to the previous hop
+ * becomes available/unavailable.  We expect this function to
+ * be called immediately when we register, and then again
+ * later if the connection ever goes down.
+ *
+ * @param cls the `struct RouteDirection`
+ * @param ready #GNUNET_YES if sending is now possible,
+ *              #GNUNET_NO if sending is no longer possible
+ *              #GNUNET_SYSERR if sending is no longer possible
+ *                             and the last envelope was discarded
+ */
+static void
+dir_ready_cb (void *cls, int ready)
+{
+  struct RouteDirection *dir = cls;
+  struct CadetRoute *route = dir->my_route;
+  struct RouteDirection *odir;
+
+  if (GNUNET_YES == ready)
+  {
+    struct GNUNET_MQ_Envelope *env;
+
+    dir->is_ready = GNUNET_YES;
+    if (NULL != (env = dir->env_head))
+    {
+      GNUNET_MQ_dll_remove (&dir->env_head, &dir->env_tail, env);
+      cur_buffers--;
+      GNUNET_STATISTICS_set (stats, "# buffer use", cur_buffers, GNUNET_NO);
+      lower_rung (dir);
+      dir->is_ready = GNUNET_NO;
+      GCP_send (dir->mqm, env);
+    }
+    return;
+  }
+  odir = (dir == &route->next) ? &route->prev : &route->next;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending BROKEN due to MQ going down\n");
+  send_broken (&route->next, &route->cid, GCP_get_id (odir->hop), &my_full_id);
+  destroy_route (route);
+}
+
+
+/**
+ * Initialize one of the directions of a route.
+ *
+ * @param route route the direction belongs to
+ * @param dir direction to initialize
+ * @param hop next hop on in the @a dir
+ */
+static void
+dir_init (struct RouteDirection *dir,
+          struct CadetRoute *route,
+          struct CadetPeer *hop)
+{
+  dir->hop = hop;
+  dir->my_route = route;
+  dir->mqm = GCP_request_mq (hop, &dir_ready_cb, dir);
+  GNUNET_CONTAINER_DLL_insert (rung_head->rd_head, rung_head->rd_tail, dir);
+  dir->rung = rung_head;
+  GNUNET_assert (GNUNET_YES == dir->is_ready);
+}
+
+
+/**
+ * We could not create the desired route.  Send a
+ * #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
+ * message to @a target.
+ *
+ * @param target who should receive the message
+ * @param cid identifier of the connection/route that failed
+ * @param failure_at neighbour with which we failed to route,
+ *        or NULL.
+ */
+static void
+send_broken_without_mqm (
+  struct CadetPeer *target,
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+  const struct GNUNET_PeerIdentity *failure_at)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_ConnectionBrokenMessage *bm;
+
+  env = GNUNET_MQ_msg (bm, GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
+  bm->cid = *cid;
+  bm->peer1 = my_full_id;
+  if (NULL != failure_at)
+    bm->peer2 = *failure_at;
+  GCP_send_ooo (target, env);
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_connection_create (
+  void *cls,
+  const struct GNUNET_CADET_ConnectionCreateMessage *msg)
+{
+  struct CadetPeer *sender = cls;
+  struct CadetPeer *next;
+  const struct GNUNET_PeerIdentity *pids =
+    (const struct GNUNET_PeerIdentity *) &msg[1];
+  struct CadetRoute *route;
+  uint16_t size = ntohs (msg->header.size) - sizeof(*msg);
+  unsigned int path_length;
+  unsigned int off;
+  struct CadetTunnel *t;
+
+  path_length = size / sizeof(struct GNUNET_PeerIdentity);
+  if (0 == path_length)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping CADET_CONNECTION_CREATE with empty path\n");
+    GNUNET_break_op (0);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Handling CADET_CONNECTION_CREATE from %s for CID %s with %u hops\n",
+       GCP_2s (sender),
+       GNUNET_sh2s (&msg->cid.connection_of_tunnel),
+       path_length);
+  /* Check for loops */
+  {
+    struct GNUNET_CONTAINER_MultiPeerMap *map;
+
+    map = GNUNET_CONTAINER_multipeermap_create (path_length * 2, GNUNET_YES);
+    GNUNET_assert (NULL != map);
+    for (unsigned int i = 0; i < path_length; i++)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "CADET_CONNECTION_CREATE has peer %s at offset %u\n",
+           GNUNET_i2s (&pids[i]),
+           i);
+      if (GNUNET_SYSERR == GNUNET_CONTAINER_multipeermap_put (
+            map,
+            &pids[i],
+            NULL,
+            GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+      {
+        /* bogus request */
+        GNUNET_CONTAINER_multipeermap_destroy (map);
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Dropping CADET_CONNECTION_CREATE with cyclic path\n");
+        GNUNET_break_op (0);
+        return;
+      }
+    }
+    GNUNET_CONTAINER_multipeermap_destroy (map);
+  }
+  /* Initiator is at offset 0, find us */
+  for (off = 1; off < path_length; off++)
+    if (0 == GNUNET_memcmp (&my_full_id, &pids[off]))
+      break;
+  if (off == path_length)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping CADET_CONNECTION_CREATE without us in the path\n");
+    GNUNET_break_op (0);
+    return;
+  }
+  /* Check previous hop */
+  if (sender != GCP_get (&pids[off - 1], GNUNET_NO))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping CADET_CONNECTION_CREATE without sender at previous hop in the path\n");
+    GNUNET_break_op (0);
+    return;
+  }
+  if (NULL != (route = get_route (&msg->cid)))
+  {
+    /* Duplicate CREATE, pass it on, previous one might have been lost! */
+
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Passing on duplicate CADET_CONNECTION_CREATE message on connection %s\n",
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    route_message (sender,
+                   &msg->cid,
+                   &msg->header,
+                   GNUNET_MQ_PRIO_CRITICAL_CONTROL
+                   | GNUNET_MQ_PREF_LOW_LATENCY);
+    return;
+  }
+  if (off == path_length - 1)
+  {
+    /* We are the destination, create connection */
+    struct CadetConnection *cc;
+    struct CadetPeerPath *path;
+    struct CadetPeer *origin;
+
+    cc = GCC_lookup (&msg->cid);
+    if (NULL != cc)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Received duplicate CADET_CONNECTION_CREATE message on connection %s\n",
+           GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+      GCC_handle_duplicate_create (cc);
+      return;
+    }
+
+    origin = GCP_get (&pids[0], GNUNET_YES);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "I am destination for CADET_CONNECTION_CREATE message from %s for connection %s, building inverse path\n",
+         GCP_2s (origin),
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    path = GCPP_get_path_from_route (path_length - 1, pids);
+    t = GCP_get_tunnel (origin, GNUNET_YES);
+
+    // Check for CADET state in case the other side has lost the tunnel (xrs,t3ss)
+    if ((GNUNET_YES == msg->has_monotime) &&
+        (GNUNET_YES == GCP_check_and_update_monotime (origin, msg->monotime)) &&
+        (GNUNET_OK == GCP_check_monotime_sig (origin, msg)) &&
+        (CADET_TUNNEL_KEY_OK == GCT_get_estate (t)))
+    {
+      GCT_change_estate (t, CADET_TUNNEL_KEY_UNINITIALIZED);
+    }
+
+    if (GNUNET_OK !=
+        GCT_add_inbound_connection (t,
+                                    &msg->cid,
+                                    path))
+    {
+      /* Send back BROKEN: duplicate connection on the same path,
+         we will use the other one. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Received CADET_CONNECTION_CREATE from %s for %s, but %s already has a connection. Sending BROKEN\n",
+           GCP_2s (sender),
+           GNUNET_sh2s (&msg->cid.connection_of_tunnel),
+           GCPP_2s (path));
+      send_broken_without_mqm (sender, &msg->cid, NULL);
+      return;
+    }
+    return;
+  }
+  /* We are merely a hop on the way, check if we can support the route */
+  next = GCP_get (&pids[off + 1], GNUNET_NO);
+  if ((NULL == next) || (GNUNET_NO == GCP_has_core_connection (next)))
+  {
+    /* unworkable, send back BROKEN notification */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CADET_CONNECTION_CREATE from %s for %s. Next hop %s:%u is down. Sending BROKEN\n",
+         GCP_2s (sender),
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel),
+         GNUNET_i2s (&pids[off + 1]),
+         off + 1);
+    send_broken_without_mqm (sender, &msg->cid, &pids[off + 1]);
+    return;
+  }
+  if (max_routes <= GNUNET_CONTAINER_multishortmap_size (routes))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CADET_CONNECTION_CREATE from %s for %s. We have reached our route limit. Sending BROKEN\n",
+         GCP_2s (sender),
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    send_broken_without_mqm (sender, &msg->cid, &pids[off - 1]);
+    return;
+  }
+
+  /* Workable route, create routing entry */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CADET_CONNECTION_CREATE from %s for %s. Next hop %s:%u is up. Creating route\n",
+       GCP_2s (sender),
+       GNUNET_sh2s (&msg->cid.connection_of_tunnel),
+       GNUNET_i2s (&pids[off + 1]),
+       off + 1);
+  route = GNUNET_new (struct CadetRoute);
+  route->cid = msg->cid;
+  route->last_use = GNUNET_TIME_absolute_get ();
+  dir_init (&route->prev, route, sender);
+  dir_init (&route->next, route, next);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multishortmap_put (
+                   routes,
+                   &route->cid.connection_of_tunnel,
+                   route,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  GNUNET_STATISTICS_set (stats,
+                         "# routes",
+                         GNUNET_CONTAINER_multishortmap_size (routes),
+                         GNUNET_NO);
+  route->hn = GNUNET_CONTAINER_heap_insert (route_heap,
+                                            route,
+                                            route->last_use.abs_value_us);
+  if (NULL == timeout_task)
+    timeout_task =
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (
+                                      keepalive_period,
+                                      3),
+                                    &timeout_cb,
+                                    NULL);
+  /* also pass CREATE message along to next hop */
+  route_message (sender,
+                 &msg->cid,
+                 &msg->header,
+                 GNUNET_MQ_PRIO_CRITICAL_CONTROL | GNUNET_MQ_PREF_LOW_LATENCY);
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_connection_create_ack (
+  void *cls,
+  const struct GNUNET_CADET_ConnectionCreateAckMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+
+  /* First, check if ACK belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->cid);
+  if (NULL != cc)
+  {
+    /* verify ACK came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received ACK from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CONNECTION_CREATE_ACK for connection %s.\n",
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    GCC_handle_connection_create_ack (cc);
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer,
+                 &msg->cid,
+                 &msg->header,
+                 GNUNET_MQ_PRIO_CRITICAL_CONTROL | GNUNET_MQ_PREF_LOW_LATENCY);
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ * @deprecated duplicate logic with #handle_destroy(); dedup!
+ */
+static void
+handle_connection_broken (
+  void *cls,
+  const struct GNUNET_CADET_ConnectionBrokenMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+  struct CadetRoute *route;
+
+  /* First, check if message belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CONNECTION_BROKEN for connection %s. Destroying it.\n",
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+    GCC_destroy_without_core (cc);
+
+    /* FIXME: also destroy the path up to the specified link! */
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer,
+                 &msg->cid,
+                 &msg->header,
+                 GNUNET_MQ_PREF_LOW_LATENCY | GNUNET_MQ_PRIO_CRITICAL_CONTROL);
+  route = get_route (&msg->cid);
+  if (NULL != route)
+    destroy_route (route);
+  /* FIXME: also destroy paths we MAY have up to the specified link! */
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_connection_destroy (
+  void *cls,
+  const struct GNUNET_CADET_ConnectionDestroyMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+  struct CadetRoute *route;
+
+  /* First, check if message belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CONNECTION_DESTROY for connection %s. Destroying connection.\n",
+         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+
+    GCC_destroy_without_core (cc);
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CONNECTION_DESTROY for connection %s. Destroying route.\n",
+       GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+  route_message (peer,
+                 &msg->cid,
+                 &msg->header,
+                 GNUNET_MQ_PREF_LOW_LATENCY | GNUNET_MQ_PRIO_CRITICAL_CONTROL);
+  route = get_route (&msg->cid);
+  if (NULL != route)
+    destroy_route (route);
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_tunnel_kx (void *cls,
+                  const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+
+  /* First, check if message belongs to a connection that ends here. */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Routing KX with ephemeral %s on CID %s\n",
+       GNUNET_e2s (&msg->ephemeral_key),
+       GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+
+
+  cc = GCC_lookup (&msg->cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    GCC_handle_kx (cc, msg);
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer,
+                 &msg->cid,
+                 &msg->header,
+                 GNUNET_MQ_PRIO_CRITICAL_CONTROL | GNUNET_MQ_PREF_LOW_LATENCY);
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_tunnel_kx_auth (
+  void *cls,
+  const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+
+  /* First, check if message belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->kx.cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    GCC_handle_kx_auth (cc, msg);
+    return;
+  }
+
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer,
+                 &msg->kx.cid,
+                 &msg->kx.header,
+                 GNUNET_MQ_PRIO_CRITICAL_CONTROL | GNUNET_MQ_PREF_LOW_LATENCY);
+}
+
+
+/**
+ * Check if the encrypted message has the appropriate size.
+ *
+ * @param cls Closure (unused).
+ * @param msg Message to check.
+ *
+ * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
+ */
+static int
+check_tunnel_encrypted (void *cls,
+                        const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+{
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED.
+ *
+ * @param cls Closure (CadetPeer for neighbor that sent the message).
+ * @param msg Message itself.
+ */
+static void
+handle_tunnel_encrypted (void *cls,
+                         const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+{
+  struct CadetPeer *peer = cls;
+  struct CadetConnection *cc;
+
+  /* First, check if message belongs to a connection that ends here. */
+  cc = GCC_lookup (&msg->cid);
+  if (NULL != cc)
+  {
+    /* verify message came from the right direction */
+    unsigned int len;
+    struct CadetPeerPath *path = GCC_get_path (cc, &len);
+
+    if (peer != GCPP_get_peer_at_offset (path, 0))
+    {
+      /* received message from unexpected direction, ignore! */
+      GNUNET_break_op (0);
+      return;
+    }
+    GCC_handle_encrypted (cc, msg);
+    return;
+  }
+  /* We're just an intermediary peer, route the message along its path */
+  route_message (peer, &msg->cid, &msg->header, GNUNET_MQ_PRIO_BEST_EFFORT);
+}
+
+
+/**
+ * Function called after #GNUNET_CORE_connect has succeeded (or failed
+ * for good).  Note that the private key of the peer is intentionally
+ * not exposed here; if you need it, your process should try to read
+ * the private key file directly (which should work if you are
+ * authorized...).  Implementations of this function must not call
+ * #GNUNET_CORE_disconnect (other than by scheduling a new task to
+ * do this later).
+ *
+ * @param cls closure
+ * @param my_identity ID of this peer, NULL if we failed
+ */
+static void
+core_init_cb (void *cls, const struct GNUNET_PeerIdentity *my_identity)
+{
+  if (NULL == my_identity)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_break (0 == GNUNET_memcmp (my_identity, &my_full_id));
+}
+
+
+/**
+ * Method called whenever a given peer connects.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ */
+static void *
+core_connect_cb (void *cls,
+                 const struct GNUNET_PeerIdentity *peer,
+                 struct GNUNET_MQ_Handle *mq)
+{
+  struct CadetPeer *cp;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "CORE connection to peer %s was established.\n",
+       GNUNET_i2s (peer));
+  cp = GCP_get (peer, GNUNET_YES);
+  GCP_set_mq (cp, mq);
+  return cp;
+}
+
+
+/**
+ * Method called whenever a peer disconnects.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ */
+static void
+core_disconnect_cb (void *cls,
+                    const struct GNUNET_PeerIdentity *peer,
+                    void *peer_cls)
+{
+  struct CadetPeer *cp = peer_cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "CORE connection to peer %s went down.\n",
+       GNUNET_i2s (peer));
+  GCP_set_mq (cp, NULL);
+}
+
+
+/**
+ * Initialize the CORE subsystem.
+ *
+ * @param c Configuration.
+ */
+void
+GCO_init (const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] =
+  { GNUNET_MQ_hd_var_size (connection_create,
+                           GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE,
+                           struct GNUNET_CADET_ConnectionCreateMessage,
+                           NULL),
+    GNUNET_MQ_hd_fixed_size (connection_create_ack,
+                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK,
+                             struct GNUNET_CADET_ConnectionCreateAckMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (connection_broken,
+                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN,
+                             struct GNUNET_CADET_ConnectionBrokenMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (connection_destroy,
+                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY,
+                             struct GNUNET_CADET_ConnectionDestroyMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (tunnel_kx,
+                             GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX,
+                             struct GNUNET_CADET_TunnelKeyExchangeMessage,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (tunnel_kx_auth,
+                             GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH,
+                             struct GNUNET_CADET_TunnelKeyExchangeAuthMessage,
+                             NULL),
+    GNUNET_MQ_hd_var_size (tunnel_encrypted,
+                           GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED,
+                           struct GNUNET_CADET_TunnelEncryptedMessage,
+                           NULL),
+    GNUNET_MQ_handler_end () };
+
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (c,
+                                                          "CADET",
+                                                          "MAX_ROUTES",
+                                                          &max_routes))
+    max_routes = 5000;
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (c,
+                                                          "CADET",
+                                                          "MAX_MSGS_QUEUE",
+                                                          &max_buffers))
+    max_buffers = 10000;
+  routes = GNUNET_CONTAINER_multishortmap_create (1024, GNUNET_NO);
+  route_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
+  core = GNUNET_CORE_connect (c,
+                              NULL,
+                              &core_init_cb,
+                              &core_connect_cb,
+                              &core_disconnect_cb,
+                              handlers);
+}
+
+
+void
+GCO_shutdown ()
+{
+  if (NULL != core)
+  {
+    GNUNET_CORE_disconnect (core);
+    core = NULL;
+  }
+  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (routes));
+  GNUNET_CONTAINER_multishortmap_destroy (routes);
+  routes = NULL;
+  GNUNET_CONTAINER_heap_destroy (route_heap);
+  route_heap = NULL;
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+/* end of gnunet-cadet-service_core.c */
diff --git a/src/service/cadet/gnunet-service-cadet_core.h b/src/service/cadet/gnunet-service-cadet_core.h
new file mode 100644
index 000000000..3438dcb31
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_core.h
@@ -0,0 +1,69 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_core.h
+ * @brief cadet service; interaction with CORE service
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * All functions in this file should use the prefix GCO (Gnunet Cadet cOre (bottom))
+ */
+
+#ifndef GNUNET_SERVICE_CADET_CORE_H
+#define GNUNET_SERVICE_CADET_CORE_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#include "gnunet_util_lib.h"
+
+
+/**
+ * Initialize the CORE subsystem.
+ *
+ * @param c Configuration.
+ */
+void
+GCO_init (const struct GNUNET_CONFIGURATION_Handle *c);
+
+
+/**
+ * Shut down the CORE subsystem.
+ */
+void
+GCO_shutdown (void);
+
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+/* ifndef GNUNET_CADET_SERVICE_CORE_H */
+#endif
+/* end of gnunet-cadet-service_core.h */
diff --git a/src/service/cadet/gnunet-service-cadet_dht.c b/src/service/cadet/gnunet-service-cadet_dht.c
new file mode 100644
index 000000000..dcd1efaa2
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_dht.c
@@ -0,0 +1,340 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2013, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/gnunet-service-cadet_dht.c
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_dht_service.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+
+/**
+ * How long do we wait before first announcing our presence to the DHT.
+ * Used to wait for our HELLO to be available.  Note that we also get
+ * notifications when our HELLO is ready, so this is just the maximum
+ * we wait for the first notification.
+ */
+#define STARTUP_DELAY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MILLISECONDS, 500)
+
+/**
+ * How long do we wait after we get an updated HELLO before publishing?
+ * Allows for the HELLO to be updated again quickly, for example in
+ * case multiple addresses changed and we got a partial update.
+ */
+#define CHANGE_DELAY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MILLISECONDS, 100)
+
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-dht", __VA_ARGS__)
+
+
+struct GCD_search_handle
+{
+  /**
+   * DHT_GET handle.
+   */
+  struct GNUNET_DHT_GetHandle *dhtget;
+};
+
+
+/**
+ * Handle to use DHT.
+ */
+static struct GNUNET_DHT_Handle *dht_handle;
+
+/**
+ * How often to PUT own ID in the DHT.
+ */
+static struct GNUNET_TIME_Relative id_announce_time;
+
+/**
+ * DHT replication level, see DHT API: #GNUNET_DHT_get_start(), #GNUNET_DHT_put().
+ */
+static unsigned long long dht_replication_level;
+
+/**
+ * Task to periodically announce itself in the network.
+ */
+static struct GNUNET_SCHEDULER_Task *announce_id_task;
+
+/**
+ * Delay for the next ID announce.
+ */
+static struct GNUNET_TIME_Relative announce_delay;
+
+
+/**
+ * Function to process paths received for a new peer addition. The recorded
+ * paths form the initial tunnel, which can be optimized later.
+ * Called on each result obtained for the DHT search.
+ *
+ * @param cls closure
+ * @param exp when will this value expire
+ * @param key key of the result
+ * @param trunc_peer peer preceeding with invalid signature, or NULL
+ * @param get_path path of the get request
+ * @param get_path_length length of @a get_path
+ * @param put_path path of the put request
+ * @param put_path_length length of the @a put_path
+ * @param type type of the result
+ * @param size number of bytes in data
+ * @param data pointer to the result data
+ */
+static void
+dht_get_id_handler (void *cls, struct GNUNET_TIME_Absolute exp,
+                    const struct GNUNET_HashCode *key,
+                    const struct GNUNET_PeerIdentity *trunc_peer,
+                    const struct GNUNET_DHT_PathElement *get_path,
+                    unsigned int get_path_length,
+                    const struct GNUNET_DHT_PathElement *put_path,
+                    unsigned int put_path_length,
+                    enum GNUNET_BLOCK_Type type,
+                    size_t size,
+                    const void *data)
+{
+  const struct GNUNET_MessageHeader *hello = data;
+  struct GNUNET_HELLO_Builder *builder = GNUNET_HELLO_builder_from_msg (hello);
+
+  GNUNET_assert (NULL != builder);
+  (void) trunc_peer;
+  GCPP_try_path_from_dht (get_path,
+                          get_path_length,
+                          put_path,
+                          put_path_length);
+
+  struct CadetPeer *peer;
+
+  peer = GCP_get (&put_path[0].pred,
+                    GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got HELLO for %s\n",
+       GCP_2s (peer));
+  GCP_set_hello (peer,
+                   hello);
+  GNUNET_HELLO_builder_free (builder);
+}
+
+
+/**
+ * Periodically announce self id in the DHT
+ *
+ * @param cls closure
+ */
+static void
+announce_id (void *cls)
+{
+  struct GNUNET_HashCode phash;
+  const struct GNUNET_MessageHeader *hello;
+  size_t size;
+  size_t block_size;
+  void *block;
+  struct GNUNET_TIME_Absolute expiration;
+  struct GNUNET_TIME_Relative next_put;
+
+  hello = GCH_get_mine ();
+  size = (NULL != hello) ? ntohs(hello->size) : 0;
+  if (0 == size)
+  {
+    expiration = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
+                                           announce_delay);
+    announce_delay = GNUNET_TIME_STD_BACKOFF (announce_delay);
+  }
+  else
+  {
+    expiration = GNUNET_HELLO_builder_get_expiration_time (hello);
+    announce_delay = GNUNET_TIME_UNIT_SECONDS;
+  }
+
+  /* Call again in id_announce_time, unless HELLO expires first,
+   * but wait at least 1s. */
+  next_put
+    = GNUNET_TIME_absolute_get_remaining (expiration);
+  next_put
+    = GNUNET_TIME_relative_min (next_put,
+                                id_announce_time);
+  next_put
+    = GNUNET_TIME_relative_max (next_put,
+                                GNUNET_TIME_UNIT_SECONDS);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (next_put,
+                                    &announce_id,
+                                    cls);
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT announce",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof(phash));
+  GNUNET_memcpy (&phash,
+                 &my_full_id,
+                 sizeof(my_full_id));
+  if (GNUNET_OK != GNUNET_HELLO_dht_msg_to_block (hello,
+                                                  &my_full_id,
+                                                  &block,
+                                                  &block_size,
+                                                  &expiration))
+    return;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Announcing my HELLO (%lu bytes) in the DHT\n",
+       (unsigned long) block_size);
+  GNUNET_DHT_put (dht_handle,    /* DHT handle */
+                  &phash,       /* Key to use */
+                  dht_replication_level,     /* Replication level */
+                  GNUNET_DHT_RO_RECORD_ROUTE
+                  | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
+                  GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
+                  block_size,  /* Size of the data */
+                  (const char *) block, /* Data itself */
+                  expiration,  /* Data expiration */
+                  NULL,         /* Continuation */
+                  NULL);        /* Continuation closure */
+  GNUNET_free (block);
+}
+
+
+void
+GCD_hello_update ()
+{
+  if (NULL == announce_id_task)
+    return; /* too early */
+  GNUNET_SCHEDULER_cancel (announce_id_task);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (CHANGE_DELAY,
+                                    &announce_id,
+                                    NULL);
+}
+
+
+void
+GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "DHT_REPLICATION_LEVEL",
+                                             &dht_replication_level))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "DHT_REPLICATION_LEVEL",
+                               "USING DEFAULT");
+    dht_replication_level = 3;
+  }
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "ID_ANNOUNCE_TIME",
+                                           &id_announce_time))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
+                               "CADET",
+                               "ID_ANNOUNCE_TIME",
+                               "MISSING");
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+
+  dht_handle = GNUNET_DHT_connect (c,
+                                   64);
+  GNUNET_break (NULL != dht_handle);
+  announce_delay = GNUNET_TIME_UNIT_SECONDS;
+  announce_id_task = GNUNET_SCHEDULER_add_delayed (STARTUP_DELAY,
+                                                   &announce_id,
+                                                   NULL);
+}
+
+
+void
+GCD_shutdown (void)
+{
+  if (NULL != dht_handle)
+  {
+    GNUNET_DHT_disconnect (dht_handle);
+    dht_handle = NULL;
+  }
+  if (NULL != announce_id_task)
+  {
+    GNUNET_SCHEDULER_cancel (announce_id_task);
+    announce_id_task = NULL;
+  }
+}
+
+
+struct GCD_search_handle *
+GCD_search (const struct GNUNET_PeerIdentity *peer_id)
+{
+  struct GNUNET_HashCode phash;
+  struct GCD_search_handle *h;
+
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT search",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof(phash));
+  GNUNET_memcpy (&phash,
+                 peer_id,
+                 sizeof(*peer_id));
+
+  h = GNUNET_new (struct GCD_search_handle);
+  h->dhtget = GNUNET_DHT_get_start (dht_handle,     /* handle */
+                                    GNUNET_BLOCK_TYPE_DHT_HELLO, /* type */
+                                    &phash,     /* key to search */
+                                    dht_replication_level, /* replication level */
+                                    GNUNET_DHT_RO_RECORD_ROUTE
+                                    | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                    NULL,       /* xquery */
+                                    0,     /* xquery bits */
+                                    &dht_get_id_handler,
+                                    h);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Starting DHT GET for peer %s (%p)\n",
+       GNUNET_i2s (peer_id),
+       h);
+  return h;
+}
+
+
+void
+GCD_search_stop (struct GCD_search_handle *h)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Stopping DHT GET %p\n",
+       h);
+  GNUNET_DHT_get_stop (h->dhtget);
+  GNUNET_free (h);
+}
+
+
+/* end of gnunet-service-cadet_dht.c */
diff --git a/src/service/cadet/gnunet-service-cadet_dht.h b/src/service/cadet/gnunet-service-cadet_dht.h
new file mode 100644
index 000000000..bcc27f5d2
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_dht.h
@@ -0,0 +1,100 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2013, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_dht.h
+ * @brief cadet service; dealing with DHT requests and results
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * All functions in this file should use the prefix GCD (Gnunet Cadet Dht)
+ */
+#ifndef GNUNET_SERVICE_CADET_DHT_H
+#define GNUNET_SERVICE_CADET_DHT_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+
+/**
+ * Handle for DHT search operation.
+ */
+struct GCD_search_handle;
+
+
+/**
+ * Initialize the DHT subsystem.
+ *
+ * @param c Configuration.
+ */
+void
+GCD_init (const struct GNUNET_CONFIGURATION_Handle *c);
+
+
+/**
+ * Shut down the DHT subsystem.
+ */
+void
+GCD_shutdown (void);
+
+
+/**
+ * Function called by the HELLO subsystem whenever OUR hello
+ * changes. Re-triggers the DHT PUT immediately.
+ */
+void
+GCD_hello_update (void);
+
+/**
+ * Search DHT for paths to @a peeR_id
+ *
+ * @param peer_id peer to search for
+ * @return handle to abort search
+ */
+struct GCD_search_handle *
+GCD_search (const struct GNUNET_PeerIdentity *peer_id);
+
+
+/**
+ * Stop DHT search started with #GCD_search().
+ *
+ * @param h handle to search to stop
+ */
+void
+GCD_search_stop (struct GCD_search_handle *h);
+
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+/* ifndef GNUNET_CADET_SERVICE_DHT_H */
+#endif
+/* end of gnunet-service-cadet_dht.h */
diff --git a/src/service/cadet/gnunet-service-cadet_hello.c b/src/service/cadet/gnunet-service-cadet_hello.c
new file mode 100644
index 000000000..cc28f0fde
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_hello.c
@@ -0,0 +1,195 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2014, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/gnunet-service-cadet_hello.c
+ * @brief spread knowledge about how to contact us (get HELLO from peerinfo),
+ *         and remember HELLOs of other peers we have an interest in
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "gnunet_common.h"
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_peerstore_service.h"
+#include "cadet_protocol.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_peer.h"
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-hll", __VA_ARGS__)
+
+/**
+ * Hello message of local peer.
+ */
+static struct GNUNET_MessageHeader *mine;
+
+/**
+ * Handle to the PEERSTORE service.
+ */
+static struct GNUNET_PEERSTORE_Handle *peerstore;
+
+/**
+ * Our peerstore notification context.  We use notification
+ * to instantly learn about new peers as they are discovered.
+ */
+static struct GNUNET_PEERSTORE_Monitor *peerstore_notify;
+
+
+/**
+ * Process each hello message received from peerinfo.
+ *
+ * @param cls Closure (unused).
+ * @param id Identity of the peer.
+ * @param hello Hello of the peer.
+ * @param err_msg Error message.
+ */
+static void
+got_hello (void *cls,
+           const struct GNUNET_PEERSTORE_Record *record,
+           const char *err_msg)
+{
+  struct CadetPeer *peer;
+  struct GNUNET_HELLO_Builder *builder;
+  struct GNUNET_MessageHeader *hello;
+
+  if (NULL == record->value)
+  {
+    GNUNET_PEERSTORE_monitor_next (peerstore_notify, 1);
+    return;
+  }
+  hello = record->value;
+  if (0 == GNUNET_memcmp (&record->peer,
+                          &my_full_id))
+  {
+    GNUNET_free (mine);
+    builder = GNUNET_HELLO_builder_from_msg (hello);
+    mine = GNUNET_HELLO_builder_to_dht_hello_msg (builder,
+                                                  my_private_key,
+                                                  GNUNET_TIME_UNIT_ZERO);
+    GNUNET_HELLO_builder_free (builder);
+    GCD_hello_update ();
+    GNUNET_PEERSTORE_monitor_next (peerstore_notify, 1);
+    return;
+  }
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Hello for %s (%d bytes), expires on %s\n",
+       GNUNET_i2s (&record->peer),
+       ntohs (hello->size),
+       GNUNET_STRINGS_absolute_time_to_string (
+         GNUNET_HELLO_builder_get_expiration_time (hello)));
+  peer = GCP_get (&record->peer,
+                  GNUNET_YES);
+  GCP_set_hello (peer,
+                 hello);
+  GNUNET_PEERSTORE_monitor_next (peerstore_notify, 1);
+}
+
+
+static void
+error_cb (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Error in PEERSTORE monitoring\n");
+}
+
+
+static void
+sync_cb (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Done with initial PEERSTORE iteration during monitoring\n");
+}
+
+
+/**
+ * Initialize the hello subsystem.
+ *
+ * @param c Configuration.
+ */
+void
+GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  GNUNET_assert (NULL == peerstore_notify);
+  peerstore = GNUNET_PEERSTORE_connect (c);
+  peerstore_notify =
+    GNUNET_PEERSTORE_monitor_start (c,
+                                    GNUNET_YES,
+                                    "peerstore",
+                                    NULL,
+                                    GNUNET_PEERSTORE_HELLO_KEY,
+                                    &error_cb,
+                                    NULL,
+                                    &sync_cb,
+                                    NULL,
+                                    &got_hello,
+                                    NULL);
+}
+
+
+/**
+ * Shut down the hello subsystem.
+ */
+void
+GCH_shutdown ()
+{
+  if (NULL != peerstore_notify)
+  {
+    GNUNET_PEERSTORE_monitor_stop (peerstore_notify);
+    peerstore_notify = NULL;
+  }
+  if (NULL != peerstore)
+  {
+    GNUNET_PEERSTORE_disconnect (peerstore);
+    peerstore = NULL;
+  }
+  if (NULL != mine)
+  {
+    GNUNET_free (mine);
+    mine = NULL;
+  }
+}
+
+
+/**
+ * Get own hello message.
+ *
+ * @return Own hello message.
+ */
+const struct GNUNET_MessageHeader *
+GCH_get_mine (void)
+{
+  struct GNUNET_HELLO_Builder *builder;
+
+  if (NULL == mine)
+  {
+    builder = GNUNET_HELLO_builder_new (&my_full_id);
+    mine = GNUNET_HELLO_builder_to_dht_hello_msg (builder,
+                                                  my_private_key,
+                                                  GNUNET_TIME_UNIT_ZERO);
+    GNUNET_HELLO_builder_free (builder);
+  }
+  return mine;
+}
+
+
+/* end of gnunet-service-cadet-new_hello.c */
diff --git a/src/service/cadet/gnunet-service-cadet_hello.h b/src/service/cadet/gnunet-service-cadet_hello.h
new file mode 100644
index 000000000..88c955bb9
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_hello.h
@@ -0,0 +1,80 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2014, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_hello.h
+ * @brief cadet service; dealing with hello messages
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * All functions in this file should use the prefix GCH (Gnunet Cadet Hello)
+ */
+
+#ifndef GNUNET_SERVICE_CADET_HELLO_H
+#define GNUNET_SERVICE_CADET_HELLO_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_hello_uri_lib.h"
+
+
+/**
+ * Initialize the hello subsystem.
+ *
+ * @param c Configuration.
+ */
+void
+GCH_init (const struct GNUNET_CONFIGURATION_Handle *c);
+
+
+/**
+ * Shut down the hello subsystem.
+ */
+void
+GCH_shutdown (void);
+
+
+/**
+ * Get own hello message.
+ *
+ * @return Own hello message.
+ */
+const struct GNUNET_MessageHeader *
+GCH_get_mine (void);
+
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+/* ifndef GNUNET_CADET_SERVICE_HELLO_H */
+#endif
+/* end of gnunet-cadet-service_hello.h */
diff --git a/src/service/cadet/gnunet-service-cadet_paths.c b/src/service/cadet/gnunet-service-cadet_paths.c
new file mode 100644
index 000000000..83b996e05
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_paths.c
@@ -0,0 +1,771 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/gnunet-service-cadet_paths.c
+ * @brief Information we track per path.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-pat", __VA_ARGS__)
+
+
+/**
+ * Information regarding a possible path to reach a peer.
+ */
+struct CadetPeerPath
+{
+  /**
+   * Array of all the peers on the path.  If @e hn is non-NULL, the
+   * last one is our owner.
+   */
+  struct CadetPeerPathEntry **entries;
+
+  /**
+   * Node of this path in the owner's heap.  Used to update our position
+   * in the heap whenever our @e desirability changes.
+   */
+  struct GNUNET_CONTAINER_HeapNode *hn;
+
+  /**
+   * Desirability of the path. How unique is it for the various peers
+   * on it?
+   */
+  GNUNET_CONTAINER_HeapCostType desirability;
+
+  /**
+   * Length of the @e entries array.
+   */
+  unsigned int entries_length;
+};
+
+
+/**
+ * Calculate the path's desirability score.
+ *
+ * @param path path to calculate the score for
+ */
+static void
+recalculate_path_desirability (struct CadetPeerPath *path)
+{
+  double result = 0.0;
+
+  for (unsigned int i = 0; i < path->entries_length; i++)
+  {
+    struct CadetPeer *cp = path->entries[i]->peer;
+
+    result += GCP_get_desirability_of_path (cp,
+                                            i);
+  }
+  path->desirability = (GNUNET_CONTAINER_HeapCostType) result;
+}
+
+
+/**
+ * Return how much we like keeping the path.  This is an aggregate
+ * score based on various factors, including the age of the path
+ * (older == better), and the value of this path to all of its adjacent
+ * peers.  For example, long paths that end at a peer that we have no
+ * shorter way to reach are very desirable, while long paths that end
+ * at a peer for which we have a shorter way as well are much less
+ * desirable.  Higher values indicate more valuable paths.  The
+ * returned value should be used to decide which paths to remember.
+ *
+ * @param path path to return the length for
+ * @return desirability of the path, larger is more desirable
+ */
+GNUNET_CONTAINER_HeapCostType
+GCPP_get_desirability (const struct CadetPeerPath *path)
+{
+  return path->desirability;
+}
+
+
+/**
+ * Return connection to @a destination using @a path, or return
+ * NULL if no such connection exists.
+ *
+ * @param path path to traverse
+ * @param destination destination node to get to, must be on path
+ * @param off offset of @a destination on @a path
+ * @return NULL if we have no existing connection
+ *         otherwise connection from us to @a destination via @a path
+ */
+struct CadetConnection *
+GCPP_get_connection (struct CadetPeerPath *path,
+                     struct CadetPeer *destination,
+                     unsigned int off)
+{
+  struct CadetPeerPathEntry *entry;
+
+  GNUNET_assert (off < path->entries_length);
+  entry = path->entries[off];
+  GNUNET_assert (entry->peer == destination);
+  return entry->cc;
+}
+
+
+/**
+ * Notify @a path that it is used for connection @a cc
+ * which ends at the path's offset @a off.
+ *
+ * @param path the path to remember the @a cc
+ * @param off the offset where the @a cc ends
+ * @param cc the connection to remember
+ */
+void
+GCPP_add_connection (struct CadetPeerPath *path,
+                     unsigned int off,
+                     struct CadetConnection *cc)
+{
+  struct CadetPeerPathEntry *entry;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Adding %s to path %s at offset %u\n",
+       GCC_2s (cc),
+       GCPP_2s (path),
+       off);
+  GNUNET_assert (off < path->entries_length);
+  entry = path->entries[off];
+  GNUNET_assert (NULL == entry->cc);
+  GNUNET_assert (NULL != cc);
+  entry->cc = cc;
+}
+
+
+/**
+ * Notify @a path that it is no longer used for connection @a cc which
+ * ended at the path's offset @a off.
+ *
+ * @param path the path to forget the @a cc
+ * @param off the offset where the @a cc ended
+ * @param cc the connection to forget
+ */
+void
+GCPP_del_connection (struct CadetPeerPath *path,
+                     unsigned int off,
+                     struct CadetConnection *cc)
+{
+  struct CadetPeerPathEntry *entry;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing connection %s to path %s at offset %u\n",
+       GCC_2s (cc),
+       GCPP_2s (path),
+       off);
+  GNUNET_assert (off < path->entries_length);
+  entry = path->entries[off];
+  GNUNET_assert (cc == entry->cc);
+  entry->cc = NULL;
+}
+
+
+/**
+ * Tries to attach @a path to a peer, working backwards from the end
+ * and stopping at @a stop_at. If path->hn is NULL on return then the
+ * path was not attached and you can assume that path->entries_length
+ * is equal to @a stop_at.
+ *
+ * @param path the path to attach
+ * @param stop_at the path length at which to stop trying
+ */
+static void
+attach_path (struct CadetPeerPath *path, unsigned int stop_at)
+{
+  GNUNET_assert (NULL == path->hn);
+
+  /* Try to attach this path to a peer, working backwards from the end. */
+  while (path->entries_length > stop_at)
+  {
+    unsigned int end = path->entries_length - 1;
+    struct CadetPeerPathEntry *entry = path->entries[end];
+    int force = GNUNET_NO;
+
+    recalculate_path_desirability (path);
+    /* If the entry already has a connection using it, force attach. */
+    if (NULL != entry->cc)
+      force = GNUNET_YES;
+    path->hn = GCP_attach_path (entry->peer,
+                                path,
+                                end,
+                                force);
+    if (NULL != path->hn)
+      break;
+
+    /* Attach failed, trim this entry from the path. */
+    GNUNET_assert (NULL == entry->cc);
+    GCP_path_entry_remove (entry->peer,
+                           entry,
+                           end);
+    GNUNET_free (entry);
+    path->entries[end] = NULL;
+    path->entries_length--;
+  }
+
+  /* Shrink array to actual path length. */
+  GNUNET_array_grow (path->entries,
+                     path->entries_length,
+                     path->entries_length);
+}
+
+
+/**
+ * The owning peer of this path is no longer interested in maintaining
+ * it, so the path should be discarded or shortened (in case a
+ * previous peer on the path finds the path desirable).
+ *
+ * @param path the path that is being released
+ */
+void
+GCPP_release (struct CadetPeerPath *path)
+{
+  struct CadetPeerPathEntry *entry;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Owner releases path %s\n",
+       GCPP_2s (path));
+  path->hn = NULL;
+  entry = path->entries[path->entries_length - 1];
+  GNUNET_assert (path == entry->path);
+  GNUNET_assert (NULL == entry->cc);
+  /* cut 'off' end of path */
+  GCP_path_entry_remove (entry->peer,
+                         entry,
+                         path->entries_length - 1);
+  GNUNET_free (entry);
+  path->entries[path->entries_length - 1] = NULL;
+  path->entries_length--;
+  /* see if new peer at the end likes this path any better */
+  attach_path (path, 0);
+  if (NULL == path->hn)
+  {
+    /* nobody wants us, discard the path */
+    GNUNET_assert (0 == path->entries_length);
+    GNUNET_assert (NULL == path->entries);
+    GNUNET_free (path);
+  }
+}
+
+
+/**
+ * Updates the score for an entry on the path based
+ * on our experiences with using @a path.
+ *
+ * @param path the path to update
+ * @param off offset of the entry to update
+ * @param delta change in the score to apply
+ */
+void
+GCPP_update_score (struct CadetPeerPath *path,
+                   unsigned int off,
+                   int delta)
+{
+  struct CadetPeerPathEntry *entry;
+
+  GNUNET_assert (off < path->entries_length);
+  entry = path->entries[off];
+
+  /* Add delta, with checks for overflows */
+  if (delta >= 0)
+  {
+    if (delta + entry->score < entry->score)
+      entry->score = INT_MAX;
+    else
+      entry->score += delta;
+  }
+  else
+  {
+    if (delta + entry->score > entry->score)
+      entry->score = INT_MIN;
+    else
+      entry->score += delta;
+  }
+  recalculate_path_desirability (path);
+}
+
+
+/**
+ * Closure for #find_peer_at() and #check_match().
+ */
+struct CheckMatchContext
+{
+  /**
+   * Set to a matching path, if any.
+   */
+  struct CadetPeerPath *match;
+
+  /**
+   * Array the combined paths.
+   */
+  struct CadetPeer **cpath;
+
+  /**
+   * How long is the @e cpath array?
+   */
+  unsigned int cpath_length;
+};
+
+
+/**
+ * Check if the given path is identical on all of the
+ * hops until @a off, and not longer than @a off.  If the
+ * @a path matches, store it in `match`.
+ *
+ * @param cls the `struct CheckMatchContext` to check against
+ * @param path the path to check
+ * @param off offset to check at
+ * @return #GNUNET_YES (continue to iterate), or if found #GNUNET_NO
+ */
+static int
+check_match (void *cls,
+             struct CadetPeerPath *path,
+             unsigned int off)
+{
+  struct CheckMatchContext *cm_ctx = cls;
+
+  GNUNET_assert (path->entries_length > off);
+  if ((path->entries_length != off + 1) &&
+      (off + 1 != cm_ctx->cpath_length))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "check_match mismatch because path %s is too long (%u vs. %u vs. %u)\n",
+         GCPP_2s (path),
+         path->entries_length,
+         off + 1,
+         cm_ctx->cpath_length);
+    return GNUNET_YES;   /* too long, goes somewhere else already, thus cannot be useful */
+  }
+  for (unsigned int i = 0; i < off; i++)
+    if (cm_ctx->cpath[i] !=
+        GCPP_get_peer_at_offset (path,
+                                 i))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "check_match path %s mismatches at offset %u\n",
+           GCPP_2s (path),
+           i);
+      return GNUNET_YES;   /* mismatch, ignore */
+    }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "check_match found match with path %s\n",
+       GCPP_2s (path));
+  cm_ctx->match = path;
+  return GNUNET_NO; /* match, we are done! */
+}
+
+
+/**
+ * Extend path @a path by the @a num_peers from the @a peers
+ * array, assuming the owners past the current owner want it.
+ *
+ * @param path path to extend
+ * @param peers list of peers beyond the end of @a path
+ * @param num_peers length of the @a peers array
+ * @param force force attachment, even if we have other
+ *        paths already
+ */
+static void
+extend_path (struct CadetPeerPath *path,
+             struct CadetPeer **peers,
+             unsigned int num_peers,
+             int force)
+{
+  unsigned int old_len = path->entries_length;
+  int i;
+
+  /* Expand path */
+  GNUNET_array_grow (path->entries,
+                     path->entries_length,
+                     old_len + num_peers);
+  for (i = num_peers - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = GNUNET_new (struct CadetPeerPathEntry);
+
+    path->entries[old_len + i] = entry;
+    entry->peer = peers[i];
+    entry->path = path;
+  }
+  for (i = num_peers - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = path->entries[old_len + i];
+
+    GCP_path_entry_add (entry->peer,
+                        entry,
+                        old_len + i);
+  }
+
+  /* If we extend an existing path, detach it from the
+     old owner and re-attach to the new one */
+  GCP_detach_path (path->entries[old_len - 1]->peer,
+                   path,
+                   path->hn);
+  path->hn = NULL;
+  path->entries_length = old_len + num_peers;
+  if (GNUNET_YES == force)
+  {
+    int end = path->entries_length - 1;
+
+    path->hn = GCP_attach_path (path->entries[end]->peer,
+                                path,
+                                end,
+                                GNUNET_YES);
+  }
+  else
+  {
+    attach_path (path, old_len);
+  }
+  if (NULL == path->hn)
+  {
+    /* none of the peers is interested in this path;
+       re-attach. */
+    GNUNET_assert (old_len == path->entries_length);
+    path->hn = GCP_attach_path (path->entries[old_len - 1]->peer,
+                                path,
+                                old_len - 1,
+                                GNUNET_YES);
+    GNUNET_assert (NULL != path->hn);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Extended path %s\n",
+       GCPP_2s (path));
+}
+
+
+/**
+ * Create a peer path based on the result of a DHT lookup.  If we
+ * already know this path, or one that is longer, simply return NULL.
+ * Otherwise, we try to extend an existing path, or create a new one
+ * if applicable.
+ *
+ * @param get_path path of the get request
+ * @param get_path_length length of @a get_path
+ * @param put_path path of the put request
+ * @param put_path_length length of the @a put_path
+ * @return a path through the network
+ */
+void
+GCPP_try_path_from_dht (const struct GNUNET_DHT_PathElement *get_path,
+                        unsigned int get_path_length,
+                        const struct GNUNET_DHT_PathElement *put_path,
+                        unsigned int put_path_length)
+{
+  struct CadetPeer *cpath[get_path_length + put_path_length];
+  struct CheckMatchContext cm_ctx;
+  struct CadetPeerPath *path;
+  unsigned int skip;
+  unsigned int total_len;
+
+  /* precompute 'cpath' so we can avoid doing the lookups lots of times */
+  skip = 0;
+  memset (cpath,
+          0,
+          sizeof(cpath));  /* Just to trigger harder errors later. */
+  total_len = get_path_length + put_path_length;
+  for (unsigned int off = 0; off < total_len; off++)
+  {
+    const struct GNUNET_PeerIdentity *pid;
+
+    pid = (off < get_path_length)
+          ? &get_path[get_path_length - off - 1].pred
+          : &put_path[get_path_length + put_path_length - off - 1].pred;
+    /* Check that I am not in the path */
+    if (0 == GNUNET_memcmp (&my_full_id,
+                            pid))
+    {
+      skip = off + 1;
+      continue;
+    }
+    cpath[off - skip] = GCP_get (pid,
+                                 GNUNET_YES);
+    /* Check that no peer is twice on the path */
+    for (unsigned int i = 0; i < off - skip; i++)
+    {
+      if (cpath[i] == cpath[off - skip])
+      {
+        skip = off - i;
+        break;
+      }
+    }
+  }
+  if (skip >= total_len)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Path discovered from DHT is one big cycle?\n");
+    return;
+  }
+  total_len -= skip;
+
+  /* First figure out if this path is a subset of an existing path, an
+     extension of an existing path, or a new path. */
+  cm_ctx.cpath_length = total_len;
+  cm_ctx.cpath = cpath;
+  cm_ctx.match = NULL;
+  for (int i = total_len - 1; i >= 0; i--)
+  {
+    GCP_iterate_paths_at (cpath[i],
+                          (unsigned int) i,
+                          &check_match,
+                          &cm_ctx);
+    if (NULL != cm_ctx.match)
+    {
+      if (i == total_len - 1)
+      {
+        /* Existing path includes this one, nothing to do! */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Path discovered from DHT is already known\n");
+        return;
+      }
+      if (cm_ctx.match->entries_length == i + 1)
+      {
+        /* Existing path ends in the middle of new path, extend it! */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Trying to extend existing path %s by additional links discovered from DHT\n",
+             GCPP_2s (cm_ctx.match));
+        extend_path (cm_ctx.match,
+                     &cpath[i + 1],
+                     total_len - i - 1,
+                     GNUNET_NO);
+        return;
+      }
+    }
+  }
+
+  /* No match at all, create completely new path */
+  path = GNUNET_new (struct CadetPeerPath);
+  path->entries_length = total_len;
+  path->entries = GNUNET_new_array (path->entries_length,
+                                    struct CadetPeerPathEntry *);
+  for (int i = path->entries_length - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = GNUNET_new (struct CadetPeerPathEntry);
+
+    path->entries[i] = entry;
+    entry->peer = cpath[i];
+    entry->path = path;
+  }
+  for (int i = path->entries_length - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = path->entries[i];
+
+    GCP_path_entry_add (entry->peer,
+                        entry,
+                        i);
+  }
+
+  /* Finally, try to attach it */
+  attach_path (path, 0);
+  if (NULL == path->hn)
+  {
+    /* None of the peers on the path care about it. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Path discovered from DHT is not interesting to us\n");
+    GNUNET_assert (0 == path->entries_length);
+    GNUNET_assert (NULL == path->entries);
+    GNUNET_free (path);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Created new path %s based on information from DHT\n",
+       GCPP_2s (path));
+}
+
+
+struct CadetPeerPath *
+GCPP_get_path_from_route (unsigned int path_length,
+                          const struct GNUNET_PeerIdentity *pids)
+{
+  struct CheckMatchContext cm_ctx;
+  struct CadetPeer *cpath[path_length];
+  struct CadetPeerPath *path;
+
+  /* precompute inverted 'cpath' so we can avoid doing the lookups and
+     have the correct order */
+  for (unsigned int off = 0; off < path_length; off++)
+    cpath[off] = GCP_get (&pids[path_length - 1 - off],
+                          GNUNET_YES);
+
+  /* First figure out if this path is a subset of an existing path, an
+     extension of an existing path, or a new path. */
+  cm_ctx.cpath = cpath;
+  cm_ctx.cpath_length = path_length;
+  cm_ctx.match = NULL;
+  for (int i = path_length - 1; i >= 0; i--)
+  {
+    GCP_iterate_paths_at (cpath[i],
+                          (unsigned int) i,
+                          &check_match,
+                          &cm_ctx);
+    if (NULL != cm_ctx.match)
+    {
+      if (i == path_length - 1)
+      {
+        /* Existing path includes this one, return the match! */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Returning existing path %s as inverse for incoming connection\n",
+             GCPP_2s (cm_ctx.match));
+        return cm_ctx.match;
+      }
+      if (cm_ctx.match->entries_length == i + 1)
+      {
+        /* Existing path ends in the middle of new path, extend it! */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Extending existing path %s to create inverse for incoming connection\n",
+             GCPP_2s (cm_ctx.match));
+        extend_path (cm_ctx.match,
+                     &cpath[i + 1],
+                     path_length - i - 1,
+                     GNUNET_YES);
+        /* Check that extension was successful */
+        GNUNET_assert (cm_ctx.match->entries_length == path_length);
+        return cm_ctx.match;
+      }
+      /* Eh, we found a match but couldn't use it? Something is wrong. */
+      GNUNET_break (0);
+    }
+  }
+
+  /* No match at all, create completely new path */
+  path = GNUNET_new (struct CadetPeerPath);
+  path->entries_length = path_length;
+  path->entries = GNUNET_new_array (path->entries_length,
+                                    struct CadetPeerPathEntry *);
+  for (int i = path_length - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = GNUNET_new (struct CadetPeerPathEntry);
+
+    path->entries[i] = entry;
+    entry->peer = cpath[i];
+    entry->path = path;
+  }
+  for (int i = path_length - 1; i >= 0; i--)
+  {
+    struct CadetPeerPathEntry *entry = path->entries[i];
+
+    GCP_path_entry_add (entry->peer,
+                        entry,
+                        i);
+  }
+  recalculate_path_desirability (path);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Created new path %s to create inverse for incoming connection\n",
+       GCPP_2s (path));
+  path->hn = GCP_attach_path (cpath[path_length - 1],
+                              path,
+                              path_length - 1,
+                              GNUNET_YES);
+  return path;
+}
+
+
+/**
+ * Return the length of the path.  Excludes one end of the
+ * path, so the loopback path has length 0.
+ *
+ * @param path path to return the length for
+ * @return number of peers on the path
+ */
+unsigned int
+GCPP_get_length (struct CadetPeerPath *path)
+{
+  return path->entries_length;
+}
+
+
+/**
+ * Find peer's offset on path.
+ *
+ * @param path path to search
+ * @param cp peer to look for
+ * @return offset of @a cp on @a path, or UINT_MAX if not found
+ */
+unsigned int
+GCPP_find_peer (struct CadetPeerPath *path,
+                struct CadetPeer *cp)
+{
+  for (unsigned int off = 0;
+       off < path->entries_length;
+       off++)
+    if (cp == GCPP_get_peer_at_offset (path,
+                                       off))
+      return off;
+  return UINT_MAX;
+}
+
+
+struct CadetPeer *
+GCPP_get_peer_at_offset (struct CadetPeerPath *path,
+                         unsigned int off)
+{
+  GNUNET_assert (off < path->entries_length);
+  return path->entries[off]->peer;
+}
+
+
+/**
+ * Convert a path to a human-readable string.
+ *
+ * @param path path to convert
+ * @return string, to be freed by caller (unlike other *_2s APIs!)
+ */
+const char *
+GCPP_2s (struct CadetPeerPath *path)
+{
+  static char buf[2048];
+  size_t off;
+  const unsigned int max_plen = (sizeof(buf) - 16) / 5 - 2; /* 5 characters per entry */
+
+  off = 0;
+  for (unsigned int i = 0;
+       i < path->entries_length;
+       i++)
+  {
+    if ((path->entries_length > max_plen) &&
+        (i == max_plen / 2))
+      off += GNUNET_snprintf (&buf[off],
+                              sizeof(buf) - off,
+                              "...-");
+    if ((path->entries_length > max_plen) &&
+        (i > max_plen / 2) &&
+        (i < path->entries_length - max_plen / 2))
+      continue;
+    off += GNUNET_snprintf (&buf[off],
+                            sizeof(buf) - off,
+                            "%s%s",
+                            GNUNET_i2s (GCP_get_id (GCPP_get_peer_at_offset (
+                                                      path,
+                                                      i))),
+                            (i == path->entries_length - 1) ? "" : "-");
+  }
+  GNUNET_snprintf (&buf[off],
+                   sizeof(buf) - off,
+                   "(%p)",
+                   path);
+  return buf;
+}
+
+
+/* end of gnunet-service-cadet-new_paths.c */
diff --git a/src/service/cadet/gnunet-service-cadet_paths.h b/src/service/cadet/gnunet-service-cadet_paths.h
new file mode 100644
index 000000000..0f3b4dd1a
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_paths.h
@@ -0,0 +1,183 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet-new_paths.h
+ * @brief Information we track per path.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_PATHS_H
+#define GNUNET_SERVICE_CADET_PATHS_H
+
+#include "gnunet_util_lib.h"
+#include "gnunet_dht_service.h"
+#include "gnunet-service-cadet.h"
+
+/**
+ * Create a peer path based on the result of a DHT lookup.  If we
+ * already know this path, or one that is longer, simply return NULL.
+ * Otherwise, we try to extend an existing path, or create a new one
+ * if applicable.
+ *
+ * @param get_path path of the get request
+ * @param get_path_length length of @a get_path
+ * @param put_path path of the put request
+ * @param put_path_length length of the @a put_path
+ */
+void
+GCPP_try_path_from_dht (const struct GNUNET_DHT_PathElement *get_path,
+                        unsigned int get_path_length,
+                        const struct GNUNET_DHT_PathElement *put_path,
+                        unsigned int put_path_length);
+
+
+/**
+ * We got an incoming connection, obtain the corresponding path.
+ *
+ * @param path_length number of segments on the @a path
+ * @param pids path through the network, in reverse order 
+ *             (we are at the end, at index @a path_length)
+ * @return corresponding path object
+ */
+struct CadetPeerPath *
+GCPP_get_path_from_route (unsigned int path_length,
+                          const struct GNUNET_PeerIdentity *pids);
+
+
+/**
+ * Return the length of the path.  Excludes one end of the
+ * path, so the loopback path has length 0.
+ *
+ * @param path path to return the length for
+ * @return number of peers on the path
+ */
+unsigned int
+GCPP_get_length (struct CadetPeerPath *path);
+
+
+/**
+ * Return connection to @a destination using @a path, or return
+ * NULL if no such connection exists.
+ *
+ * @param path path to traverse
+ * @param destination destination node to get to, must be on path
+ * @param off offset of @a destination on @a path
+ * @return NULL if we have no existing connection
+ *         otherwise connection from us to @a destination via @a path
+ */
+struct CadetConnection *
+GCPP_get_connection (struct CadetPeerPath *path,
+                     struct CadetPeer *destination,
+                     unsigned int off);
+
+
+/**
+ * Notify @a path that it is used for connection @a cc
+ * which ends at the path's offset @a off.
+ *
+ * @param path the path to remember the @a cc
+ * @param off the offset where the @a cc ends
+ * @param cc the connection to remember
+ */
+void
+GCPP_add_connection (struct CadetPeerPath *path,
+                     unsigned int off,
+                     struct CadetConnection *cc);
+
+
+/**
+ * Notify @a path that it is no longer used for connection @a cc which
+ * ended at the path's offset @a off.
+ *
+ * @param path the path to forget the @a cc
+ * @param off the offset where the @a cc ended
+ * @param cc the connection to forget
+ */
+void
+GCPP_del_connection (struct CadetPeerPath *path,
+                     unsigned int off,
+                     struct CadetConnection *cc);
+
+
+/**
+ * Find peer's offset on path.
+ *
+ * @param path path to search
+ * @param cp peer to look for
+ * @return offset of @a cp on @a path, or UINT_MAX if not found
+ */
+unsigned int
+GCPP_find_peer (struct CadetPeerPath *path,
+                struct CadetPeer *cp);
+
+
+/**
+ * Return how much we like keeping the path.  This is an aggregate
+ * score based on various factors, including the age of the path
+ * (older == better), and the value of this path to all of its adjacent
+ * peers.  For example, long paths that end at a peer that we have no
+ * shorter way to reach are very desirable, while long paths that end
+ * at a peer for which we have a shorter way as well are much less
+ * desirable.  Higher values indicate more valuable paths.  The
+ * returned value should be used to decide which paths to remember.
+ *
+ * @param path path to return the length for
+ * @return desirability of the path, larger is more desirable
+ */
+GNUNET_CONTAINER_HeapCostType
+GCPP_get_desirability (const struct CadetPeerPath *path);
+
+
+/**
+ * The given peer @a cp used to own this @a path.  However, it is no
+ * longer interested in maintaining it, so the path should be
+ * discarded or shortened (in case a previous peer on the path finds
+ * the path desirable).
+ *
+ * @param path the path that is being released
+ */
+void
+GCPP_release (struct CadetPeerPath *path);
+
+
+/**
+ * Obtain the peer at offset @a off in @a path.
+ *
+ * @param path peer path to inspect
+ * @param off offset to return, must be smaller than path length
+ * @return peer at offset @a off
+ */
+struct CadetPeer *
+GCPP_get_peer_at_offset (struct CadetPeerPath *path,
+                         unsigned int off);
+
+
+/**
+ * Convert a path to a human-readable string.
+ *
+ * @param path path to convert
+ * @return string, statically allocated
+ */
+const char *
+GCPP_2s (struct CadetPeerPath *p);
+
+
+#endif
diff --git a/src/service/cadet/gnunet-service-cadet_peer.c b/src/service/cadet/gnunet-service-cadet_peer.c
new file mode 100644
index 000000000..03e8b8962
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_peer.c
@@ -0,0 +1,1554 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_peer.c
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - optimize stopping/restarting DHT search to situations
+ *   where we actually need it (i.e. not if we have a direct connection,
+ *   or if we already have plenty of good short ones, or maybe even
+ *   to take a break if we have some connections and have searched a lot (?))
+ */
+#include "platform.h"
+#include "gnunet_time_lib.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_hello_uri_lib.h"
+#include "gnunet_signatures.h"
+#include "gnunet_transport_application_service.h"
+#include "gnunet_core_service.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
+
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-per", __VA_ARGS__)
+
+
+/**
+ * How long do we wait until tearing down an idle peer?
+ */
+#define IDLE_PEER_TIMEOUT GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MINUTES, 5)
+
+/**
+ * How long do we keep paths around if we no longer care about the peer?
+ */
+#define IDLE_PATH_TIMEOUT GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MINUTES, 2)
+
+/**
+ * Queue size when we start dropping OOO messages.
+ */
+#define MAX_OOO_QUEUE_SIZE  100
+
+/**
+ * Data structure used to track whom we have to notify about changes
+ * to our message queue.
+ */
+struct GCP_MessageQueueManager
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *next;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *prev;
+
+  /**
+   * Function to call with updated message queue object.
+   */
+  GCP_MessageQueueNotificationCallback cb;
+
+  /**
+   * Closure for @e cb.
+   */
+  void *cb_cls;
+
+  /**
+   * The peer this is for.
+   */
+  struct CadetPeer *cp;
+
+  /**
+   * Envelope this manager would like to transmit once it is its turn.
+   */
+  struct GNUNET_MQ_Envelope *env;
+};
+
+
+/**
+ * Struct containing all information regarding a given peer
+ */
+struct CadetPeer
+{
+  /**
+   * ID of the peer
+   */
+  struct GNUNET_PeerIdentity pid;
+
+  /**
+   * Last time we heard from this peer (currently not used!)
+   */
+  struct GNUNET_TIME_Absolute last_connection_create;
+
+  /**
+   * Array of DLLs of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_heads;
+
+  /**
+   * Array of DLL of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_tails;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_head;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_tail;
+
+  /**
+   * Pointer to first "ready" entry in @e mqm_head.
+   */
+  struct GCP_MessageQueueManager *mqm_ready_ptr;
+
+  /**
+   * MIN-heap of paths owned by this peer (they also end at this
+   * peer).  Ordered by desirability.
+   */
+  struct GNUNET_CONTAINER_Heap *path_heap;
+
+  /**
+   * Handle to stop the DHT search for paths to this peer
+   */
+  struct GCD_search_handle *search_h;
+
+  /**
+   * Task to clean up @e path_heap asynchronously.
+   */
+  struct GNUNET_SCHEDULER_Task *heap_cleanup_task;
+
+  /**
+   * Task to destroy this entry.
+   */
+  struct GNUNET_SCHEDULER_Task *destroy_task;
+
+  /**
+   * Tunnel to this peer, if any.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Connections that go through this peer; indexed by tid.
+   */
+  struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+  /**
+   * Handle for core transmissions.
+   */
+  struct GNUNET_MQ_Handle *core_mq;
+
+  /**
+   * Hello message of the peer.
+   */
+  struct GNUNET_MessageHeader *hello;
+
+  /**
+   * Handle to us offering the HELLO to the transport.
+   */
+  struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
+
+  /**
+   * Transport suggest handle.
+   */
+  struct GNUNET_TRANSPORT_ApplicationSuggestHandle *ash;
+
+  /**
+   * How many messages are in the queue to this peer.
+   */
+  unsigned int queue_n;
+
+  /**
+   * How many paths do we have to this peer (in all @e path_heads DLLs combined).
+   */
+  unsigned int num_paths;
+
+  /**
+   * Sum over all of the offsets of all of the paths in the @a path_heads DLLs.
+   * Used to speed-up #GCP_get_desirability_of_path() calculation.
+   */
+  unsigned int off_sum;
+
+  /**
+   * Number of message queue managers of this peer that have a message in waiting.
+   *
+   * Used to quickly see if we need to bother scanning the @e msm_head DLL.
+   * TODO: could be replaced by another DLL that would then allow us to avoid
+   * the O(n)-scan of the DLL for ready entries!
+   */
+  unsigned int mqm_ready_counter;
+
+  /**
+   * Current length of the @e path_heads and @e path_tails arrays.
+   * The arrays should be grown as needed.
+   */
+  unsigned int path_dll_length;
+};
+
+
+const char *
+GCP_2s (const struct CadetPeer *cp)
+{
+  static char buf[5];
+  char *ret;
+
+  if ((NULL == cp) ||
+      (GNUNET_YES == GNUNET_is_zero (&cp->pid.public_key)))
+    return "NULL";
+
+  ret = GNUNET_CRYPTO_eddsa_public_key_to_string (&cp->pid.public_key);
+  if (NULL == ret)
+    return "NULL";
+
+  GNUNET_strlcpy (buf,
+                  ret,
+                  sizeof(buf));
+  GNUNET_free (ret);
+  return buf;
+}
+
+
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off)
+{
+  unsigned int num_alts = cp->num_paths;
+  unsigned int off_sum;
+  double avg_sum;
+  double path_delta;
+  double weight_alts;
+
+  GNUNET_assert (num_alts >= 1);  /* 'path' should be in there! */
+  GNUNET_assert (0 != cp->path_dll_length);
+
+  /* We maintain 'off_sum' in 'peer' and thereby
+     avoid the SLOW recalculation each time. Kept here
+     just to document what is going on. */
+#if SLOW
+  off_sum = 0;
+  for (unsigned int j = 0; j < cp->path_dll_length; j++)
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[j];
+         NULL != pe;
+         pe = pe->next)
+      off_sum += j;
+  GNUNET_assert (off_sum == cp->off_sum);
+#else
+  off_sum = cp->off_sum;
+#endif
+  avg_sum = off_sum * 1.0 / cp->path_dll_length;
+  path_delta = off - avg_sum;
+  /* path_delta positive: path off of peer above average (bad path for peer),
+     path_delta negative: path off of peer below average (good path for peer) */
+  if (path_delta <= -1.0)
+    weight_alts = -num_alts / path_delta;  /* discount alternative paths */
+  else if (path_delta >= 1.0)
+    weight_alts = num_alts * path_delta; /* overcount alternative paths */
+  else
+    weight_alts = num_alts; /* count alternative paths normally */
+
+
+  /* off+1: long paths are generally harder to find and thus count
+     a bit more as they get longer.  However, above-average paths
+     still need to count less, hence the squaring of that factor. */
+  return (off + 1.0) / (weight_alts * weight_alts);
+}
+
+
+/**
+ * This peer is no longer be needed, clean it up now.
+ *
+ * @param cls peer to clean up
+ */
+static void
+destroy_peer (void *cls)
+{
+  struct CadetPeer *cp = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying state about peer %s\n",
+       GCP_2s (cp));
+  cp->destroy_task = NULL;
+  GNUNET_assert (NULL == cp->t);
+  GNUNET_assert (NULL == cp->core_mq);
+  GNUNET_assert (0 == cp->num_paths);
+  for (unsigned int i = 0; i < cp->path_dll_length; i++)
+    GNUNET_assert (NULL == cp->path_heads[i]);
+  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (peers,
+                                                       &cp->pid,
+                                                       cp));
+  GNUNET_free (cp->path_heads);
+  GNUNET_free (cp->path_tails);
+  cp->path_dll_length = 0;
+  if (NULL != cp->search_h)
+  {
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  /* FIXME: clean up search_delayedXXX! */
+
+
+  if (NULL != cp->ash)
+  {
+    GNUNET_TRANSPORT_application_suggest_cancel (cp->ash);
+    cp->ash = NULL;
+  }
+  GNUNET_CONTAINER_multishortmap_destroy (cp->connections);
+  if (NULL != cp->path_heap)
+  {
+    GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+    cp->path_heap = NULL;
+  }
+  if (NULL != cp->heap_cleanup_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->heap_cleanup_task);
+    cp->heap_cleanup_task = NULL;
+  }
+  GNUNET_free (cp->hello);
+  /* Peer should not be freed if paths exist; if there are no paths,
+     there ought to be no connections, and without connections, no
+     notifications. Thus we can assert that mqm_head is empty at this
+     point. */
+  GNUNET_assert (NULL == cp->mqm_head);
+  GNUNET_assert (NULL == cp->mqm_ready_ptr);
+  GNUNET_free (cp);
+}
+
+
+/**
+ * This peer is now on more "active" duty, activate processes related to it.
+ *
+ * @param cp the more-active peer
+ */
+static void
+consider_peer_activate (struct CadetPeer *cp)
+{
+  struct GNUNET_BANDWIDTH_Value32NBO bw;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Updating peer %s activation state (%u connections)%s%s\n",
+       GCP_2s (cp),
+       GNUNET_CONTAINER_multishortmap_size (cp->connections),
+       (NULL == cp->t) ? "" : " with tunnel",
+       (NULL == cp->core_mq) ? "" : " with CORE link");
+  if (NULL != cp->destroy_task)
+  {
+    /* It's active, do not destroy! */
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if ((0 == GNUNET_CONTAINER_multishortmap_size (cp->connections)) &&
+      (NULL == cp->t))
+  {
+    /* We're just on a path or directly connected; don't bother too much */
+    if (NULL != cp->ash)
+    {
+      GNUNET_TRANSPORT_application_suggest_cancel (cp->ash);
+      cp->ash = NULL;
+    }
+    if (NULL != cp->search_h)
+    {
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
+    }
+    return;
+  }
+  if (NULL == cp->core_mq)
+  {
+    /* Lacks direct connection, try to create one by querying the DHT */
+    if ((NULL == cp->search_h) &&
+        (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths))
+      cp->search_h
+        = GCD_search (&cp->pid);
+  }
+  else
+  {
+    /* Have direct connection, stop DHT search if active */
+    if (NULL != cp->search_h)
+    {
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
+    }
+  }
+
+  if (NULL != cp->ash)
+    GNUNET_TRANSPORT_application_suggest_cancel (cp->ash);
+  bw.value__ = 0;
+  cp->ash
+    = GNUNET_TRANSPORT_application_suggest (transport,
+                                            &cp->pid,
+                                            GNUNET_MQ_PRIO_BEST_EFFORT,
+                                            bw);
+}
+
+
+/**
+ * This peer may no longer be needed, consider cleaning it up.
+ *
+ * @param cp peer to clean up
+ */
+static void
+consider_peer_destroy (struct CadetPeer *cp);
+
+
+/**
+ * We really no longere care about a peer, stop hogging memory with paths to it.
+ * Afterwards, see if there is more to be cleaned up about this peer.
+ *
+ * @param cls a `struct CadetPeer`.
+ */
+static void
+drop_paths (void *cls)
+{
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *path;
+
+  cp->destroy_task = NULL;
+  while (NULL != (path = GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  consider_peer_destroy (cp);
+}
+
+
+/**
+ * This peer may no longer be needed, consider cleaning it up.
+ *
+ * @param cp peer to clean up
+ */
+static void
+consider_peer_destroy (struct CadetPeer *cp)
+{
+  struct GNUNET_TIME_Relative exp;
+
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if (NULL != cp->t)
+    return; /* still relevant! */
+  if (NULL != cp->core_mq)
+    return; /* still relevant! */
+  if (0 != GNUNET_CONTAINER_multishortmap_size (cp->connections))
+    return; /* still relevant! */
+  if ((NULL != cp->path_heap) &&
+      (0 < GNUNET_CONTAINER_heap_get_size (cp->path_heap)))
+  {
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PATH_TIMEOUT,
+                                                     &drop_paths,
+                                                     cp);
+    return;
+  }
+  if (0 != cp->num_paths)
+    return; /* still relevant! */
+  if (NULL != cp->hello)
+  {
+    /* relevant only until HELLO expires */
+    exp = GNUNET_TIME_absolute_get_remaining (
+      GNUNET_HELLO_builder_get_expiration_time (cp
+                                                ->hello));
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (exp,
+                                                     &destroy_peer,
+                                                     cp);
+    return;
+  }
+  cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PEER_TIMEOUT,
+                                                   &destroy_peer,
+                                                   cp);
+}
+
+
+/**
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
+ *
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
+ */
+void
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message queue for peer %s is now %p\n",
+       GCP_2s (cp),
+       mq);
+  cp->core_mq = mq;
+  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head, *next;
+       NULL != mqm;
+       mqm = next)
+  {
+    /* Save next pointer in case mqm gets freed by the callback */
+    next = mqm->next;
+    if (NULL == mq)
+    {
+      if (NULL != mqm->env)
+      {
+        GNUNET_MQ_discard (mqm->env);
+        mqm->env = NULL;
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_SYSERR);
+      }
+      else
+      {
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_NO);
+      }
+    }
+    else
+    {
+      GNUNET_assert (NULL == mqm->env);
+      mqm->cb (mqm->cb_cls,
+               GNUNET_YES);
+    }
+  }
+  if ((NULL != mq) ||
+      (NULL != cp->t))
+    consider_peer_activate (cp);
+  else
+    consider_peer_destroy (cp);
+
+  if ((NULL != mq) &&
+      (NULL != cp->t))
+  {
+    /* have a new, direct path to the target, notify tunnel */
+    struct CadetPeerPath *path;
+
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    GCT_consider_path (cp->t,
+                       path,
+                       0);
+  }
+}
+
+
+/**
+ * Debug function should NEVER return true in production code, useful to
+ * simulate losses for testcases.
+ *
+ * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
+ */
+static int
+should_I_drop (void)
+{
+  if (0 == drop_percent)
+    return GNUNET_NO;
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                101) < drop_percent)
+    return GNUNET_YES;
+  return GNUNET_NO;
+}
+
+
+/**
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
+ *
+ * @param cls the `struct CadetPeeer` where we made progress
+ */
+static void
+mqm_send_done (void *cls);
+
+
+/**
+ * Transmit current envelope from this @a mqm.
+ *
+ * @param mqm mqm to transmit message for now
+ */
+static void
+mqm_execute (struct GCP_MessageQueueManager *mqm)
+{
+  struct CadetPeer *cp = mqm->cp;
+
+  /* Move ready pointer to the next entry that might be ready. */
+  if ((mqm == cp->mqm_ready_ptr) &&
+      (NULL != mqm->next))
+    cp->mqm_ready_ptr = mqm->next;
+  /* Move entry to the end of the DLL, to be fair. */
+  if (mqm != cp->mqm_tail)
+  {
+    GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                                 cp->mqm_tail,
+                                 mqm);
+    GNUNET_CONTAINER_DLL_insert_tail (cp->mqm_head,
+                                      cp->mqm_tail,
+                                      mqm);
+  }
+  cp->mqm_ready_counter--;
+  if (GNUNET_YES == should_I_drop ())
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "DROPPING message to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_discard (mqm->env);
+    mqm->env = NULL;
+    mqm_send_done (cp);
+  }
+  else
+  {
+    {
+      const struct GNUNET_MessageHeader *mh;
+
+      mh = GNUNET_MQ_env_get_msg (mqm->env);
+      switch (ntohs (mh->type))
+      {
+      case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
+        {
+          const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg
+            = (const struct GNUNET_CADET_TunnelKeyExchangeMessage *) mh;
+          LOG (GNUNET_ERROR_TYPE_DEBUG,
+               "P2P forwarding KX with ephemeral %s to %s on CID %s\n",
+               GNUNET_e2s (&msg->ephemeral_key),
+               GCP_2s (cp),
+               GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+        }
+        break;
+
+      default:
+        break;
+      }
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Sending to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_send (cp->core_mq,
+                    mqm->env);
+    mqm->env = NULL;
+  }
+  mqm->cb (mqm->cb_cls,
+           GNUNET_YES);
+}
+
+
+/**
+ * Find the next ready message in the queue (starting
+ * the search from the `cp->mqm_ready_ptr`) and if possible
+ * execute the transmission.
+ *
+ * @param cp peer to try to send the next ready message to
+ */
+static void
+send_next_ready (struct CadetPeer *cp)
+{
+  struct GCP_MessageQueueManager *mqm;
+
+  if (0 == cp->mqm_ready_counter)
+    return;
+  while ((NULL != (mqm = cp->mqm_ready_ptr)) &&
+         (NULL == mqm->env))
+    cp->mqm_ready_ptr = mqm->next;
+  if (NULL == mqm)
+    return; /* nothing to do */
+  mqm_execute (mqm);
+}
+
+
+/**
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
+ *
+ * @param cls the `struct CadetPeeer` where we made progress
+ */
+static void
+mqm_send_done (void *cls)
+{
+  struct CadetPeer *cp = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending to peer %s completed\n",
+       GCP_2s (cp));
+  send_next_ready (cp);
+}
+
+
+/**
+ * Send the message in @a env to @a cp.
+ *
+ * @param mqm the message queue manager to use for transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
+ */
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env)
+{
+  struct CadetPeer *cp = mqm->cp;
+
+  GNUNET_assert (NULL != env);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queueing message to peer %s in MQM %p\n",
+       GCP_2s (cp),
+       mqm);
+  GNUNET_assert (NULL != cp->core_mq);
+  GNUNET_assert (NULL == mqm->env);
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  mqm->env = env;
+  cp->mqm_ready_counter++;
+  if (mqm != cp->mqm_ready_ptr)
+    cp->mqm_ready_ptr = cp->mqm_head;
+  if (1 == cp->mqm_ready_counter)
+    cp->mqm_ready_ptr = mqm;
+  if (0 != GNUNET_MQ_get_length (cp->core_mq))
+    return;
+  send_next_ready (cp);
+}
+
+
+/**
+ * Function called to destroy a peer now.
+ *
+ * @param cls NULL
+ * @param pid identity of the peer (unused)
+ * @param value the `struct CadetPeer` to clean up
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_iterator_cb (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
+{
+  struct CadetPeer *cp = value;
+
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  destroy_peer (cp);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
+ */
+void
+GCP_destroy_all_peers ()
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all peers now\n");
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         &destroy_iterator_cb,
+                                         NULL);
+}
+
+
+/**
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
+ *
+ * @param cp peer to drop paths to
+ */
+void
+GCP_drop_owned_paths (struct CadetPeer *cp)
+{
+  struct CadetPeerPath *path;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all paths to %s\n",
+       GCP_2s (cp));
+  while (NULL != (path =
+                    GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+  cp->path_heap = NULL;
+}
+
+
+/**
+ * Add an entry to the DLL of all of the paths that this peer is on.
+ *
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
+ */
+void
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off)
+{
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (entry->path,
+                                                off));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Discovered that peer %s is on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  if (off >= cp->path_dll_length)
+  {
+    unsigned int len = cp->path_dll_length;
+
+    GNUNET_array_grow (cp->path_heads,
+                       len,
+                       off + 4);
+    GNUNET_array_grow (cp->path_tails,
+                       cp->path_dll_length,
+                       off + 4);
+  }
+  GNUNET_CONTAINER_DLL_insert (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  cp->off_sum += off;
+  cp->num_paths++;
+
+  /* If we have a tunnel to this peer, tell the tunnel that there is a
+     new path available. */
+  if (NULL != cp->t)
+    GCT_consider_path (cp->t,
+                       entry->path,
+                       off);
+
+  if ((NULL != cp->search_h) &&
+      (DESIRED_CONNECTIONS_PER_TUNNEL <= cp->num_paths))
+  {
+    /* Now I have enough paths, stop search */
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  if (NULL != cp->destroy_task)
+  {
+    /* paths changed, this resets the destroy timeout counter
+       and aborts a destroy task that may no longer be valid
+       to have (as we now have more paths via this peer). */
+    consider_peer_destroy (cp);
+  }
+}
+
+
+/**
+ * Remove an entry from the DLL of all of the paths that this peer is on.
+ *
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
+ */
+void
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing knowledge about peer %s beging on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  GNUNET_CONTAINER_DLL_remove (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  GNUNET_assert (0 < cp->num_paths);
+  cp->off_sum -= off;
+  cp->num_paths--;
+  if ((NULL == cp->core_mq) &&
+      (NULL != cp->t) &&
+      (NULL == cp->search_h) &&
+      (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths))
+    cp->search_h
+      = GCD_search (&cp->pid);
+  if (NULL == cp->destroy_task)
+  {
+    /* paths changed, we might now be ready for destruction, check again */
+    consider_peer_destroy (cp);
+  }
+}
+
+
+/**
+ * Prune down the number of paths to this peer, we seem to
+ * have way too many.
+ *
+ * @param cls the `struct CadetPeer` to maintain the path heap for
+ */
+static void
+path_heap_cleanup (void *cls)
+{
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *root;
+
+  cp->heap_cleanup_task = NULL;
+  while (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+         2 * DESIRED_CONNECTIONS_PER_TUNNEL)
+  {
+    /* Now we have way too many, drop least desirable UNLESS it is in use!
+       (Note that this intentionally keeps highly desirable, but currently
+       unused paths around in the hope that we might be able to switch, even
+       if the number of paths exceeds the threshold.) */
+    root = GNUNET_CONTAINER_heap_peek (cp->path_heap);
+    GNUNET_assert (NULL != root);
+    if (NULL !=
+        GCPP_get_connection (root,
+                             cp,
+                             GCPP_get_length (root) - 1))
+      break;   /* can't fix */
+    /* Got plenty of paths to this destination, and this is a low-quality
+       one that we don't care about. Allow it to die. */
+    GNUNET_assert (root ==
+                   GNUNET_CONTAINER_heap_remove_root (cp->path_heap));
+    GCPP_release (root);
+  }
+}
+
+
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force)
+{
+  GNUNET_CONTAINER_HeapCostType desirability;
+  struct CadetPeerPath *root;
+  GNUNET_CONTAINER_HeapCostType root_desirability;
+  struct GNUNET_CONTAINER_HeapNode *hn;
+
+  GNUNET_assert (off == GCPP_get_length (path) - 1);
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (path,
+                                                off));
+  if (NULL == cp->path_heap)
+  {
+    /* #GCP_drop_owned_paths() was already called, we cannot take new ones! */
+    GNUNET_assert (GNUNET_NO == force);
+    return NULL;
+  }
+  desirability = GCPP_get_desirability (path);
+  if (GNUNET_NO == force)
+  {
+    /* FIXME: desirability is not yet initialized; tricky! */
+    if (GNUNET_NO ==
+        GNUNET_CONTAINER_heap_peek2 (cp->path_heap,
+                                     (void **) &root,
+                                     &root_desirability))
+    {
+      root = NULL;
+      root_desirability = 0;
+    }
+
+    if ((DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) &&
+        (desirability < root_desirability))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Decided to not attach path %s to peer %s due to undesirability\n",
+           GCPP_2s (path),
+           GCP_2s (cp));
+      return NULL;
+    }
+  }
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Attaching path %s to peer %s (%s)\n",
+       GCPP_2s (path),
+       GCP_2s (cp),
+       (GNUNET_NO == force) ? "desirable" : "forced");
+
+  /* Yes, we'd like to add this path, add to our heap */
+  hn = GNUNET_CONTAINER_heap_insert (cp->path_heap,
+                                     path,
+                                     desirability);
+
+  /* Consider maybe dropping other paths because of the new one */
+  if ((GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+       2 * DESIRED_CONNECTIONS_PER_TUNNEL) &&
+      (NULL != cp->heap_cleanup_task))
+    cp->heap_cleanup_task = GNUNET_SCHEDULER_add_now (&path_heap_cleanup,
+                                                      cp);
+  return hn;
+}
+
+
+/**
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
+ *
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
+ */
+void
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Detaching path %s from peer %s\n",
+       GCPP_2s (path),
+       GCP_2s (cp));
+  GNUNET_assert (path ==
+                 GNUNET_CONTAINER_heap_remove_node (hn));
+}
+
+
+/**
+ * Add a @a connection to this @a cp.
+ *
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
+ */
+void
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Adding %s to peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multishortmap_put (cp->connections,
+                                                     &GCC_get_id (
+                                                       cc)->connection_of_tunnel,
+                                                     cc,
+                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+}
+
+
+/**
+ * Remove a @a connection that went via this @a cp.
+ *
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
+ */
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing connection %s from peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (cp->connections,
+                                                        &GCC_get_id (
+                                                          cc)->
+                                                        connection_of_tunnel,
+                                                        cc));
+  consider_peer_destroy (cp);
+}
+
+
+/**
+ * Retrieve the CadetPeer structure associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
+ *
+ * @param peer_id Full identity of the peer.
+ * @param create #GNUNET_YES if a new peer should be created if unknown.
+ *               #GNUNET_NO to return NULL if peer is unknown.
+ * @return Existing or newly created peer structure.
+ *         NULL if unknown and not requested @a create
+ */
+struct CadetPeer *
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create)
+{
+  struct CadetPeer *cp;
+
+  cp = GNUNET_CONTAINER_multipeermap_get (peers,
+                                          peer_id);
+  if (NULL != cp)
+    return cp;
+  if (GNUNET_NO == create)
+    return NULL;
+  cp = GNUNET_new (struct CadetPeer);
+  cp->pid = *peer_id;
+  cp->connections = GNUNET_CONTAINER_multishortmap_create (32,
+                                                           GNUNET_YES);
+  cp->path_heap = GNUNET_CONTAINER_heap_create (
+    GNUNET_CONTAINER_HEAP_ORDER_MIN);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_put (peers,
+                                                    &cp->pid,
+                                                    cp,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating peer %s\n",
+       GCP_2s (cp));
+  return cp;
+}
+
+
+/**
+ * Obtain the peer identity for a `struct CadetPeer`.
+ *
+ * @param cp our peer handle
+ * @return the peer identity
+ */
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp)
+{
+  return &cp->pid;
+}
+
+
+/**
+ * Iterate over all known peers.
+ *
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
+ */
+void
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls)
+{
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         iter,
+                                         cls);
+}
+
+
+/**
+ * Count the number of known paths toward the peer.
+ *
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
+ */
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp)
+{
+  return cp->num_paths;
+}
+
+
+/**
+ * Iterate over the paths to a peer.
+ *
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
+ */
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls)
+{
+  unsigned int ret = 0;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Iterating over paths to peer %s%s\n",
+       GCP_2s (cp),
+       (NULL == cp->core_mq) ? "" : " including direct link");
+  if (NULL != cp->core_mq)
+  {
+    /* FIXME: this branch seems to duplicate the
+       i=0 case below (direct link). Leave out!??? -CG */
+    struct CadetPeerPath *path;
+
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    ret++;
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  path,
+                  0))
+      return ret;
+  }
+  for (unsigned int i = 0; i < cp->path_dll_length; i++)
+  {
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
+         NULL != pe;
+         pe = pe->next)
+    {
+      ret++;
+      if (GNUNET_NO ==
+          callback (callback_cls,
+                    pe->path,
+                    i))
+        return ret;
+    }
+  }
+  return ret;
+}
+
+
+/**
+ * Iterate over the paths to a peer without direct link.
+ *
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
+ */
+unsigned int
+GCP_iterate_indirect_paths (struct CadetPeer *cp,
+                            GCP_PathIterator callback,
+                            void *callback_cls)
+{
+  unsigned int ret = 0;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Iterating over paths to peer %s without direct link\n",
+       GCP_2s (cp));
+  for (unsigned int i = 1; i < cp->path_dll_length; i++)
+  {
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
+         NULL != pe;
+         pe = pe->next)
+    {
+      ret++;
+      if (GNUNET_NO ==
+          callback (callback_cls,
+                    pe->path,
+                    i))
+        return ret;
+    }
+  }
+  return ret;
+}
+
+
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls)
+{
+  unsigned int ret = 0;
+
+  if (dist >= cp->path_dll_length)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Asked to look for paths at distance %u, but maximum for me is < %u\n",
+         dist,
+         cp->path_dll_length);
+    return 0;
+  }
+  for (struct CadetPeerPathEntry *pe = cp->path_heads[dist];
+       NULL != pe;
+       pe = pe->next)
+  {
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  pe->path,
+                  dist))
+      return ret;
+    ret++;
+  }
+  return ret;
+}
+
+
+/**
+ * Get the tunnel towards a peer.
+ *
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
+ * @return Tunnel towards peer.
+ */
+struct CadetTunnel *
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create)
+{
+  if (NULL == cp)
+    return NULL;
+  if ((NULL != cp->t) ||
+      (GNUNET_NO == create))
+    return cp->t;
+  cp->t = GCT_create_tunnel (cp);
+  consider_peer_activate (cp);
+  return cp->t;
+}
+
+
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_MessageHeader *hello)
+{
+  struct GNUNET_BANDWIDTH_Value32NBO bw;
+  uint16_t size = ntohs (hello->size);
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got %u byte HELLO for peer %s\n",
+       (unsigned int) size,
+       GCP_2s (cp));
+  if (NULL != cp->hello)
+  {
+    struct GNUNET_TIME_Absolute now = GNUNET_TIME_absolute_get ();
+
+    struct GNUNET_TIME_Absolute new_hello_exp =
+      GNUNET_HELLO_builder_get_expiration_time (hello);
+    struct GNUNET_TIME_Absolute old_hello_exp =
+      GNUNET_HELLO_builder_get_expiration_time (cp
+                                                ->hello);
+
+    if (GNUNET_TIME_absolute_cmp (new_hello_exp, >, now) &&
+        GNUNET_TIME_absolute_cmp (new_hello_exp, >, old_hello_exp))
+    {
+      GNUNET_free (cp->hello);
+      cp->hello = GNUNET_malloc (size);
+      GNUNET_memcpy (cp->hello, hello, size);
+    }
+    else
+    {
+      return;
+    }
+  }
+  else
+  {
+    cp->hello = GNUNET_memdup (hello,
+                               size);
+  }
+  if (NULL != cp->ash)
+    GNUNET_TRANSPORT_application_suggest_cancel (cp->ash);
+  bw.value__ = 0;
+  cp->ash
+    = GNUNET_TRANSPORT_application_suggest (transport,
+                                            &cp->pid,
+                                            GNUNET_MQ_PRIO_BEST_EFFORT,
+                                            bw);
+  /* New HELLO means cp's destruction time may change... */
+  consider_peer_destroy (cp);
+}
+
+
+/**
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
+ *
+ * @param cp the peer affected
+ * @param t the dead tunnel
+ */
+void
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Dropping tunnel %s to peer %s\n",
+       GCT_2s (t),
+       GCP_2s (cp));
+  GNUNET_assert (cp->t == t);
+  cp->t = NULL;
+  consider_peer_destroy (cp);
+}
+
+
+/**
+ * Test if @a cp has a core-level connection
+ *
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
+ */
+int
+GCP_has_core_connection (struct CadetPeer *cp)
+{
+  return (NULL != cp->core_mq) ? GNUNET_YES : GNUNET_NO;
+}
+
+
+/**
+ * Start message queue change notifications.
+ *
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
+ */
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls)
+{
+  struct GCP_MessageQueueManager *mqm;
+
+  mqm = GNUNET_new (struct GCP_MessageQueueManager);
+  mqm->cb = cb;
+  mqm->cb_cls = cb_cls;
+  mqm->cp = cp;
+  GNUNET_CONTAINER_DLL_insert (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating MQM %p for peer %s\n",
+       mqm,
+       GCP_2s (cp));
+  if (NULL != cp->core_mq)
+    cb (cb_cls,
+        GNUNET_YES);
+  return mqm;
+}
+
+
+/**
+ * Stops message queue change notifications.
+ *
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
+ */
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env)
+{
+  struct CadetPeer *cp = mqm->cp;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying MQM %p for peer %s%s\n",
+       mqm,
+       GCP_2s (cp),
+       (NULL == last_env) ? "" : " with last ditch transmission");
+  if (NULL != mqm->env)
+    GNUNET_MQ_discard (mqm->env);
+  if (NULL != last_env)
+  {
+    if (NULL != cp->core_mq)
+    {
+      GNUNET_MQ_notify_sent (last_env,
+                             &mqm_send_done,
+                             cp);
+      GNUNET_MQ_send (cp->core_mq,
+                      last_env);
+    }
+    else
+    {
+      GNUNET_MQ_discard (last_env);
+    }
+  }
+  if (cp->mqm_ready_ptr == mqm)
+    cp->mqm_ready_ptr = mqm->next;
+  GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  GNUNET_free (mqm);
+}
+
+
+/**
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
+ *
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
+ */
+void
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message to %s out of management\n",
+       GCP_2s (cp));
+  if (NULL == cp->core_mq)
+  {
+    GNUNET_MQ_discard (env);
+    return;
+  }
+  if (GNUNET_MQ_get_length (cp->core_mq) > MAX_OOO_QUEUE_SIZE)
+  {
+    GNUNET_MQ_discard (env);
+    return;
+  }
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  GNUNET_MQ_send (cp->core_mq,
+                  env);
+}
+
+
+/**
+ * Checking if a monotime value is newer than the last monotime value received from a peer. If the time value is newer it will be stored at the peer.
+ *
+ * @param peer The peer we received a new time value from.
+ * @param monotime Time value we check against the last time value we received from a peer.
+ * @return GNUNET_YES if monotime is newer than the last received time value, GNUNET_NO if monotime is not newer.
+ */
+int
+GCP_check_and_update_monotime (struct CadetPeer *peer,
+                               struct GNUNET_TIME_AbsoluteNBO monotime)
+{
+
+  struct GNUNET_TIME_Absolute mt = GNUNET_TIME_absolute_ntoh (monotime);
+
+  if (mt.abs_value_us > *(&peer->last_connection_create.abs_value_us))
+  {
+    peer->last_connection_create = mt;
+    return GNUNET_YES;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Checking the signature for a monotime of a GNUNET_CADET_ConnectionCreateMessage.
+ *
+ * @param peer The peer that signed the monotime value.
+ * @param msg The GNUNET_CADET_ConnectionCreateMessage with the monotime value.
+ * @return GNUNET_OK if the signature is good, GNUNET_SYSERR if not.
+ */
+int
+GCP_check_monotime_sig (struct CadetPeer *peer,
+                        const struct GNUNET_CADET_ConnectionCreateMessage *msg)
+{
+  struct CadetConnectionCreatePS cp = { .purpose.purpose = htonl (
+                                          GNUNET_SIGNATURE_PURPOSE_CADET_CONNECTION_INITIATOR),
+                                        .purpose.size = htonl (sizeof(cp)),
+                                        .monotonic_time = msg->monotime};
+
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_eddsa_verify (
+        GNUNET_SIGNATURE_PURPOSE_CADET_CONNECTION_INITIATOR,
+        &cp,
+        &msg->monotime_sig,
+        &peer->pid.public_key))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/* end of gnunet-service-cadet-new_peer.c */
diff --git a/src/service/cadet/gnunet-service-cadet_peer.h b/src/service/cadet/gnunet-service-cadet_peer.h
new file mode 100644
index 000000000..4c5ad4252
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_peer.h
@@ -0,0 +1,433 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_peer.h
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_PEER_H
+#define GNUNET_SERVICE_CADET_PEER_H
+
+#include "gnunet-service-cadet.h"
+#include "gnunet_util_lib.h"
+
+
+/**
+ * Get the static string for a peer ID.
+ *
+ * @param peer Peer.
+ *
+ * @return Static string for it's ID.
+ */
+const char *
+GCP_2s (const struct CadetPeer *peer);
+
+
+/**
+ * Retrieve the CadetPeer structure associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
+ *
+ * @param peer_id Full identity of the peer.
+ * @param create #GNUNET_YES if a new peer should be created if unknown.
+ *               #GNUNET_NO to return NULL if peer is unknown.
+ * @return Existing or newly created peer structure.
+ *         NULL if unknown and not requested @a create
+ */
+struct CadetPeer *
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create);
+
+
+/**
+ * Calculate how desirable a path is for @a cp if
+ * @a cp is at offset @a off in the path.
+ *
+ * The 'desirability_table.c' program can be used to compute a list of
+ * sample outputs for different scenarios.  Basically, we score paths
+ * lower if there are many alternatives, and higher if they are
+ * shorter than average, and very high if they are much shorter than
+ * average and without many alternatives.
+ *
+ * @param cp a peer reachable via a path
+ * @param off offset of @a cp in a path
+ * @return score how useful a path is to reach @a cp,
+ *         positive scores mean path is more desirable
+ */
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off);
+
+
+/**
+ * Obtain the peer identity for a `struct CadetPeer`.
+ *
+ * @param cp our peer handle
+ * @return the peer identity
+ */
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp);
+
+
+/**
+ * Iterate over all known peers.
+ *
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
+ */
+void
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls);
+
+
+/**
+ * Count the number of known paths toward the peer.
+ *
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
+ */
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp);
+
+
+/**
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
+ *
+ * @param cp peer to drop paths to
+ */
+void
+GCP_drop_owned_paths (struct CadetPeer *cp);
+
+
+/**
+ * Peer path iterator.
+ *
+ * @param cls Closure.
+ * @param path Path itself
+ * @param off offset of the target peer in @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
+ */
+typedef int
+(*GCP_PathIterator) (void *cls,
+                     struct CadetPeerPath *path,
+                     unsigned int off);
+
+
+/**
+ * Iterate over the paths to a peer.
+ *
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
+ */
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls);
+
+/**
+ * Iterate over the paths to a peer without direct link.
+ *
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
+ */
+unsigned int
+GCP_iterate_indirect_paths (struct CadetPeer *cp,
+                            GCP_PathIterator callback,
+                            void *callback_cls);
+
+
+/**
+ * Iterate over the paths to @a peer where
+ * @a peer is at distance @a dist from us.
+ *
+ * @param cp Peer to get path info.
+ * @param dist desired distance of @a peer to us on the path
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
+ */
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls);
+
+
+/**
+ * Remove an entry from the DLL of all of the paths that this peer is on.
+ *
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
+ */
+void
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off);
+
+
+/**
+ * Add an entry to the DLL of all of the paths that this peer is on.
+ *
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
+ */
+void
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off);
+
+
+/**
+ * Get the tunnel towards a peer.
+ *
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
+ * @return Tunnel towards peer.
+ */
+struct CadetTunnel *
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create);
+
+
+/**
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
+ *
+ * @param cp the peer affected
+ * @param t the dead tunnel
+ */
+void
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t);
+
+
+/**
+ * Try adding a @a path to this @a cp.  If the peer already
+ * has plenty of paths, return NULL.
+ *
+ * @param cp peer to which the @a path leads to
+ * @param path a path looking for an owner; may not be fully initialized yet!
+ * @param off offset of @a cp in @a path
+ * @param force force attaching the path
+ * @return NULL if this peer does not care to become a new owner,
+ *         otherwise the node in the peer's path heap for the @a path.
+ */
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force);
+
+
+/**
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
+ *
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
+ */
+void
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn);
+
+
+/**
+ * Add a @a connection to this @a cp.
+ *
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
+ */
+void
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc);
+
+
+/**
+ * Remove a @a connection that went via this @a cp.
+ *
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
+ */
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc);
+
+
+/**
+ * We got a HELLO for a @a cp, remember it, and possibly
+ * trigger adequate actions (like trying to connect).
+ *
+ * @param cp the peer we got a HELLO for
+ * @param hello the HELLO to remember
+ */
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_MessageHeader *hello);
+
+
+/**
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
+ */
+void
+GCP_destroy_all_peers (void);
+
+
+/**
+ * Data structure used to track whom we have to notify about changes
+ * in our ability to transmit to a given peer.
+ *
+ * All queue managers will be given equal chance for sending messages
+ * to @a cp.  This construct this guarantees fairness for access to @a
+ * cp among the different message queues.  Each connection or route
+ * will have its respective message queue managers for each direction.
+ */
+struct GCP_MessageQueueManager;
+
+
+/**
+ * Function to call with updated message queue object.
+ *
+ * @param cls closure
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
+ */
+typedef void
+(*GCP_MessageQueueNotificationCallback)(void *cls,
+                                        int available);
+
+
+/**
+ * Start message queue change notifications.  Will create a new slot
+ * to manage the message queue to the given @a cp.
+ *
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
+ */
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls);
+
+
+/**
+ * Test if @a cp has a core-level connection
+ *
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
+ */
+int
+GCP_has_core_connection (struct CadetPeer *cp);
+
+
+/**
+ * Send the message in @a env via a @a mqm.  Must only be called at
+ * most once after the respective
+ * #GCP_MessageQueueNotificationCallback was called with `available`
+ * set to #GNUNET_YES, and not after the callback was called with
+ * `available` set to #GNUNET_NO or #GNUNET_SYSERR.
+ *
+ * @param mqm message queue manager for the transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
+ */
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
+ *
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
+ */
+void
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * Stops message queue change notifications and sends a last message.
+ * In practice, this is implemented by sending that @a last_env
+ * message immediately (if any), ignoring queue order.
+ *
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
+ */
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env);
+
+
+/**
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
+ *
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
+ */
+void
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq);
+
+/**
+ * Checking the signature for a monotime of a GNUNET_CADET_ConnectionCreateMessage.
+ *
+ * @param peer The peer that signed the monotime value.
+ * @param msg The GNUNET_CADET_ConnectionCreateMessage with the monotime value.
+ * @return GNUNET_OK if the signature is good, GNUNET_SYSERR if not.
+ */
+int
+GCP_check_monotime_sig (struct CadetPeer *peer, const struct
+                        GNUNET_CADET_ConnectionCreateMessage *msg);
+
+/**
+ * Checking if a monotime value is newer than the last monotime value received from a peer. If the time value is newer it will be stored at the peer.
+ *
+ * @param peer The peer we received a new time value from.
+ * @param monotime Time value we check against the last time value we received from a peer.
+ * @return GNUNET_YES if monotime is newer than the last received time value, GNUNET_NO if monotime is not newer.
+ */
+int
+GCP_check_and_update_monotime (struct CadetPeer *peer,
+                               struct GNUNET_TIME_AbsoluteNBO monotime);
+
+#endif
diff --git a/src/service/cadet/gnunet-service-cadet_tunnels.c b/src/service/cadet/gnunet-service-cadet_tunnels.c
new file mode 100644
index 000000000..c7e422926
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_tunnels.c
@@ -0,0 +1,3668 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2013, 2017, 2018 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file cadet/gnunet-service-cadet_tunnels.c
+ * @brief Information we track per tunnel.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * FIXME:
+ * - proper connection evaluation during connection management:
+ *   + consider quality (or quality spread?) of current connection set
+ *     when deciding how often to do maintenance
+ *   + interact with PEER to drive DHT GET/PUT operations based
+ *     on how much we like our connections
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_signatures.h"
+#include "cadet_protocol.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+
+
+#define LOG(level, ...) GNUNET_log_from (level, "cadet-tun", __VA_ARGS__)
+
+/**
+ * How often do we try to decrypt payload with unverified key
+ * material?  Used to limit CPU increase upon receiving bogus
+ * KX.
+ */
+#define MAX_UNVERIFIED_ATTEMPTS 16
+
+/**
+ * How long do we wait until tearing down an idle tunnel?
+ */
+#define IDLE_DESTROY_DELAY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_SECONDS, 90)
+
+/**
+ * How long do we wait initially before retransmitting the KX?
+ * TODO: replace by 2 RTT if/once we have connection-level RTT data!
+ */
+#define INITIAL_KX_RETRY_DELAY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MILLISECONDS, 250)
+
+/**
+ * Maximum number of skipped keys we keep in memory per tunnel.
+ */
+#define MAX_SKIPPED_KEYS 64
+
+/**
+ * Maximum number of keys (and thus ratchet steps) we are willing to
+ * skip before we decide this is either a bogus packet or a DoS-attempt.
+ */
+#define MAX_KEY_GAP 256
+
+
+/**
+ * Struct to old keys for skipped messages while advancing the Axolotl ratchet.
+ */
+struct CadetTunnelSkippedKey
+{
+  /**
+   * DLL next.
+   */
+  struct CadetTunnelSkippedKey *next;
+
+  /**
+   * DLL prev.
+   */
+  struct CadetTunnelSkippedKey *prev;
+
+  /**
+   * When was this key stored (for timeout).
+   */
+  struct GNUNET_TIME_Absolute timestamp;
+
+  /**
+   * Header key.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey HK;
+
+  /**
+   * Message key.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
+
+  /**
+   * Key number for a given HK.
+   */
+  unsigned int Kn;
+};
+
+
+/**
+ * Axolotl data, according to https://github.com/trevp/axolotl/wiki .
+ */
+struct CadetTunnelAxolotl
+{
+  /**
+   * A (double linked) list of stored message keys and associated header keys
+   * for "skipped" messages, i.e. messages that have not been
+   * received despite the reception of more recent messages, (head).
+   */
+  struct CadetTunnelSkippedKey *skipped_head;
+
+  /**
+   * Skipped messages' keys DLL, tail.
+   */
+  struct CadetTunnelSkippedKey *skipped_tail;
+
+  /**
+   * 32-byte root key which gets updated by DH ratchet.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey RK;
+
+  /**
+   * 32-byte header key (currently used for sending).
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey HKs;
+
+  /**
+   * 32-byte header key (currently used for receiving)
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey HKr;
+
+  /**
+   * 32-byte next header key (for sending), used once the
+   * ratchet advances.  We are sure that the sender has this
+   * key as well only after @e ratchet_allowed is #GNUNET_YES.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey NHKs;
+
+  /**
+   * 32-byte next header key (for receiving).  To be tried
+   * when decrypting with @e HKr fails and thus the sender
+   * may have advanced the ratchet.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey NHKr;
+
+  /**
+   * 32-byte chain keys (used for forward-secrecy) for
+   * sending messages. Updated for every message.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey CKs;
+
+  /**
+   * 32-byte chain keys (used for forward-secrecy) for
+   * receiving messages. Updated for every message. If
+   * messages are skipped, the respective derived MKs
+   * (and the current @e HKr) are kept in the @e skipped_head DLL.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey CKr;
+
+  /**
+   * ECDH for key exchange (A0 / B0).
+   */
+  struct GNUNET_CRYPTO_EcdhePrivateKey kx_0;
+
+  /**
+   * ECDH Ratchet key (our private key in the current DH).
+   */
+  struct GNUNET_CRYPTO_EcdhePrivateKey DHRs;
+
+  /**
+   * ECDH Ratchet key (other peer's public key in the current DH).
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey DHRr;
+
+  /**
+   * Last ephemeral public key received from the other peer,
+   * for duplicate detection.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey last_ephemeral;
+
+  /**
+   * Time when the current ratchet expires and a new one is triggered
+   * (if @e ratchet_allowed is #GNUNET_YES).
+   */
+  struct GNUNET_TIME_Absolute ratchet_expiration;
+
+  /**
+   * Number of elements in @a skipped_head <-> @a skipped_tail.
+   */
+  unsigned int skipped;
+
+  /**
+   * Message number (reset to 0 with each new ratchet, next message to send).
+   */
+  uint32_t Ns;
+
+  /**
+   * Message number (reset to 0 with each new ratchet, next message to recv).
+   */
+  uint32_t Nr;
+
+  /**
+   * Previous message numbers (# of msgs sent under prev ratchet)
+   */
+  uint32_t PNs;
+
+  /**
+   * True (#GNUNET_YES) if we have to send a new ratchet key in next msg.
+   */
+  int ratchet_flag;
+
+  /**
+   * True (#GNUNET_YES) if we have received a message from the
+   * other peer that uses the keys from our last ratchet step.
+   * This implies that we are again allowed to advance the ratchet,
+   * otherwise we have to wait until the other peer sees our current
+   * ephemeral key and advances first.
+   *
+   * #GNUNET_NO if we have advanced the ratched but lack any evidence
+   * that the other peer has noticed this.
+   */
+  int ratchet_allowed;
+
+  /**
+   * Number of messages received since our last ratchet advance.
+   *
+   * If this counter = 0, we cannot send a new ratchet key in the next
+   * message.
+   *
+   * If this counter > 0, we could (but don't have to) send a new key.
+   *
+   * Once the @e ratchet_counter is larger than
+   * #ratchet_messages (or @e ratchet_expiration time has past), and
+   * @e ratchet_allowed is #GNUNET_YES, we advance the ratchet.
+   */
+  unsigned int ratchet_counter;
+};
+
+
+/**
+ * Struct used to save messages in a non-ready tunnel to send once connected.
+ */
+struct CadetTunnelQueueEntry
+{
+  /**
+   * We are entries in a DLL
+   */
+  struct CadetTunnelQueueEntry *next;
+
+  /**
+   * We are entries in a DLL
+   */
+  struct CadetTunnelQueueEntry *prev;
+
+  /**
+   * Tunnel these messages belong in.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Continuation to call once sent (on the channel layer).
+   */
+  GCT_SendContinuation cont;
+
+  /**
+   * Closure for @c cont.
+   */
+  void *cont_cls;
+
+  /**
+   * Envelope of message to send follows.
+   */
+  struct GNUNET_MQ_Envelope *env;
+
+  /**
+   * Where to put the connection identifier into the payload
+   * of the message in @e env once we have it?
+   */
+  struct GNUNET_CADET_ConnectionTunnelIdentifier *cid;
+};
+
+
+/**
+ * Struct containing all information regarding a tunnel to a peer.
+ */
+struct CadetTunnel
+{
+  /**
+   * Destination of the tunnel.
+   */
+  struct CadetPeer *destination;
+
+  /**
+   * Peer's ephemeral key, to recreate @c e_key and @c d_key when own
+   * ephemeral key changes.
+   */
+  struct GNUNET_CRYPTO_EcdhePublicKey peers_ephemeral_key;
+
+  /**
+   * Encryption ("our") key. It is only "confirmed" if kx_ctx is NULL.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey e_key;
+
+  /**
+   * Decryption ("their") key. It is only "confirmed" if kx_ctx is NULL.
+   */
+  struct GNUNET_CRYPTO_SymmetricSessionKey d_key;
+
+  /**
+   * Axolotl info.
+   */
+  struct CadetTunnelAxolotl ax;
+
+  /**
+   * Unverified Axolotl info, used only if we got a fresh KX (not a
+   * KX_AUTH) while our end of the tunnel was still up.  In this case,
+   * we keep the fresh KX around but do not put it into action until
+   * we got encrypted payload that assures us of the authenticity of
+   * the KX.
+   */
+  struct CadetTunnelAxolotl *unverified_ax;
+
+  /**
+   * Task scheduled if there are no more channels using the tunnel.
+   */
+  struct GNUNET_SCHEDULER_Task *destroy_task;
+
+  /**
+   * Task to trim connections if too many are present.
+   */
+  struct GNUNET_SCHEDULER_Task *maintain_connections_task;
+
+  /**
+   * Task to send messages from queue (if possible).
+   */
+  struct GNUNET_SCHEDULER_Task *send_task;
+
+  /**
+   * Task to trigger KX.
+   */
+  struct GNUNET_SCHEDULER_Task *kx_task;
+
+  /**
+   * Tokenizer for decrypted messages.
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
+   * Dispatcher for decrypted messages only (do NOT use for sending!).
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * DLL of ready connections that are actively used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_ready_head;
+
+  /**
+   * DLL of ready connections that are actively used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_ready_tail;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_head;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_tail;
+
+  /**
+   * Channels inside this tunnel. Maps
+   * `struct GNUNET_CADET_ChannelTunnelNumber` to a `struct CadetChannel`.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
+
+  /**
+   * Channel ID for the next created channel in this tunnel.
+   */
+  struct GNUNET_CADET_ChannelTunnelNumber next_ctn;
+
+  /**
+   * Queued messages, to transmit once tunnel gets connected.
+   */
+  struct CadetTunnelQueueEntry *tq_head;
+
+  /**
+   * Queued messages, to transmit once tunnel gets connected.
+   */
+  struct CadetTunnelQueueEntry *tq_tail;
+
+  /**
+   * Identification of the connection from which we are currently processing
+   * a message. Only valid (non-NULL) during #handle_decrypted() and the
+   * handle-*()-functions called from our @e mq during that function.
+   */
+  struct CadetTConnection *current_ct;
+
+  /**
+   * How long do we wait until we retry the KX?
+   */
+  struct GNUNET_TIME_Relative kx_retry_delay;
+
+  /**
+   * When do we try the next KX?
+   */
+  struct GNUNET_TIME_Absolute next_kx_attempt;
+
+  /**
+   * Number of connections in the @e connection_ready_head DLL.
+   */
+  unsigned int num_ready_connections;
+
+  /**
+   * Number of connections in the @e connection_busy_head DLL.
+   */
+  unsigned int num_busy_connections;
+
+  /**
+   * How often have we tried and failed to decrypt a message using
+   * the unverified KX material from @e unverified_ax?  Used to
+   * stop trying after #MAX_UNVERIFIED_ATTEMPTS.
+   */
+  unsigned int unverified_attempts;
+
+  /**
+   * Number of entries in the @e tq_head DLL.
+   */
+  unsigned int tq_len;
+
+  /**
+   * State of the tunnel encryption.
+   */
+  enum CadetTunnelEState estate;
+
+  /**
+   * Force triggering KX_AUTH independent of @e estate.
+   */
+  int kx_auth_requested;
+};
+
+
+/**
+ * Am I Alice or Betty (some call her Bob), or talking to myself?
+ *
+ * @param other the other peer
+ * @return #GNUNET_YES for Alice, #GNUNET_NO for Betty, #GNUNET_SYSERR if talking to myself
+ */
+int
+GCT_alice_or_betty (const struct GNUNET_PeerIdentity *other)
+{
+  if (0 > GNUNET_memcmp (&my_full_id,
+                         other))
+    return GNUNET_YES;
+  else if (0 < GNUNET_memcmp (&my_full_id,
+                              other))
+    return GNUNET_NO;
+  else
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+}
+
+
+/**
+ * Connection @a ct is now unready, clear it's ready flag
+ * and move it from the ready DLL to the busy DLL.
+ *
+ * @param ct connection to move to unready status
+ */
+static void
+mark_connection_unready (struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = ct->t;
+
+  GNUNET_assert (GNUNET_YES == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                               t->connection_ready_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_ready_connections);
+  t->num_ready_connections--;
+  ct->is_ready = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  t->num_busy_connections++;
+}
+
+
+/**
+ * Get the static string for the peer this tunnel is directed.
+ *
+ * @param t Tunnel.
+ *
+ * @return Static string the destination peer's ID.
+ */
+const char *
+GCT_2s (const struct CadetTunnel *t)
+{
+  static char buf[64];
+
+  if (NULL == t)
+    return "Tunnel(NULL)";
+  GNUNET_snprintf (buf,
+                   sizeof(buf),
+                   "Tunnel %s",
+                   GNUNET_i2s (GCP_get_id (t->destination)));
+  return buf;
+}
+
+
+/**
+ * Get string description for tunnel encryption state.
+ *
+ * @param es Tunnel state.
+ *
+ * @return String representation.
+ */
+static const char *
+estate2s (enum CadetTunnelEState es)
+{
+  static char buf[32];
+
+  switch (es)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    return "CADET_TUNNEL_KEY_UNINITIALIZED";
+  case CADET_TUNNEL_KEY_AX_RECV:
+    return "CADET_TUNNEL_KEY_AX_RECV";
+  case CADET_TUNNEL_KEY_AX_SENT:
+    return "CADET_TUNNEL_KEY_AX_SENT";
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    return "CADET_TUNNEL_KEY_AX_SENT_AND_RECV";
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    return "CADET_TUNNEL_KEY_AX_AUTH_SENT";
+  case CADET_TUNNEL_KEY_OK:
+    return "CADET_TUNNEL_KEY_OK";
+  }
+  GNUNET_snprintf (buf,
+                   sizeof(buf),
+                   "%u (UNKNOWN STATE)",
+                   es);
+  return buf;
+}
+
+
+/**
+ * Return the peer to which this tunnel goes.
+ *
+ * @param t a tunnel
+ * @return the destination of the tunnel
+ */
+struct CadetPeer *
+GCT_get_destination (struct CadetTunnel *t)
+{
+  return t->destination;
+}
+
+
+unsigned int
+GCT_count_channels (struct CadetTunnel *t)
+{
+  return GNUNET_CONTAINER_multihashmap32_size (t->channels);
+}
+
+
+/**
+ * Lookup a channel by its @a ctn.
+ *
+ * @param t tunnel to look in
+ * @param ctn number of channel to find
+ * @return NULL if channel does not exist
+ */
+struct CadetChannel *
+lookup_channel (struct CadetTunnel *t,
+                struct GNUNET_CADET_ChannelTunnelNumber ctn)
+{
+  return GNUNET_CONTAINER_multihashmap32_get (t->channels,
+                                              ntohl (ctn.cn));
+}
+
+
+unsigned int
+GCT_count_any_connections (const struct CadetTunnel *t)
+{
+  return t->num_ready_connections + t->num_busy_connections;
+}
+
+
+/**
+ * Find first connection that is ready in the list of
+ * our connections.  Picks ready connections round-robin.
+ *
+ * @param t tunnel to search
+ * @return NULL if we have no connection that is ready
+ */
+static struct CadetTConnection *
+get_ready_connection (struct CadetTunnel *t)
+{
+  struct CadetTConnection *hd = t->connection_ready_head;
+
+  GNUNET_assert ((NULL == hd) ||
+                 (GNUNET_YES == hd->is_ready));
+  return hd;
+}
+
+
+/**
+ * Get the encryption state of a tunnel.
+ *
+ * @param t Tunnel.
+ *
+ * @return Tunnel's encryption state.
+ */
+enum CadetTunnelEState
+GCT_get_estate (struct CadetTunnel *t)
+{
+  return t->estate;
+}
+
+
+/**
+ * Called when either we have a new connection, or a new message in the
+ * queue, or some existing connection has transmission capacity.  Looks
+ * at our message queue and if there is a message, picks a connection
+ * to send it on.
+ *
+ * @param cls the `struct CadetTunnel` to process messages on
+ */
+static void
+trigger_transmissions (void *cls);
+
+
+/* ************************************** start core crypto ***************************** */
+
+
+/**
+ * Create a new Axolotl ephemeral (ratchet) key.
+ *
+ * @param ax key material to update
+ */
+static void
+new_ephemeral (struct CadetTunnelAxolotl *ax)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating new ephemeral ratchet key (DHRs)\n");
+  GNUNET_CRYPTO_ecdhe_key_create (&ax->DHRs);
+}
+
+
+/**
+ * Calculate HMAC.
+ *
+ * @param plaintext Content to HMAC.
+ * @param size Size of @c plaintext.
+ * @param iv Initialization vector for the message.
+ * @param key Key to use.
+ * @param[out] hmac Destination to store the HMAC.
+ */
+static void
+t_hmac (const void *plaintext,
+        size_t size,
+        uint32_t iv,
+        const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
+        struct GNUNET_ShortHashCode *hmac)
+{
+  static const char ctx[] = "cadet authentication key";
+  struct GNUNET_CRYPTO_AuthKey auth_key;
+  struct GNUNET_HashCode hash;
+
+  GNUNET_CRYPTO_hmac_derive_key (&auth_key,
+                                 key,
+                                 &iv, sizeof(iv),
+                                 key, sizeof(*key),
+                                 ctx, sizeof(ctx),
+                                 NULL);
+  /* Two step: GNUNET_ShortHash is only 256 bits,
+     GNUNET_HashCode is 512, so we truncate. */
+  GNUNET_CRYPTO_hmac (&auth_key,
+                      plaintext,
+                      size,
+                      &hash);
+  GNUNET_memcpy (hmac,
+                 &hash,
+                 sizeof(*hmac));
+}
+
+
+/**
+ * Perform a HMAC.
+ *
+ * @param key Key to use.
+ * @param[out] hash Resulting HMAC.
+ * @param source Source key material (data to HMAC).
+ * @param len Length of @a source.
+ */
+static void
+t_ax_hmac_hash (const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
+                struct GNUNET_HashCode *hash,
+                const void *source,
+                unsigned int len)
+{
+  static const char ctx[] = "axolotl HMAC-HASH";
+  struct GNUNET_CRYPTO_AuthKey auth_key;
+
+  GNUNET_CRYPTO_hmac_derive_key (&auth_key,
+                                 key,
+                                 ctx, sizeof(ctx),
+                                 NULL);
+  GNUNET_CRYPTO_hmac (&auth_key,
+                      source,
+                      len,
+                      hash);
+}
+
+
+/**
+ * Derive a symmetric encryption key from an HMAC-HASH.
+ *
+ * @param key Key to use for the HMAC.
+ * @param[out] out Key to generate.
+ * @param source Source key material (data to HMAC).
+ * @param len Length of @a source.
+ */
+static void
+t_hmac_derive_key (const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
+                   struct GNUNET_CRYPTO_SymmetricSessionKey *out,
+                   const void *source,
+                   unsigned int len)
+{
+  static const char ctx[] = "axolotl derive key";
+  struct GNUNET_HashCode h;
+
+  t_ax_hmac_hash (key,
+                  &h,
+                  source,
+                  len);
+  GNUNET_CRYPTO_kdf (out, sizeof(*out),
+                     ctx, sizeof(ctx),
+                     &h, sizeof(h),
+                     NULL);
+}
+
+
+/**
+ * Encrypt data with the axolotl tunnel key.
+ *
+ * @param ax key material to use.
+ * @param dst Destination with @a size bytes for the encrypted data.
+ * @param src Source of the plaintext. Can overlap with @c dst, must contain @a size bytes
+ * @param size Size of the buffers at @a src and @a dst
+ */
+static void
+t_ax_encrypt (struct CadetTunnelAxolotl *ax,
+              void *dst,
+              const void *src,
+              size_t size)
+{
+  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  size_t out_size;
+
+  ax->ratchet_counter++;
+  if ((GNUNET_YES == ax->ratchet_allowed) &&
+      ((ratchet_messages <= ax->ratchet_counter) ||
+       (0 == GNUNET_TIME_absolute_get_remaining (
+          ax->ratchet_expiration).rel_value_us)))
+  {
+    ax->ratchet_flag = GNUNET_YES;
+  }
+  if (GNUNET_YES == ax->ratchet_flag)
+  {
+    /* Advance ratchet */
+    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3];
+    struct GNUNET_HashCode dh;
+    struct GNUNET_HashCode hmac;
+    static const char ctx[] = "axolotl ratchet";
+
+    new_ephemeral (ax);
+    ax->HKs = ax->NHKs;
+
+    /* RK, NHKs, CKs = KDF( HMAC-HASH(RK, DH(DHRs, DHRr)) ) */
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
+                            &ax->DHRr,
+                            &dh);
+    t_ax_hmac_hash (&ax->RK,
+                    &hmac,
+                    &dh,
+                    sizeof(dh));
+    GNUNET_CRYPTO_kdf (keys, sizeof(keys),
+                       ctx, sizeof(ctx),
+                       &hmac, sizeof(hmac),
+                       NULL);
+    ax->RK = keys[0];
+    ax->NHKs = keys[1];
+    ax->CKs = keys[2];
+
+    ax->PNs = ax->Ns;
+    ax->Ns = 0;
+    ax->ratchet_flag = GNUNET_NO;
+    ax->ratchet_allowed = GNUNET_NO;
+    ax->ratchet_counter = 0;
+    ax->ratchet_expiration
+      = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
+                                  ratchet_time);
+  }
+
+  t_hmac_derive_key (&ax->CKs,
+                     &MK,
+                     "0",
+                     1);
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &MK,
+                                     NULL, 0,
+                                     NULL);
+
+  out_size = GNUNET_CRYPTO_symmetric_encrypt (src,
+                                              size,
+                                              &MK,
+                                              &iv,
+                                              dst);
+  GNUNET_assert (size == out_size);
+  t_hmac_derive_key (&ax->CKs,
+                     &ax->CKs,
+                     "1",
+                     1);
+}
+
+
+/**
+ * Decrypt data with the axolotl tunnel key.
+ *
+ * @param ax key material to use.
+ * @param dst Destination for the decrypted data, must contain @a size bytes.
+ * @param src Source of the ciphertext. Can overlap with @c dst, must contain @a size bytes.
+ * @param size Size of the @a src and @a dst buffers
+ */
+static void
+t_ax_decrypt (struct CadetTunnelAxolotl *ax,
+              void *dst,
+              const void *src,
+              size_t size)
+{
+  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  size_t out_size;
+
+  t_hmac_derive_key (&ax->CKr,
+                     &MK,
+                     "0",
+                     1);
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &MK,
+                                     NULL, 0,
+                                     NULL);
+  GNUNET_assert (size >= sizeof(struct GNUNET_MessageHeader));
+  out_size = GNUNET_CRYPTO_symmetric_decrypt (src,
+                                              size,
+                                              &MK,
+                                              &iv,
+                                              dst);
+  GNUNET_assert (out_size == size);
+  t_hmac_derive_key (&ax->CKr,
+                     &ax->CKr,
+                     "1",
+                     1);
+}
+
+
+/**
+ * Encrypt header with the axolotl header key.
+ *
+ * @param ax key material to use.
+ * @param[in,out] msg Message whose header to encrypt.
+ */
+static void
+t_h_encrypt (struct CadetTunnelAxolotl *ax,
+             struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+{
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  size_t out_size;
+
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &ax->HKs,
+                                     NULL, 0,
+                                     NULL);
+  out_size = GNUNET_CRYPTO_symmetric_encrypt (&msg->ax_header,
+                                              sizeof(struct
+                                                     GNUNET_CADET_AxHeader),
+                                              &ax->HKs,
+                                              &iv,
+                                              &msg->ax_header);
+  GNUNET_assert (sizeof(struct GNUNET_CADET_AxHeader) == out_size);
+}
+
+
+/**
+ * Decrypt header with the current axolotl header key.
+ *
+ * @param ax key material to use.
+ * @param src Message whose header to decrypt.
+ * @param dst Where to decrypt header to.
+ */
+static void
+t_h_decrypt (struct CadetTunnelAxolotl *ax,
+             const struct GNUNET_CADET_TunnelEncryptedMessage *src,
+             struct GNUNET_CADET_TunnelEncryptedMessage *dst)
+{
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  size_t out_size;
+
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &ax->HKr,
+                                     NULL, 0,
+                                     NULL);
+  out_size = GNUNET_CRYPTO_symmetric_decrypt (&src->ax_header.Ns,
+                                              sizeof(struct
+                                                     GNUNET_CADET_AxHeader),
+                                              &ax->HKr,
+                                              &iv,
+                                              &dst->ax_header.Ns);
+  GNUNET_assert (sizeof(struct GNUNET_CADET_AxHeader) == out_size);
+}
+
+
+/**
+ * Delete a key from the list of skipped keys.
+ *
+ * @param ax key material to delete @a key from.
+ * @param key Key to delete.
+ */
+static void
+delete_skipped_key (struct CadetTunnelAxolotl *ax,
+                    struct CadetTunnelSkippedKey *key)
+{
+  GNUNET_CONTAINER_DLL_remove (ax->skipped_head,
+                               ax->skipped_tail,
+                               key);
+  GNUNET_free (key);
+  ax->skipped--;
+}
+
+
+/**
+ * Decrypt and verify data with the appropriate tunnel key and verify that the
+ * data has not been altered since it was sent by the remote peer.
+ *
+ * @param ax key material to use.
+ * @param dst Destination for the plaintext.
+ * @param src Source of the message. Can overlap with @c dst.
+ * @param size Size of the message.
+ * @return Size of the decrypted data, -1 if an error was encountered.
+ */
+static ssize_t
+try_old_ax_keys (struct CadetTunnelAxolotl *ax,
+                 void *dst,
+                 const struct GNUNET_CADET_TunnelEncryptedMessage *src,
+                 size_t size)
+{
+  struct CadetTunnelSkippedKey *key;
+  struct GNUNET_ShortHashCode *hmac;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
+  struct GNUNET_CRYPTO_SymmetricSessionKey *valid_HK;
+  size_t esize;
+  size_t res;
+  size_t len;
+  unsigned int N;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Trying skipped keys\n");
+  hmac = &plaintext_header.hmac;
+  esize = size - sizeof(struct GNUNET_CADET_TunnelEncryptedMessage);
+
+  /* Find a correct Header Key */
+  valid_HK = NULL;
+  for (key = ax->skipped_head; NULL != key; key = key->next)
+  {
+    t_hmac (&src->ax_header,
+            sizeof(struct GNUNET_CADET_AxHeader) + esize,
+            0,
+            &key->HK,
+            hmac);
+    if (0 == GNUNET_memcmp (hmac,
+                            &src->hmac))
+    {
+      valid_HK = &key->HK;
+      break;
+    }
+  }
+  if (NULL == key)
+    return -1;
+
+  /* Should've been checked in -cadet_connection.c handle_cadet_encrypted. */
+  GNUNET_assert (size > sizeof(struct GNUNET_CADET_TunnelEncryptedMessage));
+  len = size - sizeof(struct GNUNET_CADET_TunnelEncryptedMessage);
+  GNUNET_assert (len >= sizeof(struct GNUNET_MessageHeader));
+
+  /* Decrypt header */
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &key->HK,
+                                     NULL, 0,
+                                     NULL);
+  res = GNUNET_CRYPTO_symmetric_decrypt (&src->ax_header.Ns,
+                                         sizeof(struct GNUNET_CADET_AxHeader),
+                                         &key->HK,
+                                         &iv,
+                                         &plaintext_header.ax_header.Ns);
+  GNUNET_assert (sizeof(struct GNUNET_CADET_AxHeader) == res);
+
+  /* Find the correct message key */
+  N = ntohl (plaintext_header.ax_header.Ns);
+  while ((NULL != key) &&
+         (N != key->Kn))
+    key = key->next;
+  if ((NULL == key) ||
+      (0 != GNUNET_memcmp (&key->HK,
+                           valid_HK)))
+    return -1;
+
+  /* Decrypt payload */
+  GNUNET_CRYPTO_symmetric_derive_iv (&iv,
+                                     &key->MK,
+                                     NULL,
+                                     0,
+                                     NULL);
+  res = GNUNET_CRYPTO_symmetric_decrypt (&src[1],
+                                         len,
+                                         &key->MK,
+                                         &iv,
+                                         dst);
+  delete_skipped_key (ax,
+                      key);
+  return res;
+}
+
+
+/**
+ * Delete a key from the list of skipped keys.
+ *
+ * @param ax key material to delete from.
+ * @param HKr Header Key to use.
+ */
+static void
+store_skipped_key (struct CadetTunnelAxolotl *ax,
+                   const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr)
+{
+  struct CadetTunnelSkippedKey *key;
+
+  key = GNUNET_new (struct CadetTunnelSkippedKey);
+  key->timestamp = GNUNET_TIME_absolute_get ();
+  key->Kn = ax->Nr;
+  key->HK = ax->HKr;
+  t_hmac_derive_key (&ax->CKr,
+                     &key->MK,
+                     "0",
+                     1);
+  t_hmac_derive_key (&ax->CKr,
+                     &ax->CKr,
+                     "1",
+                     1);
+  GNUNET_CONTAINER_DLL_insert (ax->skipped_head,
+                               ax->skipped_tail,
+                               key);
+  ax->skipped++;
+  ax->Nr++;
+}
+
+
+/**
+ * Stage skipped AX keys and calculate the message key.
+ * Stores each HK and MK for skipped messages.
+ *
+ * @param ax key material to use
+ * @param HKr Header key.
+ * @param Np Received meesage number.
+ * @return #GNUNET_OK if keys were stored.
+ *         #GNUNET_SYSERR if an error occurred (@a Np not expected).
+ */
+static int
+store_ax_keys (struct CadetTunnelAxolotl *ax,
+               const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr,
+               uint32_t Np)
+{
+  int gap;
+
+  gap = Np - ax->Nr;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Storing skipped keys [%u, %u)\n",
+       ax->Nr,
+       Np);
+  if (MAX_KEY_GAP < gap)
+  {
+    /* Avoid DoS (forcing peer to do more than #MAX_KEY_GAP HMAC operations) */
+    /* TODO: start new key exchange on return */
+    GNUNET_break_op (0);
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Got message %u, expected %u+\n",
+         Np,
+         ax->Nr);
+    return GNUNET_SYSERR;
+  }
+  if (0 > gap)
+  {
+    /* Delayed message: don't store keys, flag to try old keys. */
+    return GNUNET_SYSERR;
+  }
+
+  while (ax->Nr < Np)
+    store_skipped_key (ax,
+                       HKr);
+
+  while (ax->skipped > MAX_SKIPPED_KEYS)
+    delete_skipped_key (ax,
+                        ax->skipped_tail);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Decrypt and verify data with the appropriate tunnel key and verify that the
+ * data has not been altered since it was sent by the remote peer.
+ *
+ * @param ax key material to use
+ * @param dst Destination for the plaintext.
+ * @param src Source of the message. Can overlap with @c dst.
+ * @param size Size of the message.
+ * @return Size of the decrypted data, -1 if an error was encountered.
+ */
+static ssize_t
+t_ax_decrypt_and_validate (struct CadetTunnelAxolotl *ax,
+                           void *dst,
+                           const struct
+                           GNUNET_CADET_TunnelEncryptedMessage *src,
+                           size_t size)
+{
+  struct GNUNET_ShortHashCode msg_hmac;
+  struct GNUNET_HashCode hmac;
+  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
+  uint32_t Np;
+  uint32_t PNp;
+  size_t esize; /* Size of encryped payload */
+
+  esize = size - sizeof(struct GNUNET_CADET_TunnelEncryptedMessage);
+
+  /* Try current HK */
+  t_hmac (&src->ax_header,
+          sizeof(struct GNUNET_CADET_AxHeader) + esize,
+          0, &ax->HKr,
+          &msg_hmac);
+  if (0 != GNUNET_memcmp (&msg_hmac,
+                          &src->hmac))
+  {
+    static const char ctx[] = "axolotl ratchet";
+    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3];   /* RKp, NHKp, CKp */
+    struct GNUNET_CRYPTO_SymmetricSessionKey HK;
+    struct GNUNET_HashCode dh;
+    struct GNUNET_CRYPTO_EcdhePublicKey *DHRp;
+
+    /* Try Next HK */
+    t_hmac (&src->ax_header,
+            sizeof(struct GNUNET_CADET_AxHeader) + esize,
+            0,
+            &ax->NHKr,
+            &msg_hmac);
+    if (0 != GNUNET_memcmp (&msg_hmac,
+                            &src->hmac))
+    {
+      /* Try the skipped keys, if that fails, we're out of luck. */
+      return try_old_ax_keys (ax,
+                              dst,
+                              src,
+                              size);
+    }
+    HK = ax->HKr;
+    ax->HKr = ax->NHKr;
+    t_h_decrypt (ax,
+                 src,
+                 &plaintext_header);
+    Np = ntohl (plaintext_header.ax_header.Ns);
+    PNp = ntohl (plaintext_header.ax_header.PNs);
+    DHRp = &plaintext_header.ax_header.DHRs;
+    store_ax_keys (ax,
+                   &HK,
+                   PNp);
+
+    /* RKp, NHKp, CKp = KDF (HMAC-HASH (RK, DH (DHRp, DHRs))) */
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
+                            DHRp,
+                            &dh);
+    t_ax_hmac_hash (&ax->RK,
+                    &hmac,
+                    &dh, sizeof(dh));
+    GNUNET_CRYPTO_kdf (keys, sizeof(keys),
+                       ctx, sizeof(ctx),
+                       &hmac, sizeof(hmac),
+                       NULL);
+
+    /* Commit "purported" keys */
+    ax->RK = keys[0];
+    ax->NHKr = keys[1];
+    ax->CKr = keys[2];
+    ax->DHRr = *DHRp;
+    ax->Nr = 0;
+    ax->ratchet_allowed = GNUNET_YES;
+  }
+  else
+  {
+    t_h_decrypt (ax,
+                 src,
+                 &plaintext_header);
+    Np = ntohl (plaintext_header.ax_header.Ns);
+    PNp = ntohl (plaintext_header.ax_header.PNs);
+  }
+  if ((Np != ax->Nr) &&
+      (GNUNET_OK != store_ax_keys (ax,
+                                   &ax->HKr,
+                                   Np)))
+  {
+    /* Try the skipped keys, if that fails, we're out of luck. */
+    return try_old_ax_keys (ax,
+                            dst,
+                            src,
+                            size);
+  }
+
+  t_ax_decrypt (ax,
+                dst,
+                &src[1],
+                esize);
+  ax->Nr = Np + 1;
+  return esize;
+}
+
+
+/**
+ * Our tunnel became ready for the first time, notify channels
+ * that have been waiting.
+ *
+ * @param cls our tunnel, not used
+ * @param key unique ID of the channel, not used
+ * @param value the `struct CadetChannel` to notify
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+notify_tunnel_up_cb (void *cls,
+                     uint32_t key,
+                     void *value)
+{
+  struct CadetChannel *ch = value;
+
+  GCCH_tunnel_up (ch);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Change the tunnel encryption state.
+ * If the encryption state changes to OK, stop the rekey task.
+ *
+ * @param t Tunnel whose encryption state to change, or NULL.
+ * @param state New encryption state.
+ */
+void
+GCT_change_estate (struct CadetTunnel *t,
+                   enum CadetTunnelEState state)
+{
+  enum CadetTunnelEState old = t->estate;
+
+  t->estate = state;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s estate changed from %s to %s\n",
+       GCT_2s (t),
+       estate2s (old),
+       estate2s (state));
+
+  if ((CADET_TUNNEL_KEY_OK != old) &&
+      (CADET_TUNNEL_KEY_OK == t->estate))
+  {
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    /* notify all channels that have been waiting */
+    GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
+                                             &notify_tunnel_up_cb,
+                                             t);
+    if (NULL != t->send_task)
+      GNUNET_SCHEDULER_cancel (t->send_task);
+    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                             t);
+  }
+}
+
+
+/**
+ * Send a KX message.
+ *
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
+ * @param ax axolotl key context to use
+ */
+static void
+send_kx (struct CadetTunnel *t,
+         struct CadetTConnection *ct,
+         struct CadetTunnelAxolotl *ax)
+{
+  struct CadetConnection *cc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TunnelKeyExchangeMessage *msg;
+  enum GNUNET_CADET_KX_Flags flags;
+
+  if (GNUNET_YES != GCT_alice_or_betty (GCP_get_id (t->destination)))
+    return; /* only Alice may send KX */
+  if ((NULL == ct) ||
+      (GNUNET_NO == ct->is_ready))
+    ct = get_ready_connection (t);
+  if (NULL == ct)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Wanted to send %s in state %s, but no connection is ready, deferring\n",
+         GCT_2s (t),
+         estate2s (t->estate));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
+    return;
+  }
+  cc = ct->cc;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX);
+  flags = GNUNET_CADET_KX_FLAG_FORCE_REPLY; /* always for KX */
+  msg->flags = htonl (flags);
+  msg->cid = *GCC_get_id (cc);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
+                                      &msg->ephemeral_key);
+#if DEBUG_KX
+  msg->ephemeral_key_XXX = ax->kx_0;
+  msg->private_key_XXX = *my_private_key;
+#endif
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending KX message to %s with ephemeral %s on CID %s\n",
+       GCT_2s (t),
+       GNUNET_e2s (&msg->ephemeral_key),
+       GNUNET_sh2s (&msg->cid.connection_of_tunnel));
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
+                                      &msg->ratchet_key);
+  mark_connection_unready (ct);
+  t->kx_retry_delay = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
+  t->next_kx_attempt = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
+  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT);
+  else if (CADET_TUNNEL_KEY_AX_RECV == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+  GCC_transmit (cc,
+                env);
+  GNUNET_STATISTICS_update (stats,
+                            "# KX transmitted",
+                            1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Send a KX_AUTH message.
+ *
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
+ * @param ax axolotl key context to use
+ * @param force_reply Force the other peer to reply with a KX_AUTH message
+ *         (set if we would like to transmit right now, but cannot)
+ */
+static void
+send_kx_auth (struct CadetTunnel *t,
+              struct CadetTConnection *ct,
+              struct CadetTunnelAxolotl *ax,
+              int force_reply)
+{
+  struct CadetConnection *cc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg;
+  enum GNUNET_CADET_KX_Flags flags;
+
+  if ((NULL == ct) ||
+      (GNUNET_NO == ct->is_ready))
+    ct = get_ready_connection (t);
+  if (NULL == ct)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Wanted to send KX_AUTH on %s, but no connection is ready, deferring\n",
+         GCT_2s (t));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
+    t->kx_auth_requested = GNUNET_YES;   /* queue KX_AUTH independent of estate */
+    return;
+  }
+  t->kx_auth_requested = GNUNET_NO; /* clear flag */
+  cc = ct->cc;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH);
+  flags = GNUNET_CADET_KX_FLAG_NONE;
+  if (GNUNET_YES == force_reply)
+    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
+  msg->kx.flags = htonl (flags);
+  msg->kx.cid = *GCC_get_id (cc);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
+                                      &msg->kx.ephemeral_key);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
+                                      &msg->kx.ratchet_key);
+#if DEBUG_KX
+  msg->kx.ephemeral_key_XXX = ax->kx_0;
+  msg->kx.private_key_XXX = *my_private_key;
+  msg->r_ephemeral_key_XXX = ax->last_ephemeral;
+#endif
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending KX_AUTH message to %s with ephemeral %s on CID %s\n",
+       GCT_2s (t),
+       GNUNET_e2s (&msg->kx.ephemeral_key),
+       GNUNET_sh2s (&msg->kx.cid.connection_of_tunnel));
+
+  /* Compute authenticator (this is the main difference to #send_kx()) */
+  GNUNET_CRYPTO_hash (&ax->RK,
+                      sizeof(ax->RK),
+                      &msg->auth);
+  /* Compute when to be triggered again; actual job will
+     be scheduled via #connection_ready_cb() */
+  t->kx_retry_delay
+    = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
+  t->next_kx_attempt
+    = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
+
+  /* Send via cc, mark it as unready */
+  mark_connection_unready (ct);
+
+  /* Update state machine, unless we are already OK */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_AUTH_SENT);
+  GCC_transmit (cc,
+                env);
+  GNUNET_STATISTICS_update (stats,
+                            "# KX_AUTH transmitted",
+                            1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Cleanup state used by @a ax.
+ *
+ * @param ax state to free, but not memory of @a ax itself
+ */
+static void
+cleanup_ax (struct CadetTunnelAxolotl *ax)
+{
+  while (NULL != ax->skipped_head)
+    delete_skipped_key (ax,
+                        ax->skipped_head);
+  GNUNET_assert (0 == ax->skipped);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->kx_0);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->DHRs);
+}
+
+
+/**
+ * Update our Axolotl key state based on the KX data we received.
+ * Computes the new chain keys, and root keys, etc, and also checks
+ * whether this is a replay of the current chain.
+ *
+ * @param[in,out] ax chain key state to recompute
+ * @param pid peer identity of the other peer
+ * @param ephemeral_key ephemeral public key of the other peer
+ * @param ratchet_key senders next ephemeral public key
+ * @return #GNUNET_OK on success, #GNUNET_NO if the resulting
+ *       root key is already in @a ax and thus the KX is useless;
+ *       #GNUNET_SYSERR on hard errors (i.e. @a pid is #my_full_id)
+ */
+static int
+update_ax_by_kx (struct CadetTunnelAxolotl *ax,
+                 const struct GNUNET_PeerIdentity *pid,
+                 const struct GNUNET_CRYPTO_EcdhePublicKey *ephemeral_key,
+                 const struct GNUNET_CRYPTO_EcdhePublicKey *ratchet_key)
+{
+  struct GNUNET_HashCode key_material[3];
+  struct GNUNET_CRYPTO_SymmetricSessionKey keys[5];
+  const char salt[] = "CADET Axolotl salt";
+  int am_I_alice;
+
+  if (GNUNET_SYSERR == (am_I_alice = GCT_alice_or_betty (pid)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if (0 == GNUNET_memcmp (&ax->DHRr,
+                          ratchet_key))
+  {
+    GNUNET_STATISTICS_update (stats,
+                              "# Ratchet key already known",
+                              1,
+                              GNUNET_NO);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ratchet key already known. Ignoring KX.\n");
+    return GNUNET_NO;
+  }
+
+  ax->DHRr = *ratchet_key;
+  ax->last_ephemeral = *ephemeral_key;
+  /* ECDH A B0 */
+  if (GNUNET_YES == am_I_alice)
+  {
+    GNUNET_CRYPTO_eddsa_ecdh (my_private_key,      /* a */
+                              ephemeral_key,       /* B0 */
+                              &key_material[0]);
+  }
+  else
+  {
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* b0 */
+                              &pid->public_key,     /* A */
+                              &key_material[0]);
+  }
+  /* ECDH A0 B */
+  if (GNUNET_YES == am_I_alice)
+  {
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* a0 */
+                              &pid->public_key,     /* B */
+                              &key_material[1]);
+  }
+  else
+  {
+    GNUNET_CRYPTO_eddsa_ecdh (my_private_key,      /* b  */
+                              ephemeral_key,       /* A0 */
+                              &key_material[1]);
+  }
+
+  /* ECDH A0 B0 */
+  GNUNET_CRYPTO_ecc_ecdh (&ax->kx_0,              /* a0 or b0 */
+                          ephemeral_key,         /* B0 or A0 */
+                          &key_material[2]);
+  /* KDF */
+  GNUNET_CRYPTO_kdf (keys, sizeof(keys),
+                     salt, sizeof(salt),
+                     &key_material, sizeof(key_material),
+                     NULL);
+
+  if (0 == memcmp (&ax->RK,
+                   &keys[0],
+                   sizeof(ax->RK)))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Root key already known. Ignoring KX.\n");
+    GNUNET_STATISTICS_update (stats,
+                              "# Root key already known",
+                              1,
+                              GNUNET_NO);
+    return GNUNET_NO;
+  }
+
+  ax->RK = keys[0];
+  if (GNUNET_YES == am_I_alice)
+  {
+    ax->HKr = keys[1];
+    ax->NHKs = keys[2];
+    ax->NHKr = keys[3];
+    ax->CKr = keys[4];
+    ax->ratchet_flag = GNUNET_YES;
+  }
+  else
+  {
+    ax->HKs = keys[1];
+    ax->NHKr = keys[2];
+    ax->NHKs = keys[3];
+    ax->CKs = keys[4];
+    ax->ratchet_flag = GNUNET_NO;
+    ax->ratchet_expiration
+      = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
+                                  ratchet_time);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Try to redo the KX or KX_AUTH handshake, if we can.
+ *
+ * @param cls the `struct CadetTunnel` to do KX for.
+ */
+static void
+retry_kx (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetTunnelAxolotl *ax;
+
+  t->kx_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Trying to make KX progress on %s in state %s\n",
+       GCT_2s (t),
+       estate2s (t->estate));
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:   /* first attempt */
+  case CADET_TUNNEL_KEY_AX_SENT:         /* trying again */
+    send_kx (t,
+             NULL,
+             &t->ax);
+    break;
+
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+
+  case CADET_TUNNEL_KEY_OK:
+    /* Must have been the *other* peer asking us to
+       respond with a KX_AUTH. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Sending AX_AUTH in response to AX so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* Sending AX_AUTH in response to AX_AUTH */
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  GNUNET_NO);
+    break;
+  }
+}
+
+
+void
+GCT_handle_kx (struct CadetTConnection *ct,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  int ret;
+
+  GNUNET_STATISTICS_update (stats,
+                            "# KX received",
+                            1,
+                            GNUNET_NO);
+  if (GNUNET_YES ==
+      GCT_alice_or_betty (GCP_get_id (t->destination)))
+  {
+    /* Betty/Bob is not allowed to send KX! */
+    GNUNET_break_op (0);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KX message from %s with ephemeral %s from %s on connection %s\n",
+       GCT_2s (t),
+       GNUNET_e2s (&msg->ephemeral_key),
+       GNUNET_i2s (GCP_get_id (t->destination)),
+       GCC_2s (ct->cc));
+#if 1
+  if ((0 ==
+       memcmp (&t->ax.DHRr,
+               &msg->ratchet_key,
+               sizeof(msg->ratchet_key))) &&
+      (0 ==
+       memcmp (&t->ax.last_ephemeral,
+               &msg->ephemeral_key,
+               sizeof(msg->ephemeral_key))))
+
+  {
+    GNUNET_STATISTICS_update (stats,
+                              "# Duplicate KX received",
+                              1,
+                              GNUNET_NO);
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_NO);
+    return;
+  }
+#endif
+  /* We only keep ONE unverified KX around, so if there is an existing one,
+     clean it up. */
+  if (NULL != t->unverified_ax)
+  {
+    if ((0 ==
+         memcmp (&t->unverified_ax->DHRr,
+                 &msg->ratchet_key,
+                 sizeof(msg->ratchet_key))) &&
+        (0 ==
+         memcmp (&t->unverified_ax->last_ephemeral,
+                 &msg->ephemeral_key,
+                 sizeof(msg->ephemeral_key))))
+    {
+      GNUNET_STATISTICS_update (stats,
+                                "# Duplicate unverified KX received",
+                                1,
+                                GNUNET_NO);
+#if 1
+      send_kx_auth (t,
+                    ct,
+                    t->unverified_ax,
+                    GNUNET_NO);
+      return;
+#endif
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping old unverified KX state.\n");
+    GNUNET_STATISTICS_update (stats,
+                              "# Unverified KX dropped for fresh KX",
+                              1,
+                              GNUNET_NO);
+    GNUNET_break (NULL == t->unverified_ax->skipped_head);
+    memset (t->unverified_ax,
+            0,
+            sizeof(struct CadetTunnelAxolotl));
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Creating fresh unverified KX for %s\n",
+         GCT_2s (t));
+    GNUNET_STATISTICS_update (stats,
+                              "# Fresh KX setup",
+                              1,
+                              GNUNET_NO);
+    t->unverified_ax = GNUNET_new (struct CadetTunnelAxolotl);
+  }
+  /* Set as the 'current' RK/DHRr the one we are currently using,
+     so that the duplicate-detection logic of
+   #update_ax_by_kx can work. */
+  t->unverified_ax->RK = t->ax.RK;
+  t->unverified_ax->DHRr = t->ax.DHRr;
+  t->unverified_ax->DHRs = t->ax.DHRs;
+  t->unverified_ax->kx_0 = t->ax.kx_0;
+  t->unverified_attempts = 0;
+
+  /* Update 'ax' by the new key material */
+  ret = update_ax_by_kx (t->unverified_ax,
+                         GCP_get_id (t->destination),
+                         &msg->ephemeral_key,
+                         &msg->ratchet_key);
+  GNUNET_break (GNUNET_SYSERR != ret);
+  if (GNUNET_OK != ret)
+  {
+    GNUNET_STATISTICS_update (stats,
+                              "# Useless KX",
+                              1,
+                              GNUNET_NO);
+    return;   /* duplicate KX, nothing to do */
+  }
+  /* move ahead in our state machine */
+  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_RECV);
+  else if (CADET_TUNNEL_KEY_AX_SENT == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+
+  /* KX is still not done, try again our end. */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+  {
+    if (NULL != t->kx_task)
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+    t->kx_task
+      = GNUNET_SCHEDULER_add_now (&retry_kx,
+                                  t);
+  }
+}
+
+
+#if DEBUG_KX
+static void
+check_ee (const struct GNUNET_CRYPTO_EcdhePrivateKey *e1,
+          const struct GNUNET_CRYPTO_EcdhePrivateKey *e2)
+{
+  struct GNUNET_CRYPTO_EcdhePublicKey p1;
+  struct GNUNET_CRYPTO_EcdhePublicKey p2;
+  struct GNUNET_HashCode hc1;
+  struct GNUNET_HashCode hc2;
+
+  GNUNET_CRYPTO_ecdhe_key_get_public (e1,
+                                      &p1);
+  GNUNET_CRYPTO_ecdhe_key_get_public (e2,
+                                      &p2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecc_ecdh (e1,
+                                         &p2,
+                                         &hc1));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecc_ecdh (e2,
+                                         &p1,
+                                         &hc2));
+  GNUNET_break (0 == GNUNET_memcmp (&hc1,
+                                    &hc2));
+}
+
+
+static void
+check_ed (const struct GNUNET_CRYPTO_EcdhePrivateKey *e1,
+          const struct GNUNET_CRYPTO_EddsaPrivateKey *e2)
+{
+  struct GNUNET_CRYPTO_EcdhePublicKey p1;
+  struct GNUNET_CRYPTO_EddsaPublicKey p2;
+  struct GNUNET_HashCode hc1;
+  struct GNUNET_HashCode hc2;
+
+  GNUNET_CRYPTO_ecdhe_key_get_public (e1,
+                                      &p1);
+  GNUNET_CRYPTO_eddsa_key_get_public (e2,
+                                      &p2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdh_eddsa (e1,
+                                           &p2,
+                                           &hc1));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_eddsa_ecdh (e2,
+                                           &p1,
+                                           &hc2));
+  GNUNET_break (0 == GNUNET_memcmp (&hc1,
+                                    &hc2));
+}
+
+
+static void
+test_crypto_bug (const struct GNUNET_CRYPTO_EcdhePrivateKey *e1,
+                 const struct GNUNET_CRYPTO_EcdhePrivateKey *e2,
+                 const struct GNUNET_CRYPTO_EddsaPrivateKey *d1,
+                 const struct GNUNET_CRYPTO_EddsaPrivateKey *d2)
+{
+  check_ee (e1, e2);
+  check_ed (e1, d2);
+  check_ed (e2, d1);
+}
+
+
+#endif
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx_auth (struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  struct CadetTunnelAxolotl ax_tmp;
+  struct GNUNET_HashCode kx_auth;
+  int ret;
+
+  GNUNET_STATISTICS_update (stats,
+                            "# KX_AUTH received",
+                            1,
+                            GNUNET_NO);
+  if ((CADET_TUNNEL_KEY_UNINITIALIZED == t->estate) ||
+      (CADET_TUNNEL_KEY_AX_RECV == t->estate))
+  {
+    /* Confusing, we got a KX_AUTH before we even send our own
+       KX. This should not happen. We'll send our own KX ASAP anyway,
+       so let's ignore this here. */
+    GNUNET_break_op (0);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Handling KX_AUTH message from %s with ephemeral %s\n",
+       GCT_2s (t),
+       GNUNET_e2s (&msg->kx.ephemeral_key));
+  /* We do everything in ax_tmp until we've checked the authentication
+     so we don't clobber anything we care about by accident. */
+  ax_tmp = t->ax;
+
+  /* Update 'ax' by the new key material */
+  ret = update_ax_by_kx (&ax_tmp,
+                         GCP_get_id (t->destination),
+                         &msg->kx.ephemeral_key,
+                         &msg->kx.ratchet_key);
+  if (GNUNET_OK != ret)
+  {
+    if (GNUNET_NO == ret)
+      GNUNET_STATISTICS_update (stats,
+                                "# redundant KX_AUTH received",
+                                1,
+                                GNUNET_NO);
+    else
+      GNUNET_break (0);  /* connect to self!? */
+    return;
+  }
+  GNUNET_CRYPTO_hash (&ax_tmp.RK,
+                      sizeof(ax_tmp.RK),
+                      &kx_auth);
+  if (0 != GNUNET_memcmp (&kx_auth,
+                          &msg->auth))
+  {
+    /* This KX_AUTH is not using the latest KX/KX_AUTH data
+       we transmitted to the sender, refuse it, try KX again. */
+    GNUNET_STATISTICS_update (stats,
+                              "# KX_AUTH not using our last KX received (auth failure)",
+                              1,
+                              GNUNET_NO);
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "KX AUTH mismatch!\n");
+#if DEBUG_KX
+    {
+      struct GNUNET_CRYPTO_EcdhePublicKey ephemeral_key;
+
+      GNUNET_CRYPTO_ecdhe_key_get_public (&ax_tmp.kx_0,
+                                          &ephemeral_key);
+      if (0 != GNUNET_memcmp (&ephemeral_key,
+                              &msg->r_ephemeral_key_XXX))
+      {
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             "My ephemeral is %s!\n",
+             GNUNET_e2s (&ephemeral_key));
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             "Response is for ephemeral %s!\n",
+             GNUNET_e2s (&msg->r_ephemeral_key_XXX));
+      }
+      else
+      {
+        test_crypto_bug (&ax_tmp.kx_0,
+                         &msg->kx.ephemeral_key_XXX,
+                         my_private_key,
+                         &msg->kx.private_key_XXX);
+      }
+    }
+#endif
+    if (NULL == t->kx_task)
+      t->kx_task
+        = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
+                                   &retry_kx,
+                                   t);
+    return;
+  }
+  /* Yep, we're good. */
+  t->ax = ax_tmp;
+  if (NULL != t->unverified_ax)
+  {
+    /* We got some "stale" KX before, drop that. */
+    cleanup_ax (t->unverified_ax);
+    GNUNET_free (t->unverified_ax);
+    t->unverified_ax = NULL;
+  }
+
+  /* move ahead in our state machine */
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_AX_RECV:
+    /* Checked above, this is impossible. */
+    GNUNET_assert (0);
+    break;
+
+  case CADET_TUNNEL_KEY_AX_SENT:      /* This is the normal case */
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:   /* both peers started KX */
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:   /* both peers now did KX_AUTH */
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_OK);
+    break;
+
+  case CADET_TUNNEL_KEY_OK:
+    /* Did not expect another KX_AUTH, but so what, still acceptable.
+       Nothing to do here. */
+    break;
+  }
+  if (0 != (GNUNET_CADET_KX_FLAG_FORCE_REPLY & ntohl (msg->kx.flags)))
+  {
+    send_kx_auth (t,
+                  NULL,
+                  &t->ax,
+                  GNUNET_NO);
+  }
+}
+
+
+/* ************************************** end core crypto ***************************** */
+
+
+/**
+ * Compute the next free channel tunnel number for this tunnel.
+ *
+ * @param t the tunnel
+ * @return unused number that can uniquely identify a channel in the tunnel
+ */
+static struct GNUNET_CADET_ChannelTunnelNumber
+get_next_free_ctn (struct CadetTunnel *t)
+{
+#define HIGH_BIT 0x8000000
+  struct GNUNET_CADET_ChannelTunnelNumber ret;
+  uint32_t ctn;
+  int cmp;
+  uint32_t highbit;
+
+  cmp = GNUNET_memcmp (&my_full_id,
+                       GCP_get_id (GCT_get_destination (t)));
+  if (0 < cmp)
+    highbit = HIGH_BIT;
+  else if (0 > cmp)
+    highbit = 0;
+  else
+    GNUNET_assert (0); // loopback must never go here!
+  ctn = ntohl (t->next_ctn.cn);
+  while (NULL !=
+         GNUNET_CONTAINER_multihashmap32_get (t->channels,
+                                              ctn | highbit))
+  {
+    ctn = ((ctn + 1) & (~HIGH_BIT));
+  }
+  t->next_ctn.cn = htonl ((ctn + 1) & (~HIGH_BIT));
+  ret.cn = htonl (ctn | highbit);
+  return ret;
+}
+
+
+/**
+ * Add a channel to a tunnel, and notify channel that we are ready
+ * for transmission if we are already up.  Otherwise that notification
+ * will be done later in #notify_tunnel_up_cb().
+ *
+ * @param t Tunnel.
+ * @param ch Channel
+ * @return unique number identifying @a ch within @a t
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+GCT_add_channel (struct CadetTunnel *t,
+                 struct CadetChannel *ch)
+{
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
+
+  ctn = get_next_free_ctn (t);
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (t->channels,
+                                                      ntohl (ctn.cn),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Adding %s to %s with state %d\n",
+       GCCH_2s (ch),
+       GCT_2s (t),
+       t->estate);
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    /* waiting for connection to start KX */
+    break;
+
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* we're currently waiting for KX to complete */
+    break;
+
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* waiting for OTHER peer to send us data,
+       we might need to prompt more aggressively! */
+    if (NULL == t->kx_task)
+      t->kx_task
+        = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
+                                   &retry_kx,
+                                   t);
+    break;
+
+  case CADET_TUNNEL_KEY_OK:
+    /* We are ready. Tell the new channel that we are up. */
+    GCCH_tunnel_up (ch);
+    break;
+  }
+  return ctn;
+}
+
+
+/**
+ * We lost a connection, remove it from our list and clean up
+ * the connection object itself.
+ *
+ * @param ct binding of connection to tunnel of the connection that was lost.
+ */
+void
+GCT_connection_lost (struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = ct->t;
+
+  if (GNUNET_YES == ct->is_ready)
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                                 t->connection_ready_tail,
+                                 ct);
+    t->num_ready_connections--;
+  }
+  else
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                                 t->connection_busy_tail,
+                                 ct);
+    t->num_busy_connections--;
+  }
+  GNUNET_free (ct);
+}
+
+
+/**
+ * Clean up connection @a ct of a tunnel.
+ *
+ * @param cls the `struct CadetTunnel`
+ * @param ct connection to clean up
+ */
+static void
+destroy_t_connection (void *cls,
+                      struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetConnection *cc = ct->cc;
+
+  GNUNET_assert (ct->t == t);
+  GCT_connection_lost (ct);
+  GCC_destroy_without_tunnel (cc);
+}
+
+
+/**
+ * This tunnel is no longer used, destroy it.
+ *
+ * @param cls the idle tunnel
+ */
+static void
+destroy_tunnel (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetTunnelQueueEntry *tq;
+
+  t->destroy_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying idle %s\n",
+       GCT_2s (t));
+  GNUNET_assert (0 == GCT_count_channels (t));
+  GCT_iterate_connections (t,
+                           &destroy_t_connection,
+                           t);
+  GNUNET_assert (NULL == t->connection_ready_head);
+  GNUNET_assert (NULL == t->connection_busy_head);
+  while (NULL != (tq = t->tq_head))
+  {
+    if (NULL != tq->cont)
+      tq->cont (tq->cont_cls,
+                NULL);
+    GCT_send_cancel (tq);
+  }
+  GCP_drop_tunnel (t->destination,
+                   t);
+  GNUNET_CONTAINER_multihashmap32_destroy (t->channels);
+  if (NULL != t->maintain_connections_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->maintain_connections_task);
+    t->maintain_connections_task = NULL;
+  }
+  if (NULL != t->send_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->send_task);
+    t->send_task = NULL;
+  }
+  if (NULL != t->kx_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->kx_task);
+    t->kx_task = NULL;
+  }
+  GNUNET_MST_destroy (t->mst);
+  GNUNET_MQ_destroy (t->mq);
+  if (NULL != t->unverified_ax)
+  {
+    cleanup_ax (t->unverified_ax);
+    GNUNET_free (t->unverified_ax);
+  }
+  cleanup_ax (&t->ax);
+  GNUNET_assert (NULL == t->destroy_task);
+  GNUNET_free (t);
+}
+
+
+/**
+ * Remove a channel from a tunnel.
+ *
+ * @param t Tunnel.
+ * @param ch Channel
+ * @param ctn unique number identifying @a ch within @a t
+ */
+void
+GCT_remove_channel (struct CadetTunnel *t,
+                    struct CadetChannel *ch,
+                    struct GNUNET_CADET_ChannelTunnelNumber ctn)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing %s from %s\n",
+       GCCH_2s (ch),
+       GCT_2s (t));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (t->channels,
+                                                         ntohl (ctn.cn),
+                                                         ch));
+  if ((0 ==
+       GCT_count_channels (t)) &&
+      (NULL == t->destroy_task))
+  {
+    t->destroy_task
+      = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
+                                      &destroy_tunnel,
+                                      t);
+  }
+}
+
+
+/**
+ * Destroy remaining channels during shutdown.
+ *
+ * @param cls the `struct CadetTunnel` of the channel
+ * @param key key of the channel
+ * @param value the `struct CadetChannel`
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_remaining_channels (void *cls,
+                            uint32_t key,
+                            void *value)
+{
+  struct CadetChannel *ch = value;
+
+  GCCH_handle_remote_destroy (ch,
+                              NULL);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Destroys the tunnel @a t now, without delay. Used during shutdown.
+ *
+ * @param t tunnel to destroy
+ */
+void
+GCT_destroy_tunnel_now (struct CadetTunnel *t)
+{
+  GNUNET_assert (GNUNET_YES == shutting_down);
+  GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
+                                           &destroy_remaining_channels,
+                                           t);
+  GNUNET_assert (0 ==
+                 GCT_count_channels (t));
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
+  destroy_tunnel (t);
+}
+
+
+/**
+ * Send normal payload from queue in @a t via connection @a ct.
+ * Does nothing if our payload queue is empty.
+ *
+ * @param t tunnel to send data from
+ * @param ct connection to use for transmission (is ready)
+ */
+static void
+try_send_normal_payload (struct CadetTunnel *t,
+                         struct CadetTConnection *ct)
+{
+  struct CadetTunnelQueueEntry *tq;
+
+  GNUNET_assert (GNUNET_YES == ct->is_ready);
+  tq = t->tq_head;
+  if (NULL == tq)
+  {
+    /* no messages pending right now */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Not sending payload of %s on ready %s (nothing pending)\n",
+         GCT_2s (t),
+         GCC_2s (ct->cc));
+    return;
+  }
+  /* ready to send message 'tq' on tunnel 'ct' */
+  GNUNET_assert (t == tq->t);
+  GNUNET_CONTAINER_DLL_remove (t->tq_head,
+                               t->tq_tail,
+                               tq);
+  if (NULL != tq->cid)
+    *tq->cid = *GCC_get_id (ct->cc);
+  mark_connection_unready (ct);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending payload of %s on %s\n",
+       GCT_2s (t),
+       GCC_2s (ct->cc));
+  GCC_transmit (ct->cc,
+                tq->env);
+  if (NULL != tq->cont)
+    tq->cont (tq->cont_cls,
+              GCC_get_id (ct->cc));
+  GNUNET_free (tq);
+}
+
+
+/**
+ * A connection is @a is_ready for transmission.  Looks at our message
+ * queue and if there is a message, sends it out via the connection.
+ *
+ * @param cls the `struct CadetTConnection` that is @a is_ready
+ * @param is_ready #GNUNET_YES if connection are now ready,
+ *                 #GNUNET_NO if connection are no longer ready
+ */
+static void
+connection_ready_cb (void *cls,
+                     int is_ready)
+{
+  struct CadetTConnection *ct = cls;
+  struct CadetTunnel *t = ct->t;
+
+  if (GNUNET_NO == is_ready)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s no longer ready for %s\n",
+         GCC_2s (ct->cc),
+         GCT_2s (t));
+    mark_connection_unready (ct);
+    return;
+  }
+  GNUNET_assert (GNUNET_NO == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_busy_connections);
+  t->num_busy_connections--;
+  ct->is_ready = GNUNET_YES;
+  GNUNET_CONTAINER_DLL_insert_tail (t->connection_ready_head,
+                                    t->connection_ready_tail,
+                                    ct);
+  t->num_ready_connections++;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s now ready for %s in state %s\n",
+       GCC_2s (ct->cc),
+       GCT_2s (t),
+       estate2s (t->estate));
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Do not begin KX for %s if WE have no channels waiting. Retrying after %llu\n",
+         GCT_2s (t),
+         (unsigned long long) GNUNET_TIME_absolute_get_remaining (
+           t->next_kx_attempt).rel_value_us);
+    /* Do not begin KX if WE have no channels waiting! */
+    if (0 != GNUNET_TIME_absolute_get_remaining (
+          t->next_kx_attempt).rel_value_us)
+      return;   /* wait for timeout before retrying */
+    /* We are uninitialized, just transmit immediately,
+       without undue delay. */
+
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Why for %s \n",
+         GCT_2s (t));
+
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
+    if ((0 ==
+         GCT_count_channels (t)) &&
+        (NULL == t->destroy_task))
+    {
+      t->destroy_task
+        = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
+                                        &destroy_tunnel,
+                                        t);
+    }
+    break;
+
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* we're currently waiting for KX to complete, schedule job */
+    if (NULL == t->kx_task)
+      t->kx_task
+        = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
+                                   &retry_kx,
+                                   t);
+    break;
+
+  case CADET_TUNNEL_KEY_OK:
+    if (GNUNET_YES == t->kx_auth_requested)
+    {
+      if (0 != GNUNET_TIME_absolute_get_remaining (
+            t->next_kx_attempt).rel_value_us)
+        return;     /* wait for timeout */
+      if (NULL != t->kx_task)
+      {
+        GNUNET_SCHEDULER_cancel (t->kx_task);
+        t->kx_task = NULL;
+      }
+      send_kx_auth (t,
+                    ct,
+                    &t->ax,
+                    GNUNET_NO);
+      return;
+    }
+    try_send_normal_payload (t,
+                             ct);
+    break;
+  }
+}
+
+
+/**
+ * Called when either we have a new connection, or a new message in the
+ * queue, or some existing connection has transmission capacity.  Looks
+ * at our message queue and if there is a message, picks a connection
+ * to send it on.
+ *
+ * @param cls the `struct CadetTunnel` to process messages on
+ */
+static void
+trigger_transmissions (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetTConnection *ct;
+
+  t->send_task = NULL;
+  if (NULL == t->tq_head)
+    return; /* no messages pending right now */
+  ct = get_ready_connection (t);
+  if (NULL == ct)
+    return; /* no connections ready */
+  try_send_normal_payload (t,
+                           ct);
+}
+
+
+/**
+ * Closure for #evaluate_connection. Used to assemble summary information
+ * about the existing connections so we can evaluate a new path.
+ */
+struct EvaluationSummary
+{
+  /**
+   * Minimum length of any of our connections, `UINT_MAX` if we have none.
+   */
+  unsigned int min_length;
+
+  /**
+   * Maximum length of any of our connections, 0 if we have none.
+   */
+  unsigned int max_length;
+
+  /**
+   * Minimum desirability of any of our connections, UINT64_MAX if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType min_desire;
+
+  /**
+   * Maximum desirability of any of our connections, 0 if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType max_desire;
+
+  /**
+   * Path we are comparing against for #evaluate_connection, can be NULL.
+   */
+  struct CadetPeerPath *path;
+
+  /**
+   * Connection deemed the "worst" so far encountered by #evaluate_connection,
+   * NULL if we did not yet encounter any connections.
+   */
+  struct CadetTConnection *worst;
+
+  /**
+   * Numeric score of @e worst, only set if @e worst is non-NULL.
+   */
+  double worst_score;
+
+  /**
+   * Set to #GNUNET_YES if we have a connection over @e path already.
+   */
+  int duplicate;
+};
+
+
+/**
+ * Evaluate a connection, updating our summary information in @a cls about
+ * what kinds of connections we have.
+ *
+ * @param cls the `struct EvaluationSummary *` to update
+ * @param ct a connection to include in the summary
+ */
+static void
+evaluate_connection (void *cls,
+                     struct CadetTConnection *ct)
+{
+  struct EvaluationSummary *es = cls;
+  struct CadetConnection *cc = ct->cc;
+  unsigned int ct_length;
+  struct CadetPeerPath *ps;
+  const struct CadetConnectionMetrics *metrics;
+  GNUNET_CONTAINER_HeapCostType ct_desirability;
+  struct GNUNET_TIME_Relative uptime;
+  struct GNUNET_TIME_Relative last_use;
+  double score;
+  double success_rate;
+
+  ps = GCC_get_path (cc,
+                     &ct_length);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Evaluating path %s of existing %s\n",
+       GCPP_2s (ps),
+       GCC_2s (cc));
+  if (ps == es->path)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate path %s.\n",
+         GCPP_2s (es->path));
+    es->duplicate = GNUNET_YES;
+    return;
+  }
+  if (NULL != es->path)
+  {
+    int duplicate = GNUNET_YES;
+
+    for (unsigned int i = 0; i < ct_length; i++)
+    {
+      GNUNET_assert (GCPP_get_length (es->path) > i);
+      if (GCPP_get_peer_at_offset (es->path,
+                                   i) !=
+          GCPP_get_peer_at_offset (ps,
+                                   i))
+      {
+        duplicate = GNUNET_NO;
+        break;
+      }
+    }
+    if (GNUNET_YES == duplicate)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Ignoring overlapping path %s.\n",
+           GCPP_2s (es->path));
+      es->duplicate = GNUNET_YES;
+      return;
+    }
+    else
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Known path %s differs from proposed path\n",
+           GCPP_2s (ps));
+    }
+  }
+
+  ct_desirability = GCPP_get_desirability (ps);
+  metrics = GCC_get_metrics (cc);
+  uptime = GNUNET_TIME_absolute_get_duration (metrics->age);
+  last_use = GNUNET_TIME_absolute_get_duration (metrics->last_use);
+  /* We add 1.0 here to avoid division by zero. */
+  success_rate = (metrics->num_acked_transmissions + 1.0)
+                 / (metrics->num_successes + 1.0);
+  score
+    = ct_desirability
+      + 100.0 / (1.0 + ct_length) /* longer paths = better */
+      + sqrt (uptime.rel_value_us / 60000000LL) /* larger uptime = better */
+      - last_use.rel_value_us / 1000L;        /* longer idle = worse */
+  score *= success_rate;        /* weigh overall by success rate */
+
+  if ((NULL == es->worst) ||
+      (score < es->worst_score))
+  {
+    es->worst = ct;
+    es->worst_score = score;
+  }
+  es->min_length = GNUNET_MIN (es->min_length,
+                               ct_length);
+  es->max_length = GNUNET_MAX (es->max_length,
+                               ct_length);
+  es->min_desire = GNUNET_MIN (es->min_desire,
+                               ct_desirability);
+  es->max_desire = GNUNET_MAX (es->max_desire,
+                               ct_desirability);
+}
+
+
+/**
+ * Consider using the path @a p for the tunnel @a t.
+ * The tunnel destination is at offset @a off in path @a p.
+ *
+ * @param cls our tunnel
+ * @param path a path to our destination
+ * @param off offset of the destination on path @a path
+ * @return #GNUNET_YES (should keep iterating)
+ */
+static int
+consider_path_cb (void *cls,
+                  struct CadetPeerPath *path,
+                  unsigned int off)
+{
+  struct CadetTunnel *t = cls;
+  struct EvaluationSummary es;
+  struct CadetTConnection *ct;
+
+  GNUNET_assert (off < GCPP_get_length (path));
+  GNUNET_assert (GCPP_get_peer_at_offset (path,
+                                          off) == t->destination);
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = path;
+  es.duplicate = GNUNET_NO;
+  es.worst = NULL;
+
+  /* Compute evaluation summary over existing connections. */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Evaluating proposed path %s for target %s\n",
+       GCPP_2s (path),
+       GCT_2s (t));
+  /* FIXME: suspect this does not ACTUALLY iterate
+     over all existing paths, otherwise dup detection
+     should work!!! */
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if (GNUNET_YES == es.duplicate)
+    return GNUNET_YES;
+
+  /* FIXME: not sure we should really just count
+     'num_connections' here, as they may all have
+     consistently failed to connect. */
+
+  /* We iterate by increasing path length; if we have enough paths and
+     this one is more than twice as long than what we are currently
+     using, then ignore all of these super-long ones! */
+  if ((GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL) &&
+      (es.min_length * 2 < off) &&
+      (es.max_length < off))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring paths of length %u, they are way too long.\n",
+         es.min_length * 2);
+    return GNUNET_NO;
+  }
+  /* If we have enough paths and this one looks no better, ignore it. */
+  if ((GCT_count_any_connections (t) >= DESIRED_CONNECTIONS_PER_TUNNEL) &&
+      (es.min_length < GCPP_get_length (path)) &&
+      (es.min_desire > GCPP_get_desirability (path)) &&
+      (es.max_length < off))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring path (%u/%llu) to %s, got something better already.\n",
+         GCPP_get_length (path),
+         (unsigned long long) GCPP_get_desirability (path),
+         GCP_2s (t->destination));
+    return GNUNET_YES;
+  }
+
+  /* Path is interesting (better by some metric, or we don't have
+     enough paths yet). */
+  ct = GNUNET_new (struct CadetTConnection);
+  ct->created = GNUNET_TIME_absolute_get ();
+  ct->t = t;
+  ct->cc = GCC_create (t->destination,
+                       path,
+                       off,
+                       ct,
+                       &connection_ready_cb,
+                       ct);
+
+  /* FIXME: schedule job to kill connection (and path?)  if it takes
+     too long to get ready! (And track performance data on how long
+     other connections took with the tunnel!)
+     => Note: to be done within 'connection'-logic! */
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  t->num_busy_connections++;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Found interesting path %s for %s, created %s\n",
+       GCPP_2s (path),
+       GCT_2s (t),
+       GCC_2s (ct->cc));
+  return GNUNET_YES;
+}
+
+
+/**
+ * Function called to maintain the connections underlying our tunnel.
+ * Tries to maintain (incl. tear down) connections for the tunnel, and
+ * if there is a significant change, may trigger transmissions.
+ *
+ * Basically, needs to check if there are connections that perform
+ * badly, and if so eventually kill them and trigger a replacement.
+ * The strategy is to open one more connection than
+ * #DESIRED_CONNECTIONS_PER_TUNNEL, and then periodically kick out the
+ * least-performing one, and then inquire for new ones.
+ *
+ * @param cls the `struct CadetTunnel`
+ */
+static void
+maintain_connections_cb (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct GNUNET_TIME_Relative delay;
+  struct EvaluationSummary es;
+
+  t->maintain_connections_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Performing connection maintenance for %s.\n",
+       GCT_2s (t));
+
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = NULL;
+  es.worst = NULL;
+  es.duplicate = GNUNET_NO;
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if ((NULL != es.worst) &&
+      (GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL))
+  {
+    /* Clear out worst-performing connection 'es.worst'. */
+    destroy_t_connection (t,
+                          es.worst);
+  }
+
+  /* Consider additional paths */
+  (void) GCP_iterate_paths (t->destination,
+                            &consider_path_cb,
+                            t);
+
+  /* FIXME: calculate when to try again based on how well we are doing;
+     in particular, if we have to few connections, we might be able
+     to do without this (as PATHS should tell us whenever a new path
+     is available instantly; however, need to make sure this job is
+     restarted after that happens).
+     Furthermore, if the paths we do know are in a reasonably narrow
+     quality band and are plentyful, we might also consider us stabilized
+     and then reduce the frequency accordingly.  */delay = GNUNET_TIME_UNIT_MINUTES;
+  t->maintain_connections_task
+    = GNUNET_SCHEDULER_add_delayed (delay,
+                                    &maintain_connections_cb,
+                                    t);
+}
+
+
+void
+GCT_consider_path (struct CadetTunnel *t,
+                   struct CadetPeerPath *p,
+                   unsigned int off)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Considering %s for %s (offset %u)\n",
+       GCPP_2s (p),
+       GCT_2s (t),
+       off);
+  (void) consider_path_cb (t,
+                           p,
+                           off);
+}
+
+
+/**
+ * We got a keepalive. Track in statistics.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param msg  the message we received on the tunnel
+ */
+static void
+handle_plaintext_keepalive (void *cls,
+                            const struct GNUNET_MessageHeader *msg)
+{
+  struct CadetTunnel *t = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KEEPALIVE on %s\n",
+       GCT_2s (t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives received",
+                            1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Check that @a msg is well-formed.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param msg  the message we received on the tunnel
+ * @return #GNUNET_OK (any variable-size payload goes)
+ */
+static int
+check_plaintext_data (void *cls,
+                      const struct GNUNET_CADET_ChannelAppDataMessage *msg)
+{
+  return GNUNET_OK;
+}
+
+
+/**
+ * We received payload data for a channel.  Locate the channel
+ * and process the data, or return an error if the channel is unknown.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param msg the message we received on the tunnel
+ */
+static void
+handle_plaintext_data (void *cls,
+                       const struct GNUNET_CADET_ChannelAppDataMessage *msg)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (t,
+                       msg->ctn);
+  if (NULL == ch)
+  {
+    /* We don't know about such a channel, might have been destroyed on our
+       end in the meantime, or never existed. Send back a DESTROY. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received %u bytes of application data for unknown channel %u, sending DESTROY\n",
+         (unsigned int) (ntohs (msg->header.size) - sizeof(*msg)),
+         ntohl (msg->ctn.cn));
+    GCT_send_channel_destroy (t,
+                              msg->ctn);
+    return;
+  }
+  GCCH_handle_channel_plaintext_data (ch,
+                                      GCC_get_id (t->current_ct->cc),
+                                      msg);
+}
+
+
+/**
+ * We received an acknowledgement for data we sent on a channel.
+ * Locate the channel and process it, or return an error if the
+ * channel is unknown.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param ack the message we received on the tunnel
+ */
+static void
+handle_plaintext_data_ack (void *cls,
+                           const struct GNUNET_CADET_ChannelDataAckMessage *ack)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (t,
+                       ack->ctn);
+  if (NULL == ch)
+  {
+    /* We don't know about such a channel, might have been destroyed on our
+       end in the meantime, or never existed. Send back a DESTROY. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received DATA_ACK for unknown channel %u, sending DESTROY\n",
+         ntohl (ack->ctn.cn));
+    GCT_send_channel_destroy (t,
+                              ack->ctn);
+    return;
+  }
+  GCCH_handle_channel_plaintext_data_ack (ch,
+                                          GCC_get_id (t->current_ct->cc),
+                                          ack);
+}
+
+
+/**
+ * We have received a request to open a channel to a port from
+ * another peer.  Creates the incoming channel.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param copen the message we received on the tunnel
+ */
+static void
+handle_plaintext_channel_open (void *cls,
+                               const struct
+                               GNUNET_CADET_ChannelOpenMessage *copen)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetChannel *ch;
+
+  ch = GNUNET_CONTAINER_multihashmap32_get (t->channels,
+                                            ntohl (copen->ctn.cn));
+  if (NULL != ch)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received duplicate channel CHANNEL_OPEN on h_port %s from %s (%s), resending ACK\n",
+         GNUNET_h2s (&copen->h_port),
+         GCT_2s (t),
+         GCCH_2s (ch));
+    GCCH_handle_duplicate_open (ch,
+                                GCC_get_id (t->current_ct->cc));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CHANNEL_OPEN on h_port %s from %s\n",
+       GNUNET_h2s (&copen->h_port),
+       GCT_2s (t));
+  ch = GCCH_channel_incoming_new (t,
+                                  copen->ctn,
+                                  &copen->h_port,
+                                  ntohl (copen->opt));
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (t->channels,
+                                                      ntohl (copen->ctn.cn),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+}
+
+
+/**
+ * Send a DESTROY message via the tunnel.
+ *
+ * @param t the tunnel to transmit over
+ * @param ctn ID of the channel to destroy
+ */
+void
+GCT_send_channel_destroy (struct CadetTunnel *t,
+                          struct GNUNET_CADET_ChannelTunnelNumber ctn)
+{
+  struct GNUNET_CADET_ChannelDestroyMessage msg;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending DESTROY message for channel ID %u\n",
+       ntohl (ctn.cn));
+  msg.header.size = htons (sizeof(msg));
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
+  msg.reserved = htonl (0);
+  msg.ctn = ctn;
+  GCT_send (t,
+            &msg.header,
+            NULL,
+            NULL,
+            &ctn);
+}
+
+
+/**
+ * We have received confirmation from the target peer that the
+ * given channel could be established (the port is open).
+ * Tell the client.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param cm the message we received on the tunnel
+ */
+static void
+handle_plaintext_channel_open_ack (void *cls,
+                                   const struct
+                                   GNUNET_CADET_ChannelOpenAckMessage *cm)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (t,
+                       cm->ctn);
+  if (NULL == ch)
+  {
+    /* We don't know about such a channel, might have been destroyed on our
+       end in the meantime, or never existed. Send back a DESTROY. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received channel OPEN_ACK for unknown channel %u, sending DESTROY\n",
+         ntohl (cm->ctn.cn));
+    GCT_send_channel_destroy (t,
+                              cm->ctn);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received channel OPEN_ACK on channel %s from %s\n",
+       GCCH_2s (ch),
+       GCT_2s (t));
+  GCCH_handle_channel_open_ack (ch,
+                                GCC_get_id (t->current_ct->cc),
+                                &cm->port);
+}
+
+
+/**
+ * We received a message saying that a channel should be destroyed.
+ * Pass it on to the correct channel.
+ *
+ * @param cls the `struct CadetTunnel` for which we decrypted the message
+ * @param cm the message we received on the tunnel
+ */
+static void
+handle_plaintext_channel_destroy (void *cls,
+                                  const struct
+                                  GNUNET_CADET_ChannelDestroyMessage *cm)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (t,
+                       cm->ctn);
+  if (NULL == ch)
+  {
+    /* We don't know about such a channel, might have been destroyed on our
+       end in the meantime, or never existed. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received channel DESTROY for unknown channel %u. Ignoring.\n",
+         ntohl (cm->ctn.cn));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received channel DESTROY on %s from %s\n",
+       GCCH_2s (ch),
+       GCT_2s (t));
+  GCCH_handle_remote_destroy (ch,
+                              GCC_get_id (t->current_ct->cc));
+}
+
+
+/**
+ * Handles a message we decrypted, by injecting it into
+ * our message queue (which will do the dispatching).
+ *
+ * @param cls the `struct CadetTunnel` that got the message
+ * @param msg the message
+ * @return #GNUNET_OK on success (always)
+ *    #GNUNET_NO to stop further processing (no error)
+ *    #GNUNET_SYSERR to stop further processing with error
+ */
+static int
+handle_decrypted (void *cls,
+                  const struct GNUNET_MessageHeader *msg)
+{
+  struct CadetTunnel *t = cls;
+
+  GNUNET_assert (NULL != t->current_ct);
+  GNUNET_MQ_inject_message (t->mq,
+                            msg);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called if we had an error processing
+ * an incoming decrypted message.
+ *
+ * @param cls the `struct CadetTunnel`
+ * @param error error code
+ */
+static void
+decrypted_error_cb (void *cls,
+                    enum GNUNET_MQ_Error error)
+{
+  GNUNET_break_op (0);
+}
+
+
+/**
+ * Create a tunnel to @a destination.  Must only be called
+ * from within #GCP_get_tunnel().
+ *
+ * @param destination where to create the tunnel to
+ * @return new tunnel to @a destination
+ */
+struct CadetTunnel *
+GCT_create_tunnel (struct CadetPeer *destination)
+{
+  struct CadetTunnel *t = GNUNET_new (struct CadetTunnel);
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (plaintext_keepalive,
+                             GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE,
+                             struct GNUNET_MessageHeader,
+                             t),
+    GNUNET_MQ_hd_var_size (plaintext_data,
+                           GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA,
+                           struct GNUNET_CADET_ChannelAppDataMessage,
+                           t),
+    GNUNET_MQ_hd_fixed_size (plaintext_data_ack,
+                             GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK,
+                             struct GNUNET_CADET_ChannelDataAckMessage,
+                             t),
+    GNUNET_MQ_hd_fixed_size (plaintext_channel_open,
+                             GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN,
+                             struct GNUNET_CADET_ChannelOpenMessage,
+                             t),
+    GNUNET_MQ_hd_fixed_size (plaintext_channel_open_ack,
+                             GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK,
+                             struct GNUNET_CADET_ChannelOpenAckMessage,
+                             t),
+    GNUNET_MQ_hd_fixed_size (plaintext_channel_destroy,
+                             GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY,
+                             struct GNUNET_CADET_ChannelDestroyMessage,
+                             t),
+    GNUNET_MQ_handler_end ()
+  };
+
+  t->kx_retry_delay = INITIAL_KX_RETRY_DELAY;
+  new_ephemeral (&t->ax);
+  GNUNET_CRYPTO_ecdhe_key_create (&t->ax.kx_0);
+  t->destination = destination;
+  t->channels = GNUNET_CONTAINER_multihashmap32_create (8);
+  t->maintain_connections_task
+    = GNUNET_SCHEDULER_add_now (&maintain_connections_cb,
+                                t);
+  t->mq = GNUNET_MQ_queue_for_callbacks (NULL,
+                                         NULL,
+                                         NULL,
+                                         NULL,
+                                         handlers,
+                                         &decrypted_error_cb,
+                                         t);
+  t->mst = GNUNET_MST_create (&handle_decrypted,
+                              t);
+  return t;
+}
+
+
+int
+GCT_add_inbound_connection (struct CadetTunnel *t,
+                            const struct
+                            GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                            struct CadetPeerPath *path)
+{
+  struct CadetTConnection *ct;
+
+  ct = GNUNET_new (struct CadetTConnection);
+  ct->created = GNUNET_TIME_absolute_get ();
+  ct->t = t;
+  ct->cc = GCC_create_inbound (t->destination,
+                               path,
+                               ct,
+                               cid,
+                               &connection_ready_cb,
+                               ct);
+  if (NULL == ct->cc)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s refused inbound %s (duplicate)\n",
+         GCT_2s (t),
+         GCC_2s (ct->cc));
+    GNUNET_free (ct);
+    return GNUNET_SYSERR;
+  }
+  /* FIXME: schedule job to kill connection (and path?)  if it takes
+     too long to get ready! (And track performance data on how long
+     other connections took with the tunnel!)
+     => Note: to be done within 'connection'-logic! */
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  t->num_busy_connections++;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s has new %s\n",
+       GCT_2s (t),
+       GCC_2s (ct->cc));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle encrypted message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the encrypted message to decrypt
+ */
+void
+GCT_handle_encrypted (struct CadetTConnection *ct,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  uint16_t size = ntohs (msg->header.size);
+  char cbuf[size] GNUNET_ALIGN;
+  ssize_t decrypted_size;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s received %u bytes of encrypted data in state %d\n",
+       GCT_2s (t),
+       (unsigned int) size,
+       t->estate);
+
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_AX_RECV:
+    /* We did not even SEND our KX, how can the other peer
+       send us encrypted data? Must have been that we went
+       down and the other peer still things we are up.
+       Let's send it KX back. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without any KX",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
+    return;
+
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We send KX, and other peer send KX to us at the same time.
+       Neither KX is AUTH'ed, so let's try KX_AUTH this time. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without KX_AUTH",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_YES);
+    return;
+
+  case CADET_TUNNEL_KEY_AX_SENT:
+    /* We did not get the KX of the other peer, but that
+       might have been lost.  Send our KX again immediately. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without KX",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
+    return;
+
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+  /* Great, first payload, we might graduate to OK! */
+  case CADET_TUNNEL_KEY_OK:
+    /* We are up and running, all good. */
+    break;
+  }
+
+  decrypted_size = -1;
+  if (CADET_TUNNEL_KEY_OK == t->estate)
+  {
+    /* We have well-established key material available,
+       try that. (This is the common case.) */
+    decrypted_size = t_ax_decrypt_and_validate (&t->ax,
+                                                cbuf,
+                                                msg,
+                                                size);
+  }
+
+  if ((-1 == decrypted_size) &&
+      (NULL != t->unverified_ax))
+  {
+    /* We have un-authenticated KX material available. We should try
+       this as a back-up option, in case the sender crashed and
+       switched keys. */
+    decrypted_size = t_ax_decrypt_and_validate (t->unverified_ax,
+                                                cbuf,
+                                                msg,
+                                                size);
+    if (-1 != decrypted_size)
+    {
+      /* It worked! Treat this as authentication of the AX data! */
+      cleanup_ax (&t->ax);
+      t->ax = *t->unverified_ax;
+      GNUNET_free (t->unverified_ax);
+      t->unverified_ax = NULL;
+    }
+    if (CADET_TUNNEL_KEY_AX_AUTH_SENT == t->estate)
+    {
+      /* First time it worked, move tunnel into production! */
+      GCT_change_estate (t,
+                         CADET_TUNNEL_KEY_OK);
+      if (NULL != t->send_task)
+        GNUNET_SCHEDULER_cancel (t->send_task);
+      t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                               t);
+    }
+  }
+  if (NULL != t->unverified_ax)
+  {
+    /* We had unverified KX material that was useless; so increment
+       counter and eventually move to ignore it.  Note that we even do
+       this increment if we successfully decrypted with the old KX
+       material and thus didn't even both with the new one.  This is
+       the ideal case, as a malicious injection of bogus KX data
+       basically only causes us to increment a counter a few times. */t->unverified_attempts++;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Failed to decrypt message with unverified KX data %u times\n",
+         t->unverified_attempts);
+    if (t->unverified_attempts > MAX_UNVERIFIED_ATTEMPTS)
+    {
+      cleanup_ax (t->unverified_ax);
+      GNUNET_free (t->unverified_ax);
+      t->unverified_ax = NULL;
+    }
+  }
+
+  if (-1 == decrypted_size)
+  {
+    /* Decryption failed for good, complain. */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "%s failed to decrypt and validate encrypted data, retrying KX\n",
+         GCT_2s (t));
+    GNUNET_STATISTICS_update (stats,
+                              "# unable to decrypt",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
+    return;
+  }
+  GNUNET_STATISTICS_update (stats,
+                            "# decrypted bytes",
+                            decrypted_size,
+                            GNUNET_NO);
+
+  /* The MST will ultimately call #handle_decrypted() on each message. */
+  t->current_ct = ct;
+  GNUNET_break_op (GNUNET_OK ==
+                   GNUNET_MST_from_buffer (t->mst,
+                                           cbuf,
+                                           decrypted_size,
+                                           GNUNET_YES,
+                                           GNUNET_NO));
+  t->current_ct = NULL;
+}
+
+
+struct CadetTunnelQueueEntry *
+GCT_send (struct CadetTunnel *t,
+          const struct GNUNET_MessageHeader *message,
+          GCT_SendContinuation cont,
+          void *cont_cls,
+          struct GNUNET_CADET_ChannelTunnelNumber *ctn)
+{
+  struct CadetTunnelQueueEntry *tq;
+  uint16_t payload_size;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TunnelEncryptedMessage *ax_msg;
+  struct CadetChannel *ch;
+
+  if (NULL != ctn)
+  {
+    ch = lookup_channel (t,
+                         *ctn);
+    if ((NULL != ch) && GCCH_is_type_to_drop (ch, message))
+    {
+      GNUNET_break (0);
+      return NULL;
+    }
+  }
+
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  payload_size = ntohs (message->size);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Encrypting %u bytes for %s\n",
+       (unsigned int) payload_size,
+       GCT_2s (t));
+  env = GNUNET_MQ_msg_extra (ax_msg,
+                             payload_size,
+                             GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED);
+  t_ax_encrypt (&t->ax,
+                &ax_msg[1],
+                message,
+                payload_size);
+  GNUNET_STATISTICS_update (stats,
+                            "# encrypted bytes",
+                            payload_size,
+                            GNUNET_NO);
+  ax_msg->ax_header.Ns = htonl (t->ax.Ns++);
+  ax_msg->ax_header.PNs = htonl (t->ax.PNs);
+  /* FIXME: we should do this once, not once per message;
+     this is a point multiplication, and DHRs does not
+     change all the time. */
+  GNUNET_CRYPTO_ecdhe_key_get_public (&t->ax.DHRs,
+                                      &ax_msg->ax_header.DHRs);
+  t_h_encrypt (&t->ax,
+               ax_msg);
+  t_hmac (&ax_msg->ax_header,
+          sizeof(struct GNUNET_CADET_AxHeader) + payload_size,
+          0,
+          &t->ax.HKs,
+          &ax_msg->hmac);
+
+  tq = GNUNET_malloc (sizeof(*tq));
+  tq->t = t;
+  tq->env = env;
+  tq->cid = &ax_msg->cid; /* will initialize 'ax_msg->cid' once we know the connection */
+  tq->cont = cont;
+  tq->cont_cls = cont_cls;
+  GNUNET_CONTAINER_DLL_insert_tail (t->tq_head,
+                                    t->tq_tail,
+                                    tq);
+  if (NULL != t->send_task)
+    GNUNET_SCHEDULER_cancel (t->send_task);
+  t->send_task
+    = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                t);
+  return tq;
+}
+
+
+void
+GCT_send_cancel (struct CadetTunnelQueueEntry *tq)
+{
+  struct CadetTunnel *t = tq->t;
+
+  GNUNET_CONTAINER_DLL_remove (t->tq_head,
+                               t->tq_tail,
+                               tq);
+  GNUNET_MQ_discard (tq->env);
+  GNUNET_free (tq);
+}
+
+
+/**
+ * Iterate over all connections of a tunnel.
+ *
+ * @param t Tunnel whose connections to iterate.
+ * @param iter Iterator.
+ * @param iter_cls Closure for @c iter.
+ */
+void
+GCT_iterate_connections (struct CadetTunnel *t,
+                         GCT_ConnectionIterator iter,
+                         void *iter_cls)
+{
+  struct CadetTConnection *n;
+
+  for (struct CadetTConnection *ct = t->connection_ready_head;
+       NULL != ct;
+       ct = n)
+  {
+    n = ct->next;
+    iter (iter_cls,
+          ct);
+  }
+  for (struct CadetTConnection *ct = t->connection_busy_head;
+       NULL != ct;
+       ct = n)
+  {
+    n = ct->next;
+    iter (iter_cls,
+          ct);
+  }
+}
+
+
+/**
+ * Closure for #iterate_channels_cb.
+ */
+struct ChanIterCls
+{
+  /**
+   * Function to call.
+   */
+  GCT_ChannelIterator iter;
+
+  /**
+   * Closure for @e iter.
+   */
+  void *iter_cls;
+};
+
+
+/**
+ * Helper function for #GCT_iterate_channels.
+ *
+ * @param cls the `struct ChanIterCls`
+ * @param key unused
+ * @param value a `struct CadetChannel`
+ * @return #GNUNET_OK
+ */
+static int
+iterate_channels_cb (void *cls,
+                     uint32_t key,
+                     void *value)
+{
+  struct ChanIterCls *ctx = cls;
+  struct CadetChannel *ch = value;
+
+  ctx->iter (ctx->iter_cls,
+             ch);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Iterate over all channels of a tunnel.
+ *
+ * @param t Tunnel whose channels to iterate.
+ * @param iter Iterator.
+ * @param iter_cls Closure for @c iter.
+ */
+void
+GCT_iterate_channels (struct CadetTunnel *t,
+                      GCT_ChannelIterator iter,
+                      void *iter_cls)
+{
+  struct ChanIterCls ctx;
+
+  ctx.iter = iter;
+  ctx.iter_cls = iter_cls;
+  GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
+                                           &iterate_channels_cb,
+                                           &ctx);
+}
+
+
+/**
+ * Call #GCCH_debug() on a channel.
+ *
+ * @param cls points to the log level to use
+ * @param key unused
+ * @param value the `struct CadetChannel` to dump
+ * @return #GNUNET_OK (continue iteration)
+ */
+static int
+debug_channel (void *cls,
+               uint32_t key,
+               void *value)
+{
+  const enum GNUNET_ErrorType *level = cls;
+  struct CadetChannel *ch = value;
+
+  GCCH_debug (ch, *level);
+  return GNUNET_OK;
+}
+
+
+#define LOG2(level, ...) GNUNET_log_from_nocheck (level, "cadet-tun", \
+                                                  __VA_ARGS__)
+
+
+/**
+ * Log all possible info about the tunnel state.
+ *
+ * @param t Tunnel to debug.
+ * @param level Debug level to use.
+ */
+void
+GCT_debug (const struct CadetTunnel *t,
+           enum GNUNET_ErrorType level)
+{
+#if ! defined(GNUNET_CULL_LOGGING)
+  struct CadetTConnection *iter_c;
+  int do_log;
+
+  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
+                                       "cadet-tun",
+                                       __FILE__, __FUNCTION__, __LINE__);
+  if (0 == do_log)
+    return;
+
+  LOG2 (level,
+        "TTT TUNNEL TOWARDS %s in estate %s tq_len: %u #cons: %u\n",
+        GCT_2s (t),
+        estate2s (t->estate),
+        t->tq_len,
+        GCT_count_any_connections (t));
+  LOG2 (level,
+        "TTT channels:\n");
+  GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
+                                           &debug_channel,
+                                           &level);
+  LOG2 (level,
+        "TTT connections:\n");
+  for (iter_c = t->connection_ready_head; NULL != iter_c; iter_c = iter_c->next)
+    GCC_debug (iter_c->cc,
+               level);
+  for (iter_c = t->connection_busy_head; NULL != iter_c; iter_c = iter_c->next)
+    GCC_debug (iter_c->cc,
+               level);
+
+  LOG2 (level,
+        "TTT TUNNEL END\n");
+#endif
+}
+
+
+/* end of gnunet-service-cadet_tunnels.c */
diff --git a/src/service/cadet/gnunet-service-cadet_tunnels.h b/src/service/cadet/gnunet-service-cadet_tunnels.h
new file mode 100644
index 000000000..e893955a0
--- /dev/null
+++ b/src/service/cadet/gnunet-service-cadet_tunnels.h
@@ -0,0 +1,391 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/gnunet-service-cadet_tunnels.h
+ * @brief Information we track per tunnel.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_CADET_TUNNELS_H
+#define GNUNET_SERVICE_CADET_TUNNELS_H
+
+#include "gnunet-service-cadet.h"
+#include "cadet_protocol.h"
+
+
+/**
+ * How many connections would we like to have per tunnel?
+ */
+#define DESIRED_CONNECTIONS_PER_TUNNEL 3
+
+
+/**
+ * All the encryption states a tunnel can be in.
+ */
+enum CadetTunnelEState
+{
+  /**
+   * Uninitialized status, we need to send KX.  We will stay
+   * in this state until the first connection is up.
+   */
+  CADET_TUNNEL_KEY_UNINITIALIZED,
+
+  /**
+   * KX message sent, waiting for other peer's KX_AUTH.
+   */
+  CADET_TUNNEL_KEY_AX_SENT,
+
+  /**
+   * KX message received, trying to send back KX_AUTH.
+   */
+  CADET_TUNNEL_KEY_AX_RECV,
+
+  /**
+   * KX message sent and received, trying to send back KX_AUTH.
+   */
+  CADET_TUNNEL_KEY_AX_SENT_AND_RECV,
+
+  /**
+   * KX received and we sent KX_AUTH back, but we got no traffic yet,
+   * so we're waiting for either KX_AUTH or ENCRYPED traffic from
+   * the other peer.
+   *
+   * We will not yet send traffic, as this might have been a replay.
+   * The other (initiating) peer should send a CHANNEL_OPEN next
+   * anyway, and then we are in business!
+   */
+  CADET_TUNNEL_KEY_AX_AUTH_SENT,
+
+  /**
+   * Handshake completed: session key available.
+   */
+  CADET_TUNNEL_KEY_OK
+};
+
+/**
+ * Am I Alice or Betty (some call her Bob), or talking to myself?
+ *
+ * @param other the other peer
+ * @return #GNUNET_YES for Alice, #GNUNET_NO for Betty, #GNUNET_SYSERR if talking to myself
+ */
+int
+GCT_alice_or_betty (const struct GNUNET_PeerIdentity *other);
+
+/**
+ * Get the static string for the peer this tunnel is directed.
+ *
+ * @param t Tunnel.
+ *
+ * @return Static string the destination peer's ID.
+ */
+const char *
+GCT_2s (const struct CadetTunnel *t);
+
+
+/**
+ * Create a tunnel to @a destination.  Must only be called
+ * from within #GCP_get_tunnel().
+ *
+ * @param destination where to create the tunnel to
+ * @return new tunnel to @a destination
+ */
+struct CadetTunnel *
+GCT_create_tunnel (struct CadetPeer *destination);
+
+
+/**
+ * Destroys the tunnel @a t now, without delay. Used during shutdown.
+ *
+ * @param t tunnel to destroy
+ */
+void
+GCT_destroy_tunnel_now (struct CadetTunnel *t);
+
+
+/**
+ * Add a @a connection to the @a tunnel.
+ *
+ * @param t a tunnel
+ * @param cid connection identifier to use for the connection
+ * @param path path to use for the connection
+ * @return #GNUNET_OK on success,
+ *         #GNUNET_SYSERR on failure (duplicate connection)
+ */
+int
+GCT_add_inbound_connection (struct CadetTunnel *t,
+                            const struct
+                            GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                            struct CadetPeerPath *path);
+
+
+/**
+ * We lost a connection, remove it from our list and clean up
+ * the connection object itself.
+ *
+ * @param ct binding of connection to tunnel of the connection that was lost.
+ */
+void
+GCT_connection_lost (struct CadetTConnection *ct);
+
+
+/**
+ * Return the peer to which this tunnel goes.
+ *
+ * @param t a tunnel
+ * @return the destination of the tunnel
+ */
+struct CadetPeer *
+GCT_get_destination (struct CadetTunnel *t);
+
+
+/**
+ * Consider using the path @a p for the tunnel @a t.
+ * The tunnel destination is at offset @a off in path @a p.
+ *
+ * @param t our tunnel
+ * @param p a path to our destination
+ * @param off offset of the destination on path @a path
+ */
+void
+GCT_consider_path (struct CadetTunnel *t,
+                   struct CadetPeerPath *p,
+                   unsigned int off);
+
+
+/**
+ * Add a channel to a tunnel.
+ *
+ * @param t Tunnel.
+ * @param ch Channel
+ * @return unique number identifying @a ch within @a t
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+GCT_add_channel (struct CadetTunnel *t,
+                 struct CadetChannel *ch);
+
+
+/**
+ * Remove a channel from a tunnel.
+ *
+ * @param t Tunnel.
+ * @param ch Channel
+ * @param ctn unique number identifying @a ch within @a t
+ */
+void
+GCT_remove_channel (struct CadetTunnel *t,
+                    struct CadetChannel *ch,
+                    struct GNUNET_CADET_ChannelTunnelNumber ctn);
+
+
+/**
+ * Send a DESTROY message via the tunnel.
+ *
+ * @param t the tunnel to transmit over
+ * @param ctn ID of the channel to destroy
+ */
+void
+GCT_send_channel_destroy (struct CadetTunnel *t,
+                          struct GNUNET_CADET_ChannelTunnelNumber ctn);
+
+
+/**
+ * Function called when a transmission requested using #GCT_send is done.
+ *
+ * @param cls closure
+ * @param ctn identifier of the connection used for transmission, NULL if
+ *            the transmission failed (to be used to match ACKs to the
+ *            respective connection for connection performance evaluation)
+ */
+typedef void
+(*GCT_SendContinuation)(void *cls,
+                        const struct
+                        GNUNET_CADET_ConnectionTunnelIdentifier *cid);
+
+
+/**
+ * Sends an already built message on a tunnel, encrypting it and
+ * choosing the best connection if not provided.
+ *
+ * @param message Message to send. Function modifies it.
+ * @param t Tunnel on which this message is transmitted.
+ * @param cont Continuation to call once message is really sent.
+ * @param cont_cls Closure for @c cont.
+ * @return Handle to cancel message.
+ */
+struct CadetTunnelQueueEntry *
+GCT_send (struct CadetTunnel *t,
+          const struct GNUNET_MessageHeader *message,
+          GCT_SendContinuation cont,
+          void *cont_cls,
+          struct GNUNET_CADET_ChannelTunnelNumber *ctn);
+
+
+/**
+ * Cancel a previously sent message while it's in the queue.
+ *
+ * ONLY can be called before the continuation given to the send
+ * function is called. Once the continuation is called, the message is
+ * no longer in the queue!
+ *
+ * @param q Handle to the queue entry to cancel.
+ */
+void
+GCT_send_cancel (struct CadetTunnelQueueEntry *q);
+
+
+/**
+ * Returns the number of channels using a tunnel.
+ *
+ * @param t Tunnel in question.
+ * @return Number of channels using the tunnel.
+ */
+unsigned int
+GCT_count_channels (struct CadetTunnel *t);
+
+
+/**
+ * Counts the number of connections created for a tunnel,
+ * including busy connections.
+ *
+ * @param t Tunnel to be counted.
+ * @return Number of connections created for the tunnel.
+ */
+unsigned int
+GCT_count_any_connections (const struct CadetTunnel *t);
+
+
+/**
+ * Iterator over connections.
+ *
+ * @param cls closure
+ * @param ct one of the connections
+ */
+typedef void
+(*GCT_ConnectionIterator) (void *cls,
+                           struct CadetTConnection *ct);
+
+
+/**
+ * Iterate over all connections of a tunnel.
+ *
+ * @param t Tunnel whose connections to iterate.
+ * @param iter Iterator.
+ * @param iter_cls Closure for @c iter.
+ */
+void
+GCT_iterate_connections (struct CadetTunnel *t,
+                         GCT_ConnectionIterator iter,
+                         void *iter_cls);
+
+
+/**
+ * Iterator over channels.
+ *
+ * @param cls closure
+ * @param ch one of the channels
+ */
+typedef void
+(*GCT_ChannelIterator) (void *cls,
+                        struct CadetChannel *ch);
+
+
+/**
+ * Iterate over all channels of a tunnel.
+ *
+ * @param t Tunnel whose channels to iterate.
+ * @param iter Iterator.
+ * @param iter_cls Closure for @c iter.
+ */
+void
+GCT_iterate_channels (struct CadetTunnel *t,
+                      GCT_ChannelIterator iter,
+                      void *iter_cls);
+
+
+/**
+ * Get the encryption state of a tunnel.
+ *
+ * @param t Tunnel.
+ *
+ * @return Tunnel's encryption state.
+ */
+enum CadetTunnelEState
+GCT_get_estate (struct CadetTunnel *t);
+
+/**
+ * Change the tunnel encryption state.
+ * If the encryption state changes to OK, stop the rekey task.
+ *
+ * @param t Tunnel whose encryption state to change, or NULL.
+ * @param state New encryption state.
+ */
+void
+GCT_change_estate (struct CadetTunnel *t,
+                   enum CadetTunnelEState state);
+
+/**
+ * Handle KX message that lacks authentication (and which will thus
+ * only be considered authenticated after we respond with our own
+ * KX_AUTH and finally successfully decrypt the payload).
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx (struct CadetTConnection *ct,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx_auth (struct CadetTConnection *ct,
+                    const struct
+                    GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
+
+
+/**
+ * Handle encrypted message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the encrypted message to decrypt
+ */
+void
+GCT_handle_encrypted (struct CadetTConnection *ct,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
+
+
+/**
+ * Log all possible info about the tunnel state.
+ *
+ * @param t Tunnel to debug.
+ * @param level Debug level to use.
+ */
+void
+GCT_debug (const struct CadetTunnel *t,
+           enum GNUNET_ErrorType level);
+
+
+#endif
diff --git a/src/service/cadet/loopcheck.sh b/src/service/cadet/loopcheck.sh
new file mode 100755
index 000000000..9b867e95b
--- /dev/null
+++ b/src/service/cadet/loopcheck.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# This script is in the public domain
+# POSIX shell solution for named pipes and pipestatus,
+# http://shell.cfajohnson.com/cus-faq-2.html#Q11
+# run() {
+#     j=1
+#     while eval "\${pipestatus_$j+:} false"; do
+#         unset pipestatus_$j
+#         j=$(($j+1))
+#     done
+#     j=1 com= k=1 l=
+#     for a; do
+#         if [ "x$a" = 'x|' ]; then
+#             com="$com { $l "'3>&-
+#                          echo "pipestatus_'$j'=$?" >&3
+#                        } 4>&- |'
+#             j=$(($j+1)) l=
+#         else
+#             l="$l \"\$$k\""
+#         fi
+#         k=$(($k+1))
+#     done
+#     com="$com $l"' 3>&- >&4 4>&-
+#                     echo "pipestatus_'$j'=$?"'
+#     exec 4>&1
+#     eval "$(exec 3>&1; eval "$com")"
+#     exec 4>&-
+#     j=1
+#     while eval "\${pipestatus_$j+:} false"; do
+#         eval "[ \$pipestatus_$j -eq 0 ]" || return 1
+#         j=$(($j+1))
+#     done
+#     return 0
+# }
+
+# # https://mywiki.wooledge.org/Bashism has another solution:
+# # mkfifo fifo; command2 <fifo & command1 >fifo; echo "$?"
+
+while true; do
+    if [ "$1" = "" ]; then
+        echo All
+        taskset 1 make check || break;
+    else
+        echo One
+        LOGFILE="test_`date "+%m.%d-%H:%M:%S"`.log"
+        taskset 01 $1 2>&1 | tee $LOGFILE | grep -v DEBUG;
+        # TODO: Replace $PIPESTATUS with more portable code
+        if [ "${PIPESTATUS[0]}" != "0" ]; then
+            echo "Failed";
+            date;
+            break;
+        fi
+    fi
+    grep cadet test_*.log | grep -B 10 ERROR && break
+    grep cadet test_*.log | grep -B 10 Assert && break
+    ls core* > /dev/null 2>&1 && break
+done
diff --git a/src/service/cadet/meson.build b/src/service/cadet/meson.build
new file mode 100644
index 000000000..0f465e0e2
--- /dev/null
+++ b/src/service/cadet/meson.build
@@ -0,0 +1,72 @@
+libgnunetcadet_src = ['cadet_api.c',
+                      'cadet_api_drop_message.c',
+                      'cadet_api_get_channel.c',
+                      'cadet_api_get_path.c',
+                      'cadet_api_list_peers.c',
+                      'cadet_api_list_tunnels.c',
+                      'cadet_api_helper.c']
+
+gnunetcadet_src = ['gnunet-cadet.c']
+gnunetservicecadet_src = ['gnunet-service-cadet.c',
+                          'gnunet-service-cadet_channel.c',
+                          'gnunet-service-cadet_connection.c',
+                          'gnunet-service-cadet_core.c',
+                          'gnunet-service-cadet_dht.c',
+                          'gnunet-service-cadet_hello.c',
+                          'gnunet-service-cadet_tunnels.c',
+                          'gnunet-service-cadet_paths.c',
+                          'gnunet-service-cadet_peer.c']
+
+configure_file(input : 'cadet.conf.in',
+               output : 'cadet.conf',
+               configuration : cdata,
+               install: true,
+               install_dir: pkgcfgdir)
+
+if get_option('monolith')
+  foreach p : libgnunetcadet_src + gnunetservicecadet_src
+    gnunet_src += 'cadet/' + p
+  endforeach
+endif
+
+libgnunetcadet = library('gnunetcadet',
+          libgnunetcadet_src,
+          dependencies: libgnunetutil_dep,
+          include_directories: [incdir, configuration_inc],
+          install: true,
+          soversion: '7',
+          version: '7.0.0',
+          install_dir: get_option('libdir'))
+libgnunetcadet_dep = declare_dependency(link_with : libgnunetcadet)
+pkg.generate(libgnunetcadet, url: 'https://www.gnunet.org',
+             description : 'API for multicast and multi-hop routing between GNUnet peers')
+
+executable ('gnunet-service-cadet',
+            gnunetservicecadet_src,
+            dependencies: [libgnunetcadet_dep,
+                           libgnunetutil_dep,
+                           m_dep,
+                           libgnunetcore_dep,
+                           libgnunetdht_dep,
+                           libgnunetstatistics_dep,
+                           libgnunettransportapplication_dep,
+                           libgnunetpeerstore_dep,
+                           libgnunethello_dep,
+                           libgnunetblock_dep],
+            include_directories: [incdir, configuration_inc],
+            install: true,
+            install_dir: get_option('libdir') / 'gnunet' / 'libexec')
+
+testcadetlocalmq = executable ('test_cadet_local_mq',
+            ['test_cadet_local_mq.c'],
+            dependencies: [libgnunetcadet_dep,
+                           libgnunettesting_dep,
+                           libgnunetutil_dep],
+            include_directories: [incdir, configuration_inc],
+            install: false)
+configure_file(input : 'test_cadet.conf',
+               output : 'test_cadet.conf',
+               configuration : cdata,
+               install: false)
+test('test_cadet_local_mq', testcadetlocalmq, workdir: meson.current_build_dir(), suite: 'cadet')
+
diff --git a/src/service/cadet/profiler.conf b/src/service/cadet/profiler.conf
new file mode 100644
index 000000000..7ce283870
--- /dev/null
+++ b/src/service/cadet/profiler.conf
@@ -0,0 +1,19 @@
+@INLINE@ test_cadet.conf
+
+[testbed]
+OVERLAY_TOPOLOGY = RANDOM
+OVERLAY_RANDOM_LINKS = %LINKS%
+MAX_PARALLEL_SERVICE_CONNECTIONS=4000
+SETUP_TIMEOUT = 60 m
+
+[transport]
+#MANIPULATE_DELAY_IN = 50 ms
+MANIPULATE_DELAY_OUT = 10 ms
+
+[cadet]
+REFRESH_CONNECTION_TIME = 1 h
+DISABLE_TRY_CONNECT = YES
+ID_ANNOUNCE_TIME = 5 s
+
+[dht]
+FORCE_NSE = %NSE%
diff --git a/src/service/cadet/run_profiler.sh b/src/service/cadet/run_profiler.sh
new file mode 100755
index 000000000..4273dcd27
--- /dev/null
+++ b/src/service/cadet/run_profiler.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+if [ "$#" -lt "3" ]; then
+    echo "usage: $0 ROUND_TIME PEERS PINGING_PEERS";
+    echo "example: $0 30s 16 1";
+    exit 1;
+fi
+
+ROUNDTIME=$1
+PEERS=$2
+PINGS=$3
+
+if [ $PEERS -eq 1 ]; then
+    echo "cannot run 1 peer";
+    exit 1;
+fi
+
+LINKS=`echo "l($PEERS) * l($PEERS) * $PEERS / 2" | bc -l`
+LINKS=`printf "%.0f" $LINKS`
+NSE=`echo "l($PEERS)/l(2)" | bc -l`
+echo "using $PEERS peers, $LINKS links";
+    
+sed -e "s/%LINKS%/$LINKS/;s/%NSE%/$NSE/" profiler.conf > .profiler.conf
+
+./gnunet-cadet-profiler $ROUNDTIME $PEERS $PINGS $4 2>&1 | tee log | grep -v DEBUG
diff --git a/src/service/cadet/small.dat b/src/service/cadet/small.dat
new file mode 100644
index 000000000..c3805ee80
--- /dev/null
+++ b/src/service/cadet/small.dat
@@ -0,0 +1,21 @@
+16
+1:2
+1:9
+2:3
+3:4
+3:11
+4:5
+5:6
+5:13
+6:7
+7:8
+7:15
+8:9
+9:10
+10:11
+11:12
+12:13
+13:14
+14:15
+15:16
+16:1
diff --git a/src/service/cadet/test_cadet.conf b/src/service/cadet/test_cadet.conf
new file mode 100644
index 000000000..79c86bb59
--- /dev/null
+++ b/src/service/cadet/test_cadet.conf
@@ -0,0 +1,109 @@
+[testbed]
+HOSTNAME = localhost
+OVERLAY_TOPOLOGY = LINE
+#PREFIX = xterm -geometry 100x85 -T peer1 -e libtool --mode=execute gdb --args
+
+[cadet]
+#BINARY = gnunet-service-cadet-enc
+#PREFIX = valgrind --leak-check=full
+#PREFIX = xterm -geometry 100x85 -T peer1 -e gdb --args
+REFRESH_CONNECTION_TIME = 1 s
+ID_ANNOUNCE_TIME = 5 s
+CONNECT_TIMEOUT = 30 s
+DEFAULT_TTL = 16
+DHT_REPLICATION_LEVEL = 10
+MAX_TUNNELS = 10
+MAX_CONNECTIONS = 10
+MAX_MSGS_QUEUE = 20
+DISABLE_TRY_CONNECT = YES
+REKEY_PERIOD = 2 s
+
+[dht]
+DISABLE_TRY_CONNECT = YES
+FORCE_NSE = 3
+
+[dhtcache]
+QUOTA = 1 MB
+DATABASE = heap
+
+[transport]
+#PLUGINS = udp
+PLUGINS = tcp
+NEIGHBOUR_LIMIT = 50
+#MANIPULATE_DELAY_IN = 10 ms
+#MANIPULATE_DELAY_OUT = 10 ms
+
+[nat]
+ENABLE_UPNP = NO
+
+[ats]
+# Network specific inbound/outbound quotas
+UNSPECIFIED_QUOTA_IN = unlimited
+UNSPECIFIED_QUOTA_OUT = unlimited
+# LOOPBACK
+LOOPBACK_QUOTA_IN = unlimited
+LOOPBACK_QUOTA_OUT = unlimited
+# LAN
+LAN_QUOTA_IN = unlimited
+LAN_QUOTA_OUT = unlimited
+#WAN
+WAN_QUOTA_OUT = unlimited
+WAN_QUOTA_IN = unlimited
+# WLAN
+WLAN_QUOTA_IN = unlimited
+WLAN_QUOTA_OUT = unlimited
+# BLUETOOTH
+BLUETOOTH_QUOTA_IN = unlimited
+BLUETOOTH_QUOTA_OUT = unlimited
+
+[core]
+USE_EPHEMERAL_KEYS = NO
+
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/test-cadet/
+
+[peerinfo]
+NO_IO = YES
+
+[nse]
+WORKBITS = 2
+
+[hostlist]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[fs]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[vpn]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[revocation]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[gns]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[namestore]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[namecache]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[topology]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[rps]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
+
+[rest]
+IMMEDIATE_START = NO
+START_ON_DEMAND = NO
diff --git a/src/service/cadet/test_cadet_local_mq.c b/src/service/cadet/test_cadet_local_mq.c
new file mode 100644
index 000000000..6f75dfd1f
--- /dev/null
+++ b/src/service/cadet/test_cadet_local_mq.c
@@ -0,0 +1,338 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file cadet/test_cadet_local_mq.c
+ * @brief test cadet local: test of cadet channels with just one peer
+ * @author Bartlomiej Polot
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_dht_service.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_cadet_service.h"
+
+#define TEST_MESSAGE_TYPE 1
+#define TEST_PORT_ID 1
+
+/**
+ * Test message structure.
+ */
+struct GNUNET_CADET_TestMsg
+{
+  /**
+   * Type: #TEST_MESSAGE_TYPE
+   *
+   * Size: sizeof(struct GNUNET_CADET_TestMsg)
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Test payload.
+   */
+  uint64_t payload;
+};
+
+struct GNUNET_TESTING_Peer *me;
+
+static struct GNUNET_CADET_Handle *cadet_peer_1;
+
+static struct GNUNET_CADET_Handle *cadet_peer_2;
+
+static struct GNUNET_CADET_Channel *ch;
+
+static int result = GNUNET_OK;
+
+static int got_data = GNUNET_NO;
+
+static struct GNUNET_SCHEDULER_Task *abort_task;
+
+static struct GNUNET_SCHEDULER_Task *connect_task;
+
+
+/**
+ * Connect to other client and send data
+ *
+ * @param cls Closue (unused).
+ */
+static void
+do_connect (void *cls);
+
+
+/**
+ * Shutdown nicely
+ */
+static void
+do_shutdown (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "shutdown\n");
+  if (NULL != abort_task)
+  {
+    GNUNET_SCHEDULER_cancel (abort_task);
+    abort_task = NULL;
+  }
+  if (NULL != ch)
+  {
+    GNUNET_CADET_channel_destroy (ch);
+    ch = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Disconnect client 1\n");
+  if (NULL != cadet_peer_1)
+  {
+    GNUNET_CADET_disconnect (cadet_peer_1);
+    cadet_peer_1 = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Disconnect client 2\n");
+  if (NULL != cadet_peer_2)
+  {
+    GNUNET_CADET_disconnect (cadet_peer_2);
+    cadet_peer_2 = NULL;
+  }
+  if (NULL != connect_task)
+  {
+    GNUNET_SCHEDULER_cancel (connect_task);
+    connect_task = NULL;
+  }
+}
+
+
+/**
+ * Something went wrong and timed out. Kill everything and set error flag
+ */
+static void
+do_abort (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "ABORT from line %ld\n",
+              (long) cls);
+  result = GNUNET_SYSERR;
+  abort_task = NULL;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Method called whenever a peer connects to a port in MQ-based CADET.
+ *
+ * @param cls Closure from #GNUNET_CADET_open_port.
+ * @param channel New handle to the channel.
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
+ */
+static void *
+connected (void *cls,
+           struct GNUNET_CADET_Channel *channel,
+           const struct GNUNET_PeerIdentity *source)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "connected %s, cls: %p\n",
+              GNUNET_i2s (source),
+              cls);
+  return channel;
+}
+
+
+/**
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
+ *
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ */
+static void
+disconnected (void *cls,
+              const struct GNUNET_CADET_Channel *channel)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "disconnected channel %p, cls: %p\n",
+              channel, cls);
+  if (channel == ch)
+    ch = NULL;
+}
+
+
+/**
+ * Handle test data
+ *
+ * @param h     The cadet handle
+ * @param msg   A message with the details of the new incoming channel
+ */
+static void
+handle_data_received (void *cls,
+                      const struct GNUNET_CADET_TestMsg *msg)
+{
+  uint64_t payload;
+
+  payload = GNUNET_ntohll (msg->payload);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Data callback payload %llu with cls: %p! Shutting down.\n",
+              (unsigned long long) payload,
+              cls);
+  GNUNET_assert (42 == payload);
+  got_data = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Signature of the main function of a task.
+ *
+ * @param cls Closure (unused).
+ */
+static void
+message_sent (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "message sent\n");
+}
+
+
+/**
+ * Connect to other client and send data
+ *
+ * @param cls Closure (unused).
+ */
+static void
+do_connect (void *cls)
+{
+  struct GNUNET_PeerIdentity id;
+  struct GNUNET_MQ_Handle *mq;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TestMsg *msg;
+
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (data_received,
+                             TEST_MESSAGE_TYPE,
+                             struct GNUNET_CADET_TestMsg,
+                             cadet_peer_1),
+    GNUNET_MQ_handler_end ()
+  };
+
+  connect_task = NULL;
+  GNUNET_TESTING_peer_get_identity (me, &id);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "creating channel\n");
+  ch = GNUNET_CADET_channel_create (cadet_peer_1,  /* cadet handle */
+                                    NULL,         /* channel cls */
+                                    &id,          /* destination */
+                                    GC_u2h (TEST_MESSAGE_TYPE), /* port */
+                                    NULL,          /* window change */
+                                    &disconnected, /* disconnect handler */
+                                    handlers       /* traffic handlers */
+                                    );
+  env = GNUNET_MQ_msg (msg, TEST_MESSAGE_TYPE);
+  msg->payload = GNUNET_htonll (42);
+  mq = GNUNET_CADET_get_mq (ch);
+  GNUNET_MQ_notify_sent (env, &message_sent, NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "sending message\n");
+  GNUNET_MQ_send (mq, env);
+}
+
+
+/**
+ * Initialize framework and start test
+ *
+ * @param cls Closure (unused).
+ * @param cfg Configuration handle.
+ * @param peer Testing peer handle.
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (data_received,
+                             TEST_MESSAGE_TYPE,
+                             struct GNUNET_CADET_TestMsg,
+                             cadet_peer_2),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_TIME_Relative delay;
+
+  me = peer;
+  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
+                                 NULL);
+  delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15);
+  abort_task = GNUNET_SCHEDULER_add_delayed (delay,
+                                             &do_abort,
+                                             (void *) (long) __LINE__);
+  cadet_peer_1 = GNUNET_CADET_connect (cfg);
+  cadet_peer_2 = GNUNET_CADET_connect (cfg);
+
+  if ((NULL == cadet_peer_1) ||
+      (NULL == cadet_peer_2))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Couldn't connect to cadet\n");
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 1: %p\n", cadet_peer_1);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 2: %p\n", cadet_peer_2);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handlers 2: %p\n", handlers);
+  GNUNET_CADET_open_port (cadet_peer_2,           /* cadet handle */
+                          GC_u2h (TEST_PORT_ID), /* port id */
+                          &connected,            /* connect handler */
+                          (void *) 2L,           /* handle for #connected */
+                          NULL,                  /* window size handler */
+                          &disconnected,         /* disconnect handler */
+                          handlers);             /* traffic handlers */
+  delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2);
+  if (NULL == connect_task)
+    connect_task = GNUNET_SCHEDULER_add_delayed (delay,
+                                                 &do_connect,
+                                                 NULL);
+}
+
+
+/**
+ * Main
+ */
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
+                                    "test_cadet.conf",
+                                    &run, NULL))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "run failed\n");
+    return 2;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Final result: %d\n", result);
+  return (result == GNUNET_OK) ? 0 : 1;
+}
+
+
+/* end of test_cadet_local_1.c */
diff --git a/src/service/cadet/valgrind-cadet.supp b/src/service/cadet/valgrind-cadet.supp
new file mode 100644
index 000000000..fecd5185b
--- /dev/null
+++ b/src/service/cadet/valgrind-cadet.supp
@@ -0,0 +1,116 @@
+{
+   logsetup_addr
+   Memcheck:Addr8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:get_type
+   fun:GNUNET_log_setup
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   scanf_addr
+   Memcheck:Addr8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:vsscanf
+   fun:sscanf
+   fun:GNUNET_CONFIGURATION_get_value_number
+   fun:GNUNET_SERVICE_get_server_addresses
+   fun:setup_service
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   mylog_addr
+   Memcheck:Addr8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:service_task
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   mylog_uninit
+   Memcheck:Value8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:mylog
+   fun:GNUNET_log_from_nocheck
+   fun:service_task
+   ...
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   mylog_from_cond
+   Memcheck:Cond
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:mylog
+   fun:GNUNET_log_from_nocheck
+   ...
+   fun:service_task
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   mylog_cond
+   Memcheck:Cond
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:mylog
+   fun:GNUNET_log_nocheck
+   ...
+   fun:service_task
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   inet_ntop_cond
+   Memcheck:Cond
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:inet_ntop
+   fun:GNUNET_a2s
+   ...
+   fun:service_task
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   create_key_from_file
+   Memcheck:Addr8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:GNUNET_CRYPTO_rsa_key_create_from_file
+   fun:run
+   fun:service_task
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
+
+{
+   main_notify_handler
+   Memcheck:Addr8
+   obj:/lib/libc-2.14.1.so
+   ...
+   fun:main_notify_handler
+   fun:receive_ready
+   fun:GNUNET_SCHEDULER_run
+   fun:GNUNET_SERVICE_run
+   fun:main
+}
\ No newline at end of file