86 files changed, 39472 insertions, 0 deletions

diff --git a/src/service/fs/.gitignore b/src/service/fs/.gitignore
new file mode 100644
index 000000000..f3637f186
--- /dev/null
+++ b/src/service/fs/.gitignore
@@ -0,0 +1,36 @@
+gnunet-daemon-fsprofiler
+gnunet-fs-profiler
+gnunet-helper-fs-publish
+gnunet-service-fs
+test_fs_directory
+test_fs_download
+test_fs_download_cadet
+test_fs_download_indexed
+test_fs_download_persistence
+test_fs_file_information
+test_fs_getopt
+test_fs_list_indexed
+test_fs_namespace
+test_fs_namespace_list_updateable
+test_fs_publish
+test_fs_publish_persistence
+test_fs_search
+test_fs_search_persistence
+test_fs_search_probes
+test_fs_search_with_and
+test_fs_start_stop
+test_fs_test_lib
+test_fs_unindex
+test_fs_unindex_persistence
+test_fs_uri
+test_gnunet_fs_idx.py
+test_gnunet_fs_psd.py
+test_gnunet_fs_rec.py
+test_gnunet_service_fs_migration
+test_gnunet_service_fs_p2p
+test_gnunet_service_fs_p2p_cadet
+test_plugin_block_fs
+perf_gnunet_service_fs_p2p
+perf_gnunet_service_fs_p2p_index
+perf_gnunet_service_fs_p2p_respect
+rdir.gnd
diff --git a/src/service/fs/Makefile.am b/src/service/fs/Makefile.am
new file mode 100644
index 000000000..0a10f688a
--- /dev/null
+++ b/src/service/fs/Makefile.am
@@ -0,0 +1,376 @@
+# This Makefile.am is in the public domain
+AM_CPPFLAGS = -I$(top_srcdir)/src/include
+
+if USE_COVERAGE
+  AM_CFLAGS = --coverage -O0
+  XLIB = -lgcov
+endif
+
+pkgcfgdir= $(pkgdatadir)/config.d/
+
+libexecdir= $(pkglibdir)/libexec/
+
+pkgcfg_DATA = \
+  fs.conf
+
+lib_LTLIBRARIES = libgnunetfs.la
+
+libgnunetfs_la_SOURCES = \
+  fs_api.c fs_api.h fs.h \
+  fs_directory.c \
+  fs_dirmetascan.c \
+  fs_download.c \
+  fs_file_information.c \
+  fs_getopt.c \
+  fs_list_indexed.c \
+  fs_publish.c \
+  fs_publish_ksk.c \
+  fs_publish_ublock.c fs_publish_ublock.h \
+  fs_misc.c \
+  fs_namespace.c \
+  fs_search.c \
+  fs_sharetree.c \
+  fs_tree.c fs_tree.h \
+  fs_unindex.c \
+  fs_uri.c \
+  meta_data.c
+
+libgnunetfs_la_LIBADD = \
+  $(top_builddir)/src/service/datastore/libgnunetdatastore.la \
+  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la \
+  $(GN_LIBINTL) $(XLIB) $(LIBGCRYPT_LIBS) -lunistring
+
+if HAVE_LIBEXTRACTOR
+libgnunetfs_la_LIBADD += \
+  -lextractor
+endif
+
+libgnunetfs_la_LDFLAGS = \
+  $(GN_LIB_LDFLAGS)   \
+  -version-info 3:1:1
+
+
+libexec_PROGRAMS = \
+  gnunet-helper-fs-publish \
+  gnunet-service-fs
+
+noinst_PROGRAMS = \
+  gnunet-daemon-fsprofiler
+
+gnunet_helper_fs_publish_SOURCES =  \
+ gnunet-helper-fs-publish.c
+gnunet_helper_fs_publish_LDADD =  \
+  libgnunetfs.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la \
+  $(GN_LIBINTL)
+
+if HAVE_LIBEXTRACTOR
+gnunet_helper_fs_publish_LDADD += \
+  -lextractor
+endif
+
+gnunet_daemon_fsprofiler_SOURCES =  \
+ gnunet-daemon-fsprofiler.c
+gnunet_daemon_fsprofiler_LDADD = \
+  libgnunetfs.la \
+  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la \
+  $(GN_LIBINTL)
+
+gnunet_service_fs_SOURCES =  \
+ gnunet-service-fs.c gnunet-service-fs.h \
+ gnunet-service-fs_cp.c gnunet-service-fs_cp.h \
+ gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h \
+ gnunet-service-fs_pe.c gnunet-service-fs_pe.h \
+ gnunet-service-fs_pr.c gnunet-service-fs_pr.h \
+ gnunet-service-fs_push.c gnunet-service-fs_push.h \
+ gnunet-service-fs_put.c gnunet-service-fs_put.h \
+ gnunet-service-fs_cadet_client.c gnunet-service-fs_cadet.h \
+ gnunet-service-fs_cadet_server.c
+gnunet_service_fs_LDADD =  \
+ libgnunetfs.la \
+ $(top_builddir)/src/service/dht/libgnunetdht.la \
+ $(top_builddir)/src/lib/block/libgnunetblock.la \
+ $(top_builddir)/src/service/datastore/libgnunetdatastore.la \
+ $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+ $(top_builddir)/src/service/cadet/libgnunetcadet.la \
+ $(top_builddir)/src/service/core/libgnunetcore.la \
+ $(top_builddir)/src/lib/util/libgnunetutil.la \
+ $(top_builddir)/src/service/peerstore/libgnunetpeerstore.la \
+ $(GN_LIBINTL) -lm
+
+check_PROGRAMS = \
+ test_fs_directory \
+ test_fs_download \
+ test_fs_download_cadet \
+ test_fs_download_indexed \
+ test_fs_download_persistence \
+ test_fs_file_information \
+ test_fs_getopt \
+ test_fs_list_indexed \
+ test_fs_namespace \
+ test_fs_namespace_list_updateable \
+ test_fs_publish \
+ test_fs_publish_persistence \
+ test_fs_search \
+ test_fs_search_with_and \
+ test_fs_search_probes \
+ test_fs_search_persistence \
+ test_fs_start_stop \
+ test_fs_unindex \
+ test_fs_unindex_persistence \
+ test_fs_uri \
+ test_fs_meta_data \
+ $(FS_BENCHMARKS)
+# test_gnunet_service_fs_migration
+
+if ENABLE_TEST_RUN
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
+TESTS = test_fs_directory \
+ test_fs_file_information \
+ test_fs_namespace \
+ test_fs_namespace_list_updateable \
+ test_fs_search \
+ test_fs_search_with_and \
+ test_fs_search_probes \
+ test_fs_search_persistence \
+ test_fs_start_stop \
+ test_fs_uri \
+ test_fs_meta_data
+endif
+
+
+test_fs_directory_SOURCES = \
+ test_fs_directory.c
+test_fs_directory_LDADD = \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+if HAVE_LIBEXTRACTOR
+test_fs_directory_LDADD += \
+  -lextractor
+endif
+
+
+test_fs_download_SOURCES = \
+ test_fs_download.c
+test_fs_download_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la  \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_download_indexed_SOURCES = \
+ test_fs_download.c
+test_fs_download_indexed_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la  \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_download_cadet_SOURCES = \
+ test_fs_download.c
+test_fs_download_cadet_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la  \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_download_persistence_SOURCES = \
+ test_fs_download_persistence.c
+test_fs_download_persistence_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la  \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_file_information_SOURCES = \
+ test_fs_file_information.c
+test_fs_file_information_LDADD = \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+if HAVE_LIBEXTRACTOR
+test_fs_file_information_LDADD += \
+  -lextractor
+endif
+
+
+test_fs_getopt_SOURCES = \
+ test_fs_getopt.c
+test_fs_getopt_LDADD = \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_list_indexed_SOURCES = \
+ test_fs_list_indexed.c
+test_fs_list_indexed_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_namespace_SOURCES = \
+ test_fs_namespace.c
+test_fs_namespace_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_namespace_list_updateable_SOURCES = \
+ test_fs_namespace_list_updateable.c
+test_fs_namespace_list_updateable_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_publish_SOURCES = \
+ test_fs_publish.c
+test_fs_publish_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_publish_persistence_SOURCES = \
+ test_fs_publish_persistence.c
+test_fs_publish_persistence_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_search_SOURCES = \
+ test_fs_search.c
+test_fs_search_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la        \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_search_with_and_SOURCES = \
+ test_fs_search_with_and.c
+test_fs_search_with_and_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la        \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_search_probes_SOURCES = \
+ test_fs_search_probes.c
+test_fs_search_probes_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la        \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_search_persistence_SOURCES = \
+ test_fs_search_persistence.c
+test_fs_search_persistence_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la        \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_start_stop_SOURCES = \
+ test_fs_start_stop.c
+test_fs_start_stop_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_unindex_SOURCES = \
+ test_fs_unindex.c
+test_fs_unindex_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_unindex_persistence_SOURCES = \
+ test_fs_unindex_persistence.c
+test_fs_unindex_persistence_LDADD = \
+  $(top_builddir)/src/service/testing/libgnunettesting.la \
+  libgnunetfs.la \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+test_fs_meta_data_SOURCES = \
+ test_fs_meta_data.c
+test_fs_meta_data_LDADD = \
+ libgnunetfs.la \
+ $(top_builddir)/src/lib/util/libgnunetutil.la
+
+
+test_fs_uri_SOURCES = \
+ test_fs_uri.c
+test_fs_uri_LDADD = \
+  libgnunetfs.la  \
+  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+# TNG
+
+#test_gnunet_service_fs_p2p_SOURCES = \
+# test_gnunet_service_fs_p2p.c
+#test_gnunet_service_fs_p2p_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#test_gnunet_service_fs_p2p_cadet_SOURCES = \
+# test_gnunet_service_fs_p2p.c
+#test_gnunet_service_fs_p2p_cadet_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#test_gnunet_service_fs_migration_SOURCES = \
+# test_gnunet_service_fs_migration.c
+#test_gnunet_service_fs_migration_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#perf_gnunet_service_fs_p2p_SOURCES = \
+# perf_gnunet_service_fs_p2p.c
+#perf_gnunet_service_fs_p2p_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#perf_gnunet_service_fs_p2p_index_SOURCES = \
+# perf_gnunet_service_fs_p2p.c
+#perf_gnunet_service_fs_p2p_index_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#perf_gnunet_service_fs_p2p_dht_SOURCES = \
+# perf_gnunet_service_fs_p2p.c
+#perf_gnunet_service_fs_p2p_dht_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+#
+#perf_gnunet_service_fs_p2p_respect_SOURCES = \
+# perf_gnunet_service_fs_p2p_respect.c
+#perf_gnunet_service_fs_p2p_respect_LDADD = \
+#  libgnunetfstest.a \
+#  $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
+#  $(top_builddir)/src/testbed/libgnunettestbed.la \
+#  libgnunetfs.la  \
+#  $(top_builddir)/src/lib/util/libgnunetutil.la
+
+EXTRA_DIST = \
+  perf_gnunet_service_fs_p2p.conf \
+  test_fs_data.conf \
+  test_fs_defaults.conf \
+  test_fs_download_data.conf \
+  test_fs_download_indexed.conf \
+  test_fs_file_information_data.conf \
+  test_fs_list_indexed_data.conf \
+  test_fs_namespace_data.conf \
+  test_fs_publish_data.conf \
+  test_fs_search_data.conf \
+  test_fs_unindex_data.conf \
+  test_gnunet_fs_rec_data.tgz \
+  test_gnunet_service_fs_migration_data.conf \
+  test_gnunet_service_fs_p2p_cadet.conf \
+  test_pseudonym_data.conf
diff --git a/src/service/fs/fs.conf.in b/src/service/fs/fs.conf.in
new file mode 100644
index 000000000..be02619bf
--- /dev/null
+++ b/src/service/fs/fs.conf.in
@@ -0,0 +1,63 @@
+[fs]
+RUN_PER_USER = YES
+START_ON_DEMAND = @START_ON_DEMAND@
+IMMEDIATE_START = YES
+INDEXDB = $GNUNET_DATA_HOME/fs/idxinfo.lst
+RESPECT = $GNUNET_DATA_HOME/fs/credit/
+STATE_DIR = $GNUNET_DATA_HOME/fs/persistence/
+UPDATE_DIR = $GNUNET_DATA_HOME/fs/updates/
+@UNIXONLY@ PORT = 2094
+HOSTNAME = localhost
+BINARY = gnunet-service-fs
+ACCEPT_FROM = 127.0.0.1;
+ACCEPT_FROM6 = ::1;
+
+# PREFIX = valgrind
+
+# Do we introduce artificial delays? (may improve anonymity)
+DELAY = YES
+
+# Do we cache content from other nodes? (may improve anonymity)
+CONTENT_CACHING = YES
+
+# Do we send unsolicited data to other nodes if we have excess bandwidth?
+# (may improve anonymity, probably not a good idea if content_caching is NO)
+CONTENT_PUSHING = YES
+
+UNIXPATH = $GNUNET_USER_RUNTIME_DIR/gnunet-service-fs.sock
+
+# Do we require users that want to access file-sharing to run this process
+# (usually not a good idea)
+UNIX_MATCH_UID = NO
+
+# Do we require users that want to access file-sharing to be in the 'gnunet' group?
+UNIX_MATCH_GID = YES
+
+# Maximum number of requests this peer tracks (important for
+# memory consumption; 2k RAM/request is not unusual)
+MAX_PENDING_REQUESTS = 65536
+
+# How many requests do we have at most waiting in the queue towards
+# the datastore? (important for memory consumption)
+DATASTORE_QUEUE_SIZE = 32
+
+# Maximum frequency we're allowed to poll the datastore
+# for content for migration (can be used to reduce
+# GNUnet's disk-IO rate)
+MIN_MIGRATION_DELAY = 100 ms
+
+# For how many neighbouring peers should we allocate hash maps?
+EXPECTED_NEIGHBOUR_COUNT = 128
+
+# Disable anonymous file-sharing (but keep non-anonymous transfers)?
+# This option is mostly for testing.
+DISABLE_ANON_TRANSFER = NO
+
+# Maximum number of non-anonymous transfers this peer will support
+# at the same time.  Excessive values mostly have the problem that
+# the service might use more memory, so we need to bound this at
+# some reasonable level.  And if we have a very, very large
+# number, we probably won't have enough bandwidth to support them
+# well anyway, so better have a moderate cap.
+MAX_CADET_CLIENTS = 128
+
diff --git a/src/service/fs/fs.h b/src/service/fs/fs.h
new file mode 100644
index 000000000..c3bae65d2
--- /dev/null
+++ b/src/service/fs/fs.h
@@ -0,0 +1,397 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003--2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs.h
+ * @brief definitions for the entire fs module
+ * @author Igor Wronsky, Christian Grothoff
+ */
+#ifndef FS_H
+#define FS_H
+
+#include "gnunet_constants.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_dht_service.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_block_lib.h"
+#include "block_fs.h"
+
+
+/**
+ * Size of the individual blocks used for file-sharing.
+ */
+#define DBLOCK_SIZE (32 * 1024)
+
+/**
+ * Blocksize to use when hashing files for indexing (blocksize for IO,
+ * not for the DBlocks).  Larger blocksizes can be more efficient but
+ * will be more disruptive as far as the scheduler is concerned.
+ */
+#define HASHING_BLOCKSIZE (1024 * 128)
+
+
+/**
+ * @brief content hash key
+ */
+struct ContentHashKey
+{
+  /**
+   * Hash of the original content, used for encryption.
+   */
+  struct GNUNET_HashCode key;
+
+  /**
+   * Hash of the encrypted content, used for querying.
+   */
+  struct GNUNET_HashCode query;
+};
+
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+
+/**
+ * Message sent from a GNUnet (fs) publishing activity to sign
+ * a LOC URI.
+ */
+struct RequestLocSignatureMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Requested signature purpose.  For now, always
+   * #GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT.
+   */
+  uint32_t purpose GNUNET_PACKED;
+
+  /**
+   * Requested expiration time.
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration_time;
+
+  /**
+   * Information about the shared file (to be signed).
+   */
+  struct ContentHashKey chk;
+
+  /**
+   * Size of the shared file (to be signed).
+   */
+  uint64_t file_length;
+};
+
+
+/**
+ * Message sent from the service with the signed LOC URI.
+ */
+struct ResponseLocSignatureMessage
+{
+  /**
+   * Message type will be
+   * #GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Purpose of the generated signature.  For now, always
+   * #GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT.
+   */
+  uint32_t purpose GNUNET_PACKED;
+
+  /**
+   * Expiration time that was actually used (rounded!).
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration_time;
+
+  /**
+   * The requested signature.
+   */
+  struct GNUNET_CRYPTO_EddsaSignature signature;
+
+  /**
+   * Identity of the peer sharing the file.
+   */
+  struct GNUNET_PeerIdentity peer;
+};
+
+
+/**
+ * Message sent from a GNUnet (fs) publishing activity to the
+ * gnunet-fs-service to initiate indexing of a file.  The service is
+ * supposed to check if the specified file is available and has the
+ * same cryptographic hash.  It should then respond with either a
+ * confirmation or a denial.
+ *
+ * On OSes where this works, it is considered acceptable if the
+ * service only checks that the path, device and inode match (it can
+ * then be assumed that the hash will also match without actually
+ * computing it; this is an optimization that should be safe given
+ * that the client is not our adversary).
+ */
+struct IndexStartMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_INDEX_START.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * For alignment.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * ID of device containing the file, as seen by the client.  This
+   * device ID is obtained using a call like "statvfs" (and converting
+   * the "f_fsid" field to a 32-bit big-endian number).  Use 0 if the
+   * OS does not support this, in which case the service must do a
+   * full hash recomputation.
+   */
+  uint64_t device GNUNET_PACKED;
+
+  /**
+   * Inode of the file on the given device, as seen by the client
+   * ("st_ino" field from "struct stat").  Use 0 if the OS does not
+   * support this, in which case the service must do a full hash
+   * recomputation.
+   */
+  uint64_t inode GNUNET_PACKED;
+
+  /**
+   * Hash of the file that we would like to index.
+   */
+  struct GNUNET_HashCode file_id;
+
+  /* this is followed by a 0-terminated
+   * filename of a file with the hash
+   * "file_id" as seen by the client */
+};
+
+
+/**
+ * Message send by FS service in response to a request
+ * asking for a list of all indexed files.
+ */
+struct IndexInfoMessage
+{
+  /**
+   * Message type will be
+   * #GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * Hash of the indexed file.
+   */
+  struct GNUNET_HashCode file_id;
+
+  /* this is followed by a 0-terminated
+   * filename of a file with the hash
+   * "file_id" as seen by the client */
+};
+
+
+/**
+ * Message sent from a GNUnet (fs) unindexing activity to the
+ * gnunet-service-fs to indicate that a file will be unindexed.  The
+ * service is supposed to remove the file from the list of indexed
+ * files and response with a confirmation message (even if the file
+ * was already not on the list).
+ */
+struct UnindexMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_UNINDEX.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * Hash of the file that we will unindex.
+   */
+  struct GNUNET_HashCode file_id;
+};
+
+
+/**
+ * No options.
+ */
+#define SEARCH_MESSAGE_OPTION_NONE 0
+
+/**
+ * Only search the local datastore (no network)
+ */
+#define SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY 1
+
+/**
+ * Request is too large to fit in 64k format.  The list of
+ * already-known search results will be continued in another message
+ * for the same type/query/target and additional already-known results
+ * following this one).
+ */
+#define SEARCH_MESSAGE_OPTION_CONTINUED 2
+
+
+/**
+ * Message sent from a GNUnet (fs) search activity to the
+ * gnunet-service-fs to start a search.
+ */
+struct SearchMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_START_SEARCH.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Bitmask with options.  Zero for no options, one for
+   * loopback-only, two for 'to be continued' (with a second search
+   * message for the same type/query/target and additional
+   * already-known results following this one).  See
+   * SEARCH_MESSAGE_OPTION_ defines.
+   *
+   * Other bits are currently not defined.
+   */
+  uint32_t options GNUNET_PACKED;
+
+  /**
+   * Type of the content that we're looking for.
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * Desired anonymity level, big-endian.
+   */
+  uint32_t anonymity_level GNUNET_PACKED;
+
+  /**
+   * If the request is for a DBLOCK or IBLOCK, this is the identity of
+   * the peer that is known to have a response.  Set to all-zeros if
+   * such a target is not known (note that even if OUR anonymity
+   * level is >0 we may happen to know the responder's identity;
+   * nevertheless, we should probably not use it for a DHT-lookup
+   * or similar blunt actions in order to avoid exposing ourselves).
+   * <p>
+   * Otherwise, "target" must be all zeros.
+   */
+  struct GNUNET_PeerIdentity target;
+
+  /**
+   * Hash of the public key for UBLOCKs; Hash of
+   * the CHK-encoded block for DBLOCKS and IBLOCKS.
+   */
+  struct GNUNET_HashCode query;
+
+  /* this is followed by the hash codes of already-known
+   * results (which should hence be excluded from what
+   * the service returns); naturally, this only applies
+   * to queries that can have multiple results (UBLOCKS).
+   */
+};
+
+
+/**
+ * Response from FS service with a result for a previous FS search.
+ * Note that queries for DBLOCKS and IBLOCKS that have received a
+ * single response are considered done.  This message is transmitted
+ * between peers.
+ */
+struct PutMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_PUT.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Type of the block (in big endian).  Should never be zero.
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * When does this result expire?
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration;
+
+  /* this is followed by the actual encrypted content */
+};
+
+/**
+ * Response from FS service with a result for a previous FS search.
+ * Note that queries for DBLOCKS and IBLOCKS that have received a
+ * single response are considered done.  This message is transmitted
+ * between the service and a client.
+ */
+struct ClientPutMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_PUT.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Type of the block (in big endian).  Should never be zero.
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * When does this result expire?
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration;
+
+  /**
+   * When was the last time we've tried to download this block?
+   * (FOREVER if unknown/not relevant)
+   */
+  struct GNUNET_TIME_AbsoluteNBO last_transmission;
+
+  /**
+   * How often did we transmit this query before getting an
+   * answer (estimate).
+   */
+  uint32_t num_transmissions;
+
+  /**
+   * How much respect did we offer (in total) before getting an
+   * answer (estimate).
+   */
+  uint32_t respect_offered;
+
+  /* this is followed by the actual encrypted content */
+};
+GNUNET_NETWORK_STRUCT_END
+
+
+#endif
+
+/* end of fs.h */
diff --git a/src/service/fs/fs_api.c b/src/service/fs/fs_api.c
new file mode 100644
index 000000000..65646c7eb
--- /dev/null
+++ b/src/service/fs/fs_api.c
@@ -0,0 +1,3321 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001--2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_api.c
+ * @brief main FS functions (master initialization, serialization, deserialization, shared code)
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+
+/**
+ * How many block requests can we have outstanding in parallel at a time by default?
+ */
+#define DEFAULT_MAX_PARALLEL_REQUESTS (1024 * 10)
+
+/**
+ * How many downloads can we have outstanding in parallel at a time by default?
+ */
+#define DEFAULT_MAX_PARALLEL_DOWNLOADS 16
+
+/**
+ * Start the given job (send signal, remove from pending queue, update
+ * counters and state).
+ *
+ * @param qe job to start
+ */
+static void
+start_job (struct GNUNET_FS_QueueEntry *qe)
+{
+  qe->active = GNUNET_YES;
+  qe->start (qe->cls);
+  qe->start_times++;
+  qe->h->active_blocks += qe->blocks;
+  qe->h->active_downloads++;
+  qe->start_time = GNUNET_TIME_absolute_get ();
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting job %p (%u active)\n",
+              qe,
+              qe->h->active_downloads);
+  GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe);
+  GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head,
+                                     qe->h->running_tail,
+                                     qe->h->running_tail,
+                                     qe);
+}
+
+
+/**
+ * Stop the given job (send signal, remove from active queue, update
+ * counters and state).
+ *
+ * @param qe job to stop
+ */
+static void
+stop_job (struct GNUNET_FS_QueueEntry *qe)
+{
+  qe->active = GNUNET_NO;
+  qe->stop (qe->cls);
+  GNUNET_assert (0 < qe->h->active_downloads);
+  qe->h->active_downloads--;
+  qe->h->active_blocks -= qe->blocks;
+  qe->run_time = GNUNET_TIME_relative_add (qe->run_time,
+                                           GNUNET_TIME_absolute_get_duration (
+                                             qe->start_time));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Stopping job %p (%u active)\n",
+              qe,
+              qe->h->active_downloads);
+  GNUNET_CONTAINER_DLL_remove (qe->h->running_head, qe->h->running_tail, qe);
+  GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head,
+                                     qe->h->pending_tail,
+                                     qe->h->pending_tail,
+                                     qe);
+}
+
+
+/**
+ * Process the jobs in the job queue, possibly starting some
+ * and stopping others.
+ *
+ * @param cls the `struct GNUNET_FS_Handle *`
+ */
+static void
+process_job_queue (void *cls)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  struct GNUNET_FS_QueueEntry *qe;
+  struct GNUNET_FS_QueueEntry *next;
+  struct GNUNET_TIME_Relative run_time;
+  struct GNUNET_TIME_Relative restart_at;
+  struct GNUNET_TIME_Relative rst;
+  struct GNUNET_TIME_Absolute end_time;
+  unsigned int num_downloads_waiting;
+  unsigned int num_downloads_active;
+  unsigned int num_downloads_expired;
+  unsigned int num_probes_active;
+  unsigned int num_probes_waiting;
+  unsigned int num_probes_expired;
+  int num_probes_change;
+  int num_downloads_change;
+  int block_limit_hit;
+
+  h->queue_job = NULL;
+  /* restart_at will be set to the time when it makes sense to
+     re-evaluate the job queue (unless, of course, jobs complete
+     or are added, then we'll be triggered immediately */
+  restart_at = GNUNET_TIME_UNIT_FOREVER_REL;
+  /* first, calculate some basic statistics on pending jobs */
+  num_probes_waiting = 0;
+  num_downloads_waiting = 0;
+  for (qe = h->pending_head; NULL != qe; qe = qe->next)
+  {
+    switch (qe->priority)
+    {
+    case GNUNET_FS_QUEUE_PRIORITY_PROBE:
+      num_probes_waiting++;
+      break;
+
+    case GNUNET_FS_QUEUE_PRIORITY_NORMAL:
+      num_downloads_waiting++;
+      break;
+
+    default:
+      GNUNET_break (0);
+      break;
+    }
+  }
+  /* now, calculate some basic statistics on running jobs */
+  num_probes_active = 0;
+  num_probes_expired = 0;
+  num_downloads_active = 0;
+  num_downloads_expired = 0;
+  next = h->running_head;
+  while (NULL != (qe = next))
+  {
+    next = qe->next;
+    switch (qe->priority)
+    {
+    case GNUNET_FS_QUEUE_PRIORITY_PROBE:
+      run_time = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2);
+      end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time);
+      rst = GNUNET_TIME_absolute_get_remaining (end_time);
+      if (0 == rst.rel_value_us)
+      {
+        num_probes_expired++;
+        stop_job (qe);
+      }
+      else
+      {
+        num_probes_active++;
+        restart_at = GNUNET_TIME_relative_min (rst, restart_at);
+      }
+      break;
+
+    case GNUNET_FS_QUEUE_PRIORITY_NORMAL:
+      run_time =
+        GNUNET_TIME_relative_saturating_multiply (h->avg_block_latency,
+                                                  qe->blocks * qe->start_times);
+      end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time);
+      rst = GNUNET_TIME_absolute_get_remaining (end_time);
+      if (0 == rst.rel_value_us)
+      {
+        num_downloads_expired++;
+        stop_job (qe);
+      }
+      else
+      {
+        num_downloads_active++;
+        restart_at = GNUNET_TIME_relative_min (rst, restart_at);
+      }
+      break;
+
+    default:
+      GNUNET_break (0);
+      break;
+    }
+  }
+  GNUNET_break (h->active_downloads ==
+                num_downloads_active + num_probes_active);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "PA: %u, PE: %u, PW: %u; DA: %u, DE: %u, DW: %u\n",
+              num_probes_active,
+              num_probes_expired,
+              num_probes_waiting,
+              num_downloads_active,
+              num_downloads_expired,
+              num_downloads_waiting);
+  GNUNET_break (h->active_downloads + num_probes_active <=
+                h->max_parallel_downloads);
+  /* calculate start/stop decisions */
+  if (h->active_downloads + num_downloads_waiting > h->max_parallel_downloads)
+  {
+    /* stop as many probes as there are downloads and probes */
+    num_probes_change = -GNUNET_MIN (num_probes_active, num_downloads_waiting);
+    /* start as many downloads as there are free slots, including those
+       we just opened up */
+    num_downloads_change =
+      h->max_parallel_downloads - h->active_downloads - num_probes_change;
+  }
+  else
+  {
+    /* start all downloads (we can) */
+    num_downloads_change = num_downloads_waiting;
+    /* also start probes if there is room, but use a lower cap of (mpd/4) + 1 */
+    if (1 + h->max_parallel_downloads / 4 >=
+        (h->active_downloads + num_downloads_change))
+      num_probes_change =
+        GNUNET_MIN (num_probes_waiting,
+                    (1 + h->max_parallel_downloads / 4)
+                    - (h->active_downloads + num_downloads_change));
+    else
+      num_probes_change = 0;
+  }
+  GNUNET_break (num_downloads_change <= num_downloads_waiting);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Changing %d probes and %d/%u/%u downloads\n",
+              num_probes_change,
+              num_downloads_change,
+              (unsigned int) h->active_downloads,
+              (unsigned int) h->max_parallel_downloads);
+  /* actually stop probes */
+  next = h->running_head;
+  while (NULL != (qe = next))
+  {
+    next = qe->next;
+    if (GNUNET_FS_QUEUE_PRIORITY_PROBE != qe->priority)
+      continue;
+    if (num_probes_change < 0)
+    {
+      stop_job (qe);
+      num_probes_change++;
+      if (0 == num_probes_change)
+        break;
+    }
+  }
+  GNUNET_break (0 <= num_probes_change);
+
+  /* start some more tasks if we now have empty slots */
+  block_limit_hit = GNUNET_NO;
+  next = h->pending_head;
+  while ((NULL != (qe = next)) &&
+         ((num_probes_change > 0) || (num_downloads_change > 0)))
+  {
+    next = qe->next;
+    switch (qe->priority)
+    {
+    case GNUNET_FS_QUEUE_PRIORITY_PROBE:
+      if (num_probes_change > 0)
+      {
+        start_job (qe);
+        num_probes_change--;
+        run_time = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2);
+        restart_at = GNUNET_TIME_relative_min (run_time, restart_at);
+      }
+      break;
+
+    case GNUNET_FS_QUEUE_PRIORITY_NORMAL:
+      if ((num_downloads_change > 0) &&
+          ((qe->blocks + h->active_blocks <= h->max_parallel_requests) ||
+           ((qe->blocks > h->max_parallel_requests) &&
+            (0 == h->active_downloads))))
+      {
+        start_job (qe);
+        num_downloads_change--;
+      }
+      else if (num_downloads_change > 0)
+        block_limit_hit = GNUNET_YES;
+      break;
+
+    default:
+      GNUNET_break (0);
+      break;
+    }
+  }
+  GNUNET_break ((0 == num_downloads_change) || (GNUNET_YES == block_limit_hit));
+  GNUNET_break (0 == num_probes_change);
+
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "AD: %u, MP: %u; %d probes and %d downloads to start, will run again in %s\n",
+    h->active_downloads,
+    h->max_parallel_requests,
+    num_probes_change,
+    num_downloads_change,
+    GNUNET_STRINGS_relative_time_to_string (restart_at, GNUNET_YES));
+
+  /* make sure we run again, callbacks might have
+     already re-scheduled the job, so cancel such
+     an operation (if it exists) */
+  if (NULL != h->queue_job)
+    GNUNET_SCHEDULER_cancel (h->queue_job);
+  h->queue_job =
+    GNUNET_SCHEDULER_add_delayed (restart_at, &process_job_queue, h);
+}
+
+
+struct GNUNET_FS_QueueEntry *
+GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h,
+                  GNUNET_SCHEDULER_TaskCallback start,
+                  GNUNET_SCHEDULER_TaskCallback stop,
+                  void *cls,
+                  unsigned int blocks,
+                  enum GNUNET_FS_QueuePriority priority)
+{
+  struct GNUNET_FS_QueueEntry *qe;
+
+  qe = GNUNET_new (struct GNUNET_FS_QueueEntry);
+  qe->h = h;
+  qe->start = start;
+  qe->stop = stop;
+  qe->cls = cls;
+  qe->queue_time = GNUNET_TIME_absolute_get ();
+  qe->blocks = blocks;
+  qe->priority = priority;
+  GNUNET_CONTAINER_DLL_insert_after (h->pending_head,
+                                     h->pending_tail,
+                                     h->pending_tail,
+                                     qe);
+  if (NULL != h->queue_job)
+    GNUNET_SCHEDULER_cancel (h->queue_job);
+  h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Queueing job %p\n", qe);
+  return qe;
+}
+
+
+/**
+ * Dequeue a job from the queue.
+ *
+ * @param qe handle for the job
+ */
+void
+GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qe)
+{
+  struct GNUNET_FS_Handle *h;
+
+  h = qe->h;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Dequeueing job %p\n", qe);
+  if (GNUNET_YES == qe->active)
+    stop_job (qe);
+  GNUNET_CONTAINER_DLL_remove (h->pending_head, h->pending_tail, qe);
+  GNUNET_free (qe);
+  if (NULL != h->queue_job)
+    GNUNET_SCHEDULER_cancel (h->queue_job);
+  h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h);
+}
+
+
+/**
+ * Create a top-level activity entry.
+ *
+ * @param h global fs handle
+ * @param ssf suspend signal function to use
+ * @param ssf_cls closure for @a ssf
+ * @return fresh top-level activity handle
+ */
+struct TopLevelActivity *
+GNUNET_FS_make_top (struct GNUNET_FS_Handle *h,
+                    SuspendSignalFunction ssf,
+                    void *ssf_cls)
+{
+  struct TopLevelActivity *ret;
+
+  ret = GNUNET_new (struct TopLevelActivity);
+  ret->ssf = ssf;
+  ret->ssf_cls = ssf_cls;
+  GNUNET_CONTAINER_DLL_insert (h->top_head, h->top_tail, ret);
+  return ret;
+}
+
+
+/**
+ * Destroy a top-level activity entry.
+ *
+ * @param h global fs handle
+ * @param top top level activity entry
+ */
+void
+GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top)
+{
+  GNUNET_CONTAINER_DLL_remove (h->top_head, h->top_tail, top);
+  GNUNET_free (top);
+}
+
+
+/**
+ * Closure for #GNUNET_FS_data_reader_file_().
+ */
+struct FileInfo
+{
+  /**
+   * Name of the file to read.
+   */
+  char *filename;
+
+  /**
+   * File descriptor, NULL if it has not yet been opened.
+   */
+  struct GNUNET_DISK_FileHandle *fd;
+};
+
+
+/**
+ * Function that provides data by reading from a file.
+ *
+ * @param cls closure with the `struct FileInfo *`
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times; set to `UINT64_MAX` to tell
+ *            the reader that we won't be reading for a while
+ *            (used to close the file descriptor but NOT fully
+ *             clean up the reader's state); in this case,
+ *            a value of '0' for @a max should be ignored
+ * @param max maximum number of bytes that should be
+ *            copied to @a buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually @a max, 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_file_ (void *cls,
+                             uint64_t offset,
+                             size_t max,
+                             void *buf,
+                             char **emsg)
+{
+  struct FileInfo *fi = cls;
+  ssize_t ret;
+
+  if (UINT64_MAX == offset)
+  {
+    if (NULL != fi->fd)
+    {
+      GNUNET_DISK_file_close (fi->fd);
+      fi->fd = NULL;
+    }
+    return 0;
+  }
+  if (0 == max)
+  {
+    if (NULL != fi->fd)
+      GNUNET_DISK_file_close (fi->fd);
+    GNUNET_free (fi->filename);
+    GNUNET_free (fi);
+    return 0;
+  }
+  if (NULL == fi->fd)
+  {
+    fi->fd = GNUNET_DISK_file_open (fi->filename,
+                                    GNUNET_DISK_OPEN_READ,
+                                    GNUNET_DISK_PERM_NONE);
+    if (NULL == fi->fd)
+    {
+      GNUNET_asprintf (emsg,
+                       _ ("Could not open file `%s': %s"),
+                       fi->filename,
+                       strerror (errno));
+      return 0;
+    }
+  }
+  if ((GNUNET_SYSERR ==
+       GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET)) ||
+      (-1 == (ret = GNUNET_DISK_file_read (fi->fd, buf, max))))
+  {
+    GNUNET_asprintf (emsg,
+                     _ ("Could not read file `%s': %s"),
+                     fi->filename,
+                     strerror (errno));
+    return 0;
+  }
+  if (ret != max)
+  {
+    GNUNET_asprintf (emsg,
+                     _ ("Short read reading from file `%s'!"),
+                     fi->filename);
+    return 0;
+  }
+  return max;
+}
+
+
+void *
+GNUNET_FS_make_file_reader_context_ (const char *filename)
+{
+  struct FileInfo *fi;
+
+  fi = GNUNET_new (struct FileInfo);
+  fi->filename = GNUNET_STRINGS_filename_expand (filename);
+  if (NULL == fi->filename)
+  {
+    GNUNET_free (fi);
+    return NULL;
+  }
+  return fi;
+}
+
+
+/**
+ * Function that provides data by copying from a buffer.
+ *
+ * @param cls closure (points to the buffer)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times; set to `UINT64_MAX` to tell
+ *            the reader that we won't be reading for a while
+ *            (used to close the file descriptor but NOT fully
+ *             clean up the reader's state); in this case,
+ *            a value of '0' for @a max should be ignored
+ * @param max maximum number of bytes that should be
+ *            copied to @a buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually @a max, 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_copy_ (void *cls,
+                             uint64_t offset,
+                             size_t max,
+                             void *buf,
+                             char **emsg)
+{
+  char *data = cls;
+
+  if (UINT64_MAX == offset)
+    return 0;
+  if (0 == max)
+  {
+    GNUNET_free (data);
+    return 0;
+  }
+  GNUNET_memcpy (buf, &data[offset], max);
+  return max;
+}
+
+
+/**
+ * Return the full filename where we would store state information
+ * (for serialization/deserialization).
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param ent entity identifier (or empty string for the directory)
+ * @return NULL on error
+ */
+static char *
+get_serialization_file_name (struct GNUNET_FS_Handle *h,
+                             const char *ext,
+                             const char *ent)
+{
+  char *basename;
+  char *ret;
+
+  if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE))
+    return NULL; /* persistence not requested */
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (h->cfg,
+                                                            "fs",
+                                                            "STATE_DIR",
+                                                            &basename))
+    return NULL;
+  GNUNET_asprintf (&ret,
+                   "%s%s%s%s%s%s%s",
+                   basename,
+                   DIR_SEPARATOR_STR,
+                   h->client_name,
+                   DIR_SEPARATOR_STR,
+                   ext,
+                   DIR_SEPARATOR_STR,
+                   ent);
+  GNUNET_free (basename);
+  return ret;
+}
+
+
+/**
+ * Return the full filename where we would store state information
+ * (for serialization/deserialization) that is associated with a
+ * parent operation.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni name of the parent operation
+ * @param ent entity identifier (or empty string for the directory)
+ * @return NULL on error
+ */
+static char *
+get_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h,
+                                    const char *ext,
+                                    const char *uni,
+                                    const char *ent)
+{
+  char *basename;
+  char *ret;
+
+  if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE))
+    return NULL; /* persistence not requested */
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (h->cfg,
+                                                            "fs",
+                                                            "STATE_DIR",
+                                                            &basename))
+    return NULL;
+  GNUNET_asprintf (&ret,
+                   "%s%s%s%s%s%s%s.dir%s%s",
+                   basename,
+                   DIR_SEPARATOR_STR,
+                   h->client_name,
+                   DIR_SEPARATOR_STR,
+                   ext,
+                   DIR_SEPARATOR_STR,
+                   uni,
+                   DIR_SEPARATOR_STR,
+                   ent);
+  GNUNET_free (basename);
+  return ret;
+}
+
+
+/**
+ * Return a read handle for deserialization.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param ent entity identifier (or empty string for the directory)
+ * @return NULL on error
+ */
+static struct GNUNET_BIO_ReadHandle *
+get_read_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent)
+{
+  char *fn;
+  struct GNUNET_BIO_ReadHandle *ret;
+
+  fn = get_serialization_file_name (h, ext, ent);
+  if (NULL == fn)
+    return NULL;
+  ret = GNUNET_BIO_read_open_file (fn);
+  GNUNET_free (fn);
+  return ret;
+}
+
+
+/**
+ * Return a write handle for serialization.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param ent entity identifier (or empty string for the directory)
+ * @return NULL on error
+ */
+static struct GNUNET_BIO_WriteHandle *
+get_write_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent)
+{
+  char *fn;
+  struct GNUNET_BIO_WriteHandle *ret;
+
+  fn = get_serialization_file_name (h, ext, ent);
+  if (NULL == fn)
+    return NULL;
+  ret = GNUNET_BIO_write_open_file (fn);
+  GNUNET_break (NULL != ret);
+  GNUNET_free (fn);
+  return ret;
+}
+
+
+/**
+ * Return a write handle for serialization.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni name of parent
+ * @param ent entity identifier (or empty string for the directory)
+ * @return NULL on error
+ */
+static struct GNUNET_BIO_WriteHandle *
+get_write_handle_in_dir (struct GNUNET_FS_Handle *h,
+                         const char *ext,
+                         const char *uni,
+                         const char *ent)
+{
+  char *fn;
+  struct GNUNET_BIO_WriteHandle *ret;
+
+  fn = get_serialization_file_name_in_dir (h, ext, uni, ent);
+  if (NULL == fn)
+    return NULL;
+  ret = GNUNET_BIO_write_open_file (fn);
+  GNUNET_free (fn);
+  return ret;
+}
+
+
+/**
+ * Remove serialization/deserialization file from disk.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param ent entity identifier
+ */
+void
+GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h,
+                             const char *ext,
+                             const char *ent)
+{
+  char *filename;
+
+  if ((NULL == ent) || (0 == strlen (ent)))
+  {
+    GNUNET_break (0);
+    return;
+  }
+  filename = get_serialization_file_name (h, ext, ent);
+  if (NULL != filename)
+  {
+    if ((0 != unlink (filename)) && (ENOENT != errno))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename);
+    GNUNET_free (filename);
+  }
+}
+
+
+/**
+ * Remove serialization/deserialization file from disk.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni parent name
+ * @param ent entity identifier
+ */
+static void
+remove_sync_file_in_dir (struct GNUNET_FS_Handle *h,
+                         const char *ext,
+                         const char *uni,
+                         const char *ent)
+{
+  char *filename;
+
+  if ((NULL == ent) || (0 == strlen (ent)))
+  {
+    GNUNET_break (0);
+    return;
+  }
+  filename = get_serialization_file_name_in_dir (h, ext, uni, ent);
+  if (NULL == filename)
+    return;
+  if (0 != unlink (filename))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename);
+  GNUNET_free (filename);
+}
+
+
+/**
+ * Remove serialization/deserialization directory from disk.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni unique name of parent
+ */
+void
+GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h,
+                            const char *ext,
+                            const char *uni)
+{
+  char *dn;
+
+  if (NULL == uni)
+    return;
+  dn = get_serialization_file_name_in_dir (h, ext, uni, "");
+  if (NULL == dn)
+    return;
+  if ((GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES)) &&
+      (GNUNET_OK != GNUNET_DISK_directory_remove (dn)))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "rmdir", dn);
+  GNUNET_free (dn);
+}
+
+
+/**
+ * Serialize a start-time.  Since we use start-times to
+ * calculate the duration of some operation, we actually
+ * do not serialize the absolute time but the (relative)
+ * duration since the start time.  When we then
+ * deserialize the start time, we take the current time and
+ * subtract that duration so that we get again an absolute
+ * time stamp that will result in correct performance
+ * calculations.
+ *
+ * @param wh handle for writing
+ * @param timestamp time to serialize
+ * @return #GNUNET_OK on success
+ */
+static int
+write_start_time (struct GNUNET_BIO_WriteHandle *wh,
+                  struct GNUNET_TIME_Absolute timestamp)
+{
+  struct GNUNET_TIME_Relative dur;
+
+  dur = GNUNET_TIME_absolute_get_duration (timestamp);
+  return GNUNET_BIO_write_int64 (wh, "start time", dur.rel_value_us);
+}
+
+
+/**
+ * Deserialize a start-time.  Since we use start-times to
+ * calculate the duration of some operation, we actually
+ * do not serialize the absolute time but the (relative)
+ * duration since the start time.  Thus, when we then
+ * deserialize the start time, we take the current time and
+ * subtract that duration so that we get again an absolute
+ * time stamp that will result in correct performance
+ * calculations.
+ *
+ * @param rh handle for reading
+ * @param timestamp where to write the deserialized timestamp
+ * @return #GNUNET_OK on success
+ */
+static int
+read_start_time (struct GNUNET_BIO_ReadHandle *rh,
+                 struct GNUNET_TIME_Absolute *timestamp)
+{
+  struct GNUNET_TIME_Relative dur;
+
+  if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, "start time",
+                                          (int64_t *) &dur.rel_value_us))
+    return GNUNET_SYSERR;
+  *timestamp = GNUNET_TIME_absolute_subtract (GNUNET_TIME_absolute_get (), dur);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Using the given serialization filename, try to deserialize
+ * the file-information tree associated with it.
+ *
+ * @param h master context
+ * @param filename name of the file (without directory) with
+ *        the information
+ * @return NULL on error
+ */
+static struct GNUNET_FS_FileInformation *
+deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename);
+
+
+/**
+ * Using the given serialization filename, try to deserialize
+ * the file-information tree associated with it.
+ *
+ * @param h master context
+ * @param fn name of the file (without directory) with
+ *        the information
+ * @param rh handle for reading
+ * @return NULL on error
+ */
+static struct GNUNET_FS_FileInformation *
+deserialize_fi_node (struct GNUNET_FS_Handle *h,
+                     const char *fn,
+                     struct GNUNET_BIO_ReadHandle *rh)
+{
+  struct GNUNET_FS_FileInformation *ret;
+  struct GNUNET_FS_FileInformation *nxt;
+  char b;
+  char *ksks;
+  char *chks;
+  char *skss;
+  char *filename;
+  uint32_t dsize;
+
+  if (GNUNET_OK != GNUNET_BIO_read (rh, "status flag", &b, sizeof(b)))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  ret = GNUNET_new (struct GNUNET_FS_FileInformation);
+  ret->h = h;
+  ksks = NULL;
+  chks = NULL;
+  skss = NULL;
+  filename = NULL;
+  if ((GNUNET_OK != GNUNET_FS_read_meta_data (rh, "metadata", &ret->meta)) ||
+      (GNUNET_OK != GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32 * 1024)) ||
+      ((NULL != ksks) &&
+       ((NULL == (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL))) ||
+        (GNUNET_YES != GNUNET_FS_uri_test_ksk (ret->keywords)))) ||
+      (GNUNET_OK != GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) ||
+      ((NULL != chks) &&
+       ((NULL == (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) ||
+        (GNUNET_YES != GNUNET_FS_uri_test_chk (ret->chk_uri)))) ||
+      (GNUNET_OK != GNUNET_BIO_read_string (rh, "sks-uri", &skss, 1024)) ||
+      ((NULL != skss) &&
+       ((NULL == (ret->sks_uri = GNUNET_FS_uri_parse (skss, NULL))) ||
+        (GNUNET_YES != GNUNET_FS_uri_test_sks (ret->sks_uri)))) ||
+      (GNUNET_OK != read_start_time (rh, &ret->start_time)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16 * 1024)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16 * 1024)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_int64 (
+         rh,
+         "expiration time",
+         (int64_t *) &ret->bo.expiration_time.abs_value_us)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "anonymity level",
+         (int32_t *) &ret->bo.anonymity_level)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "content priority",
+         (int32_t *) &ret->bo.content_priority)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "replication level",
+         (int32_t *) &ret->bo.replication_level)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  switch (b)
+  {
+  case 0:   /* file-insert */
+    if (GNUNET_OK != GNUNET_BIO_read_int64 (
+          rh,
+          "file size",
+          (int64_t *) &ret->data.file.file_size))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    ret->is_directory = GNUNET_NO;
+    ret->data.file.do_index = GNUNET_NO;
+    ret->data.file.have_hash = GNUNET_NO;
+    ret->data.file.index_start_confirmed = GNUNET_NO;
+    if (GNUNET_NO == ret->is_published)
+    {
+      if (NULL == ret->filename)
+      {
+        ret->data.file.reader = &GNUNET_FS_data_reader_copy_;
+        ret->data.file.reader_cls =
+          GNUNET_malloc_large (ret->data.file.file_size);
+        if (ret->data.file.reader_cls == NULL)
+          goto cleanup;
+        if (GNUNET_OK != GNUNET_BIO_read (rh,
+                                          "file-data",
+                                          ret->data.file.reader_cls,
+                                          ret->data.file.file_size))
+        {
+          GNUNET_break (0);
+          goto cleanup;
+        }
+      }
+      else
+      {
+        ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+        ret->data.file.reader_cls =
+          GNUNET_FS_make_file_reader_context_ (ret->filename);
+      }
+    }
+    break;
+
+  case 1:   /* file-index, no hash */
+    if (NULL == ret->filename)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if (GNUNET_OK != GNUNET_BIO_read_int64 (
+          rh,
+          "file size",
+          (int64_t *) &ret->data.file.file_size))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    ret->is_directory = GNUNET_NO;
+    ret->data.file.do_index = GNUNET_YES;
+    ret->data.file.have_hash = GNUNET_NO;
+    ret->data.file.index_start_confirmed = GNUNET_NO;
+    ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+    ret->data.file.reader_cls =
+      GNUNET_FS_make_file_reader_context_ (ret->filename);
+    break;
+
+  case 2:   /* file-index-with-hash */
+    if (NULL == ret->filename)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if ((GNUNET_OK != GNUNET_BIO_read_int64 (
+           rh,
+           "file size",
+           (int64_t *) &ret->data.file.file_size)) ||
+        (GNUNET_OK != GNUNET_BIO_read (rh,
+                                       "fileid",
+                                       &ret->data.file.file_id,
+                                       sizeof(struct GNUNET_HashCode))))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    ret->is_directory = GNUNET_NO;
+    ret->data.file.do_index = GNUNET_YES;
+    ret->data.file.have_hash = GNUNET_YES;
+    ret->data.file.index_start_confirmed = GNUNET_NO;
+    ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+    ret->data.file.reader_cls =
+      GNUNET_FS_make_file_reader_context_ (ret->filename);
+    break;
+
+  case 3:   /* file-index-with-hash-confirmed */
+    if (NULL == ret->filename)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if ((GNUNET_OK != GNUNET_BIO_read_int64 (
+           rh,
+           "file size",
+           (int64_t *) &ret->data.file.file_size)) ||
+        (GNUNET_OK != GNUNET_BIO_read (rh,
+                                       "fileid",
+                                       &ret->data.file.file_id,
+                                       sizeof(struct GNUNET_HashCode))))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    ret->is_directory = GNUNET_NO;
+    ret->data.file.do_index = GNUNET_YES;
+    ret->data.file.have_hash = GNUNET_YES;
+    ret->data.file.index_start_confirmed = GNUNET_YES;
+    ret->data.file.reader = &GNUNET_FS_data_reader_file_;
+    ret->data.file.reader_cls =
+      GNUNET_FS_make_file_reader_context_ (ret->filename);
+    break;
+
+  case 4:   /* directory */
+    ret->is_directory = GNUNET_YES;
+    if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, "dsize",
+                                             (int32_t *) &dsize)) ||
+        (GNUNET_OK !=
+         GNUNET_BIO_read_int64 (
+           rh,
+           "contents completed",
+           (int64_t *) &ret->data.dir.contents_completed)) ||
+        (GNUNET_OK !=
+         GNUNET_BIO_read_int64 (
+           rh,
+           "contents size",
+           (int64_t *) &ret->data.dir.contents_size)) ||
+        (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) ||
+        (GNUNET_OK !=
+         GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) ||
+        (GNUNET_OK !=
+         GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16 * 1024)))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    ret->data.dir.dir_size = (uint32_t) dsize;
+    if (NULL != filename)
+    {
+      ret->data.dir.entries = deserialize_file_information (h, filename);
+      GNUNET_free (filename);
+      filename = NULL;
+      nxt = ret->data.dir.entries;
+      while (NULL != nxt)
+      {
+        nxt->dir = ret;
+        nxt = nxt->next;
+      }
+    }
+    break;
+
+  default:
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  ret->serialization = GNUNET_strdup (fn);
+  if (GNUNET_OK !=
+      GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16 * 1024))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (NULL != filename)
+  {
+    ret->next = deserialize_file_information (h, filename);
+    GNUNET_free (filename);
+    filename = NULL;
+  }
+  GNUNET_free (ksks);
+  GNUNET_free (skss);
+  GNUNET_free (chks);
+  return ret;
+cleanup:
+  GNUNET_free (ksks);
+  GNUNET_free (chks);
+  GNUNET_free (skss);
+  GNUNET_free (filename);
+  GNUNET_FS_file_information_destroy (ret, NULL, NULL);
+  return NULL;
+}
+
+
+/**
+ * Using the given serialization filename, try to deserialize
+ * the file-information tree associated with it.
+ *
+ * @param h master context
+ * @param filename name of the file (without directory) with
+ *        the information
+ * @return NULL on error
+ */
+static struct GNUNET_FS_FileInformation *
+deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename)
+{
+  struct GNUNET_FS_FileInformation *ret;
+  struct GNUNET_BIO_ReadHandle *rh;
+  char *emsg;
+  char *fn;
+
+  rh = get_read_handle (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename);
+  if (NULL == rh)
+    return NULL;
+  ret = deserialize_fi_node (h, filename, rh);
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to resume publishing information `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  if (NULL == ret)
+  {
+    fn =
+      get_serialization_file_name (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename);
+    if (NULL != fn)
+    {
+      if (0 != unlink (fn))
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
+      GNUNET_free (fn);
+    }
+  }
+  return ret;
+}
+
+
+/**
+ * Given a serialization name (full absolute path), return the
+ * basename of the file (without the path), which must only
+ * consist of the 6 random characters.
+ *
+ * @param fullname name to extract the basename from
+ * @return copy of the basename, NULL on error
+ */
+static char *
+get_serialization_short_name (const char *fullname)
+{
+  const char *end;
+  const char *nxt;
+
+  end = NULL;
+  nxt = fullname;
+  /* FIXME: we could do this faster since we know
+   * the length of 'end'... */
+  while ('\0' != *nxt)
+  {
+    if (DIR_SEPARATOR == *nxt)
+      end = nxt + 1;
+    nxt++;
+  }
+  if ((NULL == end) || (0 == strlen (end)))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  GNUNET_break (6 == strlen (end));
+  return GNUNET_strdup (end);
+}
+
+
+/**
+ * Create a new random name for serialization.  Also checks if persistence
+ * is enabled and returns NULL if not.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @return NULL on error
+ */
+static char *
+make_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext)
+{
+  char *fn;
+  char *dn;
+  char *ret;
+
+  if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE))
+    return NULL; /* persistence not requested */
+  dn = get_serialization_file_name (h, ext, "");
+  if (NULL == dn)
+    return NULL;
+  if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn))
+  {
+    GNUNET_free (dn);
+    return NULL;
+  }
+  fn = GNUNET_DISK_mktemp (dn);
+  GNUNET_free (dn);
+  if (NULL == fn)
+    return NULL; /* epic fail */
+  ret = get_serialization_short_name (fn);
+  GNUNET_free (fn);
+  return ret;
+}
+
+
+/**
+ * Create a new random name for serialization.  Also checks if persistence
+ * is enabled and returns NULL if not.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni name of parent
+ * @return NULL on error
+ */
+static char *
+make_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h,
+                                     const char *ext,
+                                     const char *uni)
+{
+  char *fn;
+  char *dn;
+  char *ret;
+
+  if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE))
+    return NULL; /* persistence not requested */
+  dn = get_serialization_file_name_in_dir (h, ext, uni, "");
+  if (NULL == dn)
+    return NULL;
+  if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn))
+  {
+    GNUNET_free (dn);
+    return NULL;
+  }
+  fn = GNUNET_DISK_mktemp (dn);
+  GNUNET_free (dn);
+  if (NULL == fn)
+    return NULL; /* epic fail */
+  ret = get_serialization_short_name (fn);
+  GNUNET_free (fn);
+  return ret;
+}
+
+
+/**
+ * Copy all of the data from the reader to the write handle.
+ *
+ * @param wh write handle
+ * @param fi file with reader
+ * @return #GNUNET_OK on success
+ */
+static int
+copy_from_reader (struct GNUNET_BIO_WriteHandle *wh,
+                  struct GNUNET_FS_FileInformation *fi)
+{
+  char buf[32 * 1024];
+  uint64_t off;
+  size_t ret;
+  size_t left;
+  char *emsg;
+
+  emsg = NULL;
+  off = 0;
+  while (off < fi->data.file.file_size)
+  {
+    left = GNUNET_MIN (sizeof(buf), fi->data.file.file_size - off);
+    ret =
+      fi->data.file.reader (fi->data.file.reader_cls, off, left, buf, &emsg);
+    if (0 == ret)
+    {
+      GNUNET_free (emsg);
+      return GNUNET_SYSERR;
+    }
+    if (GNUNET_OK != GNUNET_BIO_write (wh, "copied from reader", buf, ret))
+      return GNUNET_SYSERR;
+    off += ret;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Create a temporary file on disk to store the current
+ * state of @a fi in.
+ *
+ * @param fi file information to sync with disk
+ */
+void
+GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *fi)
+{
+  char *fn;
+  struct GNUNET_BIO_WriteHandle *wh;
+  char b;
+  char *ksks;
+  char *chks;
+  char *skss;
+
+  if (NULL == fi->serialization)
+    fi->serialization =
+      make_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO);
+  if (NULL == fi->serialization)
+    return;
+  wh =
+    get_write_handle (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, fi->serialization);
+  if (NULL == wh)
+  {
+    GNUNET_free (fi->serialization);
+    fi->serialization = NULL;
+    return;
+  }
+  if (GNUNET_YES == fi->is_directory)
+    b = 4;
+  else if (GNUNET_YES == fi->data.file.index_start_confirmed)
+    b = 3;
+  else if (GNUNET_YES == fi->data.file.have_hash)
+    b = 2;
+  else if (GNUNET_YES == fi->data.file.do_index)
+    b = 1;
+  else
+    b = 0;
+  if (NULL != fi->keywords)
+    ksks = GNUNET_FS_uri_to_string (fi->keywords);
+  else
+    ksks = NULL;
+  if (NULL != fi->chk_uri)
+    chks = GNUNET_FS_uri_to_string (fi->chk_uri);
+  else
+    chks = NULL;
+  if (NULL != fi->sks_uri)
+    skss = GNUNET_FS_uri_to_string (fi->sks_uri);
+  else
+    skss = NULL;
+  struct GNUNET_BIO_WriteSpec ws1[] = {
+    GNUNET_BIO_write_spec_object ("b", &b, sizeof (b)),
+    GNUNET_FS_write_spec_meta_data ("meta", fi->meta),
+    GNUNET_BIO_write_spec_string ("ksks", ksks),
+    GNUNET_BIO_write_spec_string ("chks", chks),
+    GNUNET_BIO_write_spec_string ("skss", skss),
+    GNUNET_BIO_write_spec_end (),
+  };
+  struct GNUNET_BIO_WriteSpec ws2[] = {
+    GNUNET_BIO_write_spec_string ("emsg", fi->emsg),
+    GNUNET_BIO_write_spec_string ("filename", fi->filename),
+    GNUNET_BIO_write_spec_int64 (
+      "expiration time",
+      (int64_t *) &fi->bo.expiration_time.abs_value_us),
+    GNUNET_BIO_write_spec_int32 (
+      "anonymity level",
+      (int32_t *) &fi->bo.anonymity_level),
+    GNUNET_BIO_write_spec_int32 (
+      "content priority",
+      (int32_t *) &fi->bo.content_priority),
+    GNUNET_BIO_write_spec_int32 (
+      "replication level",
+      (int32_t *) &fi->bo.replication_level),
+    GNUNET_BIO_write_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) ||
+      (GNUNET_OK != write_start_time (wh, fi->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  GNUNET_free (chks);
+  chks = NULL;
+  GNUNET_free (ksks);
+  ksks = NULL;
+  GNUNET_free (skss);
+  skss = NULL;
+
+  switch (b)
+  {
+  case 0:   /* file-insert */
+    if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size",
+                                             fi->data.file.file_size))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if ((GNUNET_NO == fi->is_published) && (NULL == fi->filename))
+      if (GNUNET_OK != copy_from_reader (wh, fi))
+      {
+        GNUNET_break (0);
+        goto cleanup;
+      }
+    break;
+
+  case 1:   /* file-index, no hash */
+    if (NULL == fi->filename)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size",
+                                             fi->data.file.file_size))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    break;
+
+  case 2:   /* file-index-with-hash */
+  case 3:   /* file-index-with-hash-confirmed */
+    if (NULL == fi->filename)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    if ((GNUNET_OK != GNUNET_BIO_write_int64 (wh, "file size",
+                                              fi->data.file.file_size)) ||
+        (GNUNET_OK != GNUNET_BIO_write (wh,
+                                        "file id",
+                                        &fi->data.file.file_id,
+                                        sizeof(struct GNUNET_HashCode))))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    break;
+
+  case 4:   /* directory */
+    if ((NULL != fi->data.dir.entries) &&
+        (NULL == fi->data.dir.entries->serialization))
+      GNUNET_FS_file_information_sync_ (fi->data.dir.entries);
+    struct GNUNET_BIO_WriteSpec ws[] = {
+      GNUNET_BIO_write_spec_int32 ("dir size",
+                                   (int32_t *) &fi->data.dir.dir_size),
+      GNUNET_BIO_write_spec_int64 (
+        "contents completed",
+        (int64_t *) &fi->data.dir.contents_completed),
+      GNUNET_BIO_write_spec_int64 ("contents size",
+                                   (int64_t *) &fi->data.dir.contents_size),
+      GNUNET_BIO_write_spec_object ("dir data",
+                                    fi->data.dir.dir_data,
+                                    (uint32_t) fi->data.dir.dir_size),
+      GNUNET_BIO_write_spec_string ("dir entries",
+                                    (fi->data.dir.entries == NULL)
+                                    ? NULL
+                                    : fi->data.dir.entries->serialization),
+      GNUNET_BIO_write_spec_end (),
+    };
+    if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    break;
+
+  default:
+    GNUNET_assert (0);
+    goto cleanup;
+  }
+  if ((NULL != fi->next) && (NULL == fi->next->serialization))
+    GNUNET_FS_file_information_sync_ (fi->next);
+  if (GNUNET_OK != GNUNET_BIO_write_string (wh,
+                                            "serialization",
+                                            (fi->next != NULL)
+                                            ? fi->next->serialization
+                                            : NULL))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  return; /* done! */
+cleanup:
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  GNUNET_free (chks);
+  GNUNET_free (ksks);
+  GNUNET_free (skss);
+  fn = get_serialization_file_name (fi->h,
+                                    GNUNET_FS_SYNC_PATH_FILE_INFO,
+                                    fi->serialization);
+  if (NULL != fn)
+  {
+    if (0 != unlink (fn))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
+    GNUNET_free (fn);
+  }
+  GNUNET_free (fi->serialization);
+  fi->serialization = NULL;
+}
+
+
+/**
+ * Find the entry in the file information struct where the
+ * serialization filename matches the given name.
+ *
+ * @param pos file information to search
+ * @param srch filename to search for
+ * @return NULL if srch was not found in this subtree
+ */
+static struct GNUNET_FS_FileInformation *
+find_file_position (struct GNUNET_FS_FileInformation *pos, const char *srch)
+{
+  struct GNUNET_FS_FileInformation *r;
+
+  while (NULL != pos)
+  {
+    if (0 == strcmp (srch, pos->serialization))
+      return pos;
+    if ((GNUNET_YES == pos->is_directory) &&
+        (NULL != (r = find_file_position (pos->data.dir.entries, srch))))
+      return r;
+    pos = pos->next;
+  }
+  return NULL;
+}
+
+
+/**
+ * Signal the FS's progress function that we are resuming
+ * an upload.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`, for the parent (!))
+ * @param fi the entry in the publish-structure
+ * @param length length of the file or directory
+ * @param meta metadata for the file or directory (can be modified)
+ * @param uri pointer to the keywords that will be used for this entry (can be modified)
+ * @param bo block options (can be modified)
+ * @param do_index should we index?
+ * @param client_info pointer to client context set upon creation (can be modified)
+ * @return #GNUNET_OK to continue (always)
+ */
+static int
+fip_signal_resume (void *cls,
+                   struct GNUNET_FS_FileInformation *fi,
+                   uint64_t length,
+                   struct GNUNET_FS_MetaData *meta,
+                   struct GNUNET_FS_Uri **uri,
+                   struct GNUNET_FS_BlockOptions *bo,
+                   int *do_index,
+                   void **client_info)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (GNUNET_YES == pc->skip_next_fi_callback)
+  {
+    pc->skip_next_fi_callback = GNUNET_NO;
+    return GNUNET_OK;
+  }
+  pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME;
+  pi.value.publish.specifics.resume.message = fi->emsg;
+  pi.value.publish.specifics.resume.chk_uri = fi->chk_uri;
+  *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
+  if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta))
+  {
+    /* process entries in directory */
+    pc->skip_next_fi_callback = GNUNET_YES;
+    GNUNET_FS_file_information_inspect (fi, &fip_signal_resume, pc);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called with a filename of serialized publishing operation
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_Handle *`
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_publish_file (void *cls, const char *filename)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  struct GNUNET_BIO_ReadHandle *rh;
+  struct GNUNET_FS_PublishContext *pc;
+  int32_t options;
+  int32_t all_done;
+  int32_t have_ns;
+  char *fi_root;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey ns;
+  char *fi_pos;
+  char *emsg;
+
+  pc = GNUNET_new (struct GNUNET_FS_PublishContext);
+  pc->h = h;
+  pc->serialization = get_serialization_short_name (filename);
+  fi_root = NULL;
+  fi_pos = NULL;
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  struct GNUNET_BIO_ReadSpec rs[] = {
+    GNUNET_BIO_read_spec_string ("publish-nid", &pc->nid, 1024),
+    GNUNET_BIO_read_spec_string ("publish-nuid", &pc->nuid, 1024),
+    GNUNET_BIO_read_spec_int32 ("options", &options),
+    GNUNET_BIO_read_spec_int32 ("all done", &all_done),
+    GNUNET_BIO_read_spec_int32 ("have ns", &have_ns),
+    GNUNET_BIO_read_spec_string ("publish-firoot", &fi_root, 128),
+    GNUNET_BIO_read_spec_string ("publish-fipos", &fi_pos, 128),
+    GNUNET_BIO_read_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) ||
+      ((GNUNET_YES == have_ns) &&
+       (GNUNET_OK != GNUNET_BIO_read (rh, "publish-ns", &ns, sizeof(ns)))))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  pc->options = options;
+  pc->all_done = all_done;
+  if (NULL == fi_root)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  pc->fi = deserialize_file_information (h, fi_root);
+  if (NULL == pc->fi)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_YES == have_ns)
+  {
+    pc->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey);
+    *pc->ns = ns;
+  }
+  if ((0 == (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) &&
+      (GNUNET_YES != pc->all_done))
+  {
+    pc->dsh = GNUNET_DATASTORE_connect (h->cfg);
+    if (NULL == pc->dsh)
+      goto cleanup;
+  }
+  if (NULL != fi_pos)
+  {
+    pc->fi_pos = find_file_position (pc->fi, fi_pos);
+    GNUNET_free (fi_pos);
+    fi_pos = NULL;
+    if (NULL == pc->fi_pos)
+    {
+      /* failed to find position for resuming, outch! Will start from root! */
+      GNUNET_break (0);
+      if (GNUNET_YES != pc->all_done)
+        pc->fi_pos = pc->fi;
+    }
+  }
+  GNUNET_free (fi_root);
+  fi_root = NULL;
+  /* generate RESUME event(s) */
+  GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_resume, pc);
+
+  /* re-start publishing (if needed)... */
+  if (GNUNET_YES != pc->all_done)
+  {
+    GNUNET_assert (NULL == pc->upload_task);
+    pc->upload_task =
+      GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                          &GNUNET_FS_publish_main_,
+                                          pc);
+  }
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failure while resuming publishing operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc);
+  return GNUNET_OK;
+cleanup:
+  GNUNET_free (pc->nid);
+  GNUNET_free (pc->nuid);
+  GNUNET_free (fi_root);
+  GNUNET_free (fi_pos);
+  if ((NULL != rh) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to resume publishing operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  if (NULL != pc->fi)
+    GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
+  if (0 != unlink (filename))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename);
+  GNUNET_free (pc->serialization);
+  GNUNET_free (pc);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Synchronize this publishing struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param pc the struct to sync
+ */
+void
+GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  int32_t have_ns;
+
+  if (NULL == pc->serialization)
+    pc->serialization =
+      make_serialization_file_name (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH);
+  if (NULL == pc->serialization)
+    return;
+  if (NULL == pc->fi)
+    return;
+  if (NULL == pc->fi->serialization)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  wh = get_write_handle (pc->h,
+                         GNUNET_FS_SYNC_PATH_MASTER_PUBLISH,
+                         pc->serialization);
+  if (NULL == wh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  have_ns = (NULL != pc->ns) ? GNUNET_YES : GNUNET_NO;
+  struct GNUNET_BIO_WriteSpec ws[] = {
+    GNUNET_BIO_write_spec_string ("nid", pc->nid),
+    GNUNET_BIO_write_spec_string ("nuid", pc->nuid),
+    GNUNET_BIO_write_spec_int32 ("options", (int32_t *) &pc->options),
+    GNUNET_BIO_write_spec_int32 ("all done", &pc->all_done),
+    GNUNET_BIO_write_spec_int32 ("have ns", &have_ns),
+    GNUNET_BIO_write_spec_string ("serialization", pc->fi->serialization),
+    GNUNET_BIO_write_spec_string ("pos serialization", (NULL == pc->fi_pos)
+                                  ? NULL
+                                  : pc->fi_pos->serialization),
+    GNUNET_BIO_read_spec_end ()
+  };
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)) ||
+      ((NULL != pc->ns) &&
+       (GNUNET_OK !=
+        GNUNET_BIO_write (wh,
+                          "ns",
+                          pc->ns,
+                          sizeof(struct GNUNET_CRYPTO_EcdsaPrivateKey)))))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  return;
+cleanup:
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  GNUNET_FS_remove_sync_file_ (pc->h,
+                               GNUNET_FS_SYNC_PATH_MASTER_PUBLISH,
+                               pc->serialization);
+  GNUNET_free (pc->serialization);
+  pc->serialization = NULL;
+}
+
+
+/**
+ * Synchronize this unindex struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param uc the struct to sync
+ */
+void
+GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *uris;
+
+  if (NULL == uc->serialization)
+    uc->serialization =
+      make_serialization_file_name (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX);
+  if (NULL == uc->serialization)
+    return;
+  wh = get_write_handle (uc->h,
+                         GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                         uc->serialization);
+  if (NULL == wh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (NULL != uc->ksk_uri)
+    uris = GNUNET_FS_uri_to_string (uc->ksk_uri);
+  else
+    uris = NULL;
+  struct GNUNET_BIO_WriteSpec ws1[] = {
+    GNUNET_BIO_write_spec_string ("filename", uc->filename),
+    GNUNET_BIO_write_spec_int64 ("file size", (int64_t *) &uc->file_size),
+    GNUNET_BIO_write_spec_end (),
+  };
+  struct GNUNET_BIO_WriteSpec ws2[] = {
+    GNUNET_BIO_write_spec_int32 ("state", (int32_t *) &uc->state),
+    GNUNET_BIO_write_spec_object ("hashkey", &uc->chk,
+                                  sizeof (struct ContentHashKey)),
+    GNUNET_BIO_write_spec_string ("uris", uris),
+    GNUNET_BIO_write_spec_int32 ("ksk offset", (int32_t *) &uc->ksk_offset),
+    GNUNET_BIO_write_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) ||
+      (GNUNET_OK != write_start_time (wh, uc->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2)) ||
+      ((uc->state == UNINDEX_STATE_FS_NOTIFY) &&
+       (GNUNET_OK != GNUNET_BIO_write (wh,
+                                       "file id",
+                                       &uc->file_id,
+                                       sizeof(struct GNUNET_HashCode)))) ||
+      ((uc->state == UNINDEX_STATE_ERROR) &&
+       (GNUNET_OK != GNUNET_BIO_write_string (wh, "emsg", uc->emsg))))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  return;
+cleanup:
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  GNUNET_FS_remove_sync_file_ (uc->h,
+                               GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                               uc->serialization);
+  GNUNET_free (uc->serialization);
+  uc->serialization = NULL;
+}
+
+
+/**
+ * Serialize a download request.
+ *
+ * @param wh handle for writing the download request to disk
+ * @param dr the the request to write to disk
+ * @return #GNUNET_YES on success, #GNUNET_NO on error
+ */
+static int
+write_download_request (struct GNUNET_BIO_WriteHandle *wh,
+                        struct DownloadRequest *dr)
+{
+  unsigned int i;
+  struct GNUNET_BIO_WriteSpec ws[] = {
+    GNUNET_BIO_write_spec_int32 ("state", (int32_t *) &dr->state),
+    GNUNET_BIO_write_spec_int64 ("offset", (int64_t *) &dr->offset),
+    GNUNET_BIO_write_spec_int32 ("num children", (int32_t *) &dr->num_children),
+    GNUNET_BIO_write_spec_int32 ("depth", (int32_t *) &dr->depth),
+    GNUNET_BIO_write_spec_end (),
+  };
+
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)))
+    return GNUNET_NO;
+  if ((BRS_CHK_SET == dr->state) &&
+      (GNUNET_OK !=
+       GNUNET_BIO_write (wh, "hashkey",
+                         &dr->chk, sizeof(struct ContentHashKey))))
+    return GNUNET_NO;
+  for (i = 0; i < dr->num_children; i++)
+    if (GNUNET_NO == write_download_request (wh, dr->children[i]))
+      return GNUNET_NO;
+  return GNUNET_YES;
+}
+
+
+/**
+ * Read a download request tree.
+ *
+ * @param rh cadet to read from
+ * @return value the download request read from disk, NULL on error
+ */
+static struct DownloadRequest *
+read_download_request (struct GNUNET_BIO_ReadHandle *rh)
+{
+  struct DownloadRequest *dr;
+  unsigned int i;
+
+  dr = GNUNET_new (struct DownloadRequest);
+  struct GNUNET_BIO_ReadSpec rs[] = {
+    GNUNET_BIO_read_spec_int32 ("state", (int32_t *) &dr->state),
+    GNUNET_BIO_read_spec_int64 ("offset", (int64_t *) &dr->offset),
+    GNUNET_BIO_read_spec_int32 ("num children", (int32_t *) &dr->num_children),
+    GNUNET_BIO_read_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) ||
+      (dr->num_children > CHK_PER_INODE) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "depth",
+                                           (int32_t *) &dr->depth)) ||
+      ((0 == dr->depth) && (dr->num_children > 0)) ||
+      ((dr->depth > 0) && (0 == dr->num_children)))
+  {
+    GNUNET_break (0);
+    dr->num_children = 0;
+    goto cleanup;
+  }
+  if (dr->num_children > 0)
+    dr->children =
+      GNUNET_malloc (dr->num_children * sizeof(struct DownloadRequest *));
+  switch (dr->state)
+  {
+  case BRS_INIT:
+  case BRS_RECONSTRUCT_DOWN:
+  case BRS_RECONSTRUCT_META_UP:
+  case BRS_RECONSTRUCT_UP:
+    break;
+
+  case BRS_CHK_SET:
+    if (GNUNET_OK !=
+        GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof(struct ContentHashKey)))
+      goto cleanup;
+    break;
+
+  case BRS_DOWNLOAD_DOWN:
+  case BRS_DOWNLOAD_UP:
+  case BRS_ERROR:
+    break;
+
+  default:
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  for (i = 0; i < dr->num_children; i++)
+  {
+    if (NULL == (dr->children[i] = read_download_request (rh)))
+      goto cleanup;
+    dr->children[i]->parent = dr;
+  }
+  return dr;
+cleanup:
+  GNUNET_FS_free_download_request_ (dr);
+  return NULL;
+}
+
+
+/**
+ * Compute the name of the sync file (or directory) for the given download
+ * context.
+ *
+ * @param dc download context to compute for
+ * @param uni unique filename to use, use "" for the directory name
+ * @param ext extension to use, use ".dir" for our own subdirectory
+ * @return the expanded file name, NULL for none
+ */
+static char *
+get_download_sync_filename (struct GNUNET_FS_DownloadContext *dc,
+                            const char *uni,
+                            const char *ext)
+{
+  char *par;
+  char *epar;
+
+  if (dc->parent == NULL)
+    return get_serialization_file_name (dc->h,
+                                        (dc->search != NULL)
+                                        ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
+                                        : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
+                                        uni);
+  if (NULL == dc->parent->serialization)
+    return NULL;
+  par = get_download_sync_filename (dc->parent, dc->parent->serialization, "");
+  if (NULL == par)
+    return NULL;
+  GNUNET_asprintf (&epar, "%s.dir%s%s%s", par, DIR_SEPARATOR_STR, uni, ext);
+  GNUNET_free (par);
+  return epar;
+}
+
+
+/**
+ * Synchronize this download struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param dc the struct to sync
+ */
+void
+GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *uris;
+  char *fn;
+  char *dir;
+
+  if (0 != (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
+    return; /* we don't sync probes */
+  if (NULL == dc->serialization)
+  {
+    dir = get_download_sync_filename (dc, "", "");
+    if (NULL == dir)
+      return;
+    if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir))
+    {
+      GNUNET_free (dir);
+      return;
+    }
+    fn = GNUNET_DISK_mktemp (dir);
+    GNUNET_free (dir);
+    if (NULL == fn)
+      return;
+    dc->serialization = get_serialization_short_name (fn);
+  }
+  else
+  {
+    fn = get_download_sync_filename (dc, dc->serialization, "");
+    if (NULL == fn)
+    {
+      GNUNET_free (dc->serialization);
+      dc->serialization = NULL;
+      GNUNET_free (fn);
+      return;
+    }
+  }
+  wh = GNUNET_BIO_write_open_file (fn);
+  if (NULL == wh)
+  {
+    GNUNET_free (dc->serialization);
+    dc->serialization = NULL;
+    GNUNET_free (fn);
+    return;
+  }
+  GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) ||
+                 (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri)));
+  uris = GNUNET_FS_uri_to_string (dc->uri);
+  struct GNUNET_BIO_WriteSpec ws1[] = {
+    GNUNET_BIO_write_spec_string ("uris", uris),
+    GNUNET_FS_write_spec_meta_data ("metadata", dc->meta),
+    GNUNET_BIO_write_spec_string ("emsg", dc->emsg),
+    GNUNET_BIO_write_spec_string ("filename", dc->filename),
+    GNUNET_BIO_write_spec_string ("temp filename", dc->temp_filename),
+    GNUNET_BIO_write_spec_int64 ("old file size",
+                                 (int64_t *) &dc->old_file_size),
+    GNUNET_BIO_write_spec_int64 ("offset", (int64_t *) &dc->offset),
+    GNUNET_BIO_write_spec_int64 ("length", (int64_t *) &dc->length),
+    GNUNET_BIO_write_spec_int64 ("completed", (int64_t *) &dc->completed),
+    GNUNET_BIO_write_spec_end (),
+  };
+  struct GNUNET_BIO_WriteSpec ws2[] = {
+    GNUNET_BIO_write_spec_int32 ("anonymity", (int32_t *) &dc->anonymity),
+    GNUNET_BIO_write_spec_int32 ("options", (int32_t *) &dc->options),
+    GNUNET_BIO_write_spec_int32 ("has finished", (int32_t *) &dc->has_finished),
+    GNUNET_BIO_write_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws1)) ||
+      (GNUNET_OK != write_start_time (wh, dc->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws2)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (NULL == dc->emsg)
+  {
+    GNUNET_assert (dc->top_request != NULL);
+    if (GNUNET_YES != write_download_request (wh, dc->top_request))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+  }
+  GNUNET_free (uris);
+  uris = NULL;
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  GNUNET_free (fn);
+  return;
+cleanup:
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  GNUNET_free (uris);
+  if (0 != unlink (fn))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
+  GNUNET_free (fn);
+  GNUNET_free (dc->serialization);
+  dc->serialization = NULL;
+}
+
+
+/**
+ * Synchronize this search result with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param sr the struct to sync
+ */
+void
+GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *uris;
+
+  if (NULL == sr->sc)
+    return;
+  uris = NULL;
+  if (NULL == sr->serialization)
+    sr->serialization =
+      make_serialization_file_name_in_dir (sr->h,
+                                           (sr->sc->psearch_result == NULL)
+                                           ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                                           : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                                           sr->sc->serialization);
+  if (NULL == sr->serialization)
+    return;
+  wh = get_write_handle_in_dir (sr->h,
+                                (sr->sc->psearch_result == NULL)
+                                ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                                : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                                sr->sc->serialization,
+                                sr->serialization);
+  if (NULL == wh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  uris = GNUNET_FS_uri_to_string (sr->uri);
+  struct GNUNET_BIO_WriteSpec ws[] = {
+    GNUNET_BIO_write_spec_string ("uris", uris),
+    GNUNET_BIO_write_spec_string ("download serialization",
+                                  (sr->download != NULL)
+                                  ? sr->download->serialization
+                                  : NULL),
+    GNUNET_BIO_write_spec_string ("update search serialization",
+                                  (sr->update_search != NULL)
+                                  ? sr->update_search->serialization
+                                  : NULL),
+    GNUNET_FS_write_spec_meta_data ("metadata", sr->meta),
+    GNUNET_BIO_write_spec_object ("key", &sr->key,
+                                  sizeof(struct GNUNET_HashCode)),
+    GNUNET_BIO_write_spec_int32 ("mandatory missing",
+                                 (int32_t *) &sr->mandatory_missing),
+    GNUNET_BIO_write_spec_int32 ("optional support",
+                                 (int32_t *) &sr->optional_support),
+    GNUNET_BIO_write_spec_int32 ("availability success",
+                                 (int32_t *) &sr->availability_success),
+    GNUNET_BIO_write_spec_int32 ("availability trials",
+                                 (int32_t *) &sr->availability_trials),
+    GNUNET_BIO_write_spec_end (),
+  };
+  if ((GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if ((NULL != sr->uri) && (GNUNET_FS_URI_KSK == sr->sc->uri->type) &&
+      (GNUNET_OK !=
+       GNUNET_BIO_write (wh,
+                         "keyword bitmap",
+                         sr->keyword_bitmap,
+                         (sr->sc->uri->data.ksk.keywordCount + 7) / 8)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  GNUNET_free (uris);
+  return;
+cleanup:
+  GNUNET_free (uris);
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  remove_sync_file_in_dir (sr->h,
+                           (NULL == sr->sc->psearch_result)
+                           ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                           : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                           sr->sc->serialization,
+                           sr->serialization);
+  GNUNET_free (sr->serialization);
+  sr->serialization = NULL;
+}
+
+
+/**
+ * Synchronize this search struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param sc the struct to sync
+ */
+void
+GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *uris;
+  char in_pause;
+  const char *category;
+
+  category = (NULL == sc->psearch_result) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+             : GNUNET_FS_SYNC_PATH_CHILD_SEARCH;
+  if (NULL == sc->serialization)
+    sc->serialization = make_serialization_file_name (sc->h, category);
+  if (NULL == sc->serialization)
+    return;
+  uris = NULL;
+  wh = get_write_handle (sc->h, category, sc->serialization);
+  if (NULL == wh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) ||
+                 (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri)));
+  uris = GNUNET_FS_uri_to_string (sc->uri);
+  in_pause = (sc->task != NULL) ? 'r' : '\0';
+  if ((GNUNET_OK != GNUNET_BIO_write_string (wh, "uris", uris)) ||
+      (GNUNET_OK != write_start_time (wh, sc->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_write_string (wh, "emsg", sc->emsg)) ||
+      (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "options",
+                                            (uint32_t) sc->options)) ||
+      (GNUNET_OK != GNUNET_BIO_write (wh, "in pause",
+                                      &in_pause, sizeof(in_pause))) ||
+      (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "anonymity", sc->anonymity)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  GNUNET_free (uris);
+  uris = NULL;
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    wh = NULL;
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  return;
+cleanup:
+  if (NULL != wh)
+    (void) GNUNET_BIO_write_close (wh, NULL);
+  GNUNET_free (uris);
+  GNUNET_FS_remove_sync_file_ (sc->h, category, sc->serialization);
+  GNUNET_free (sc->serialization);
+  sc->serialization = NULL;
+}
+
+
+/**
+ * Function called with a filename of serialized unindexing operation
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_Handle *`
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_unindex_file (void *cls, const char *filename)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  struct GNUNET_BIO_ReadHandle *rh;
+  struct GNUNET_FS_UnindexContext *uc;
+  struct GNUNET_FS_ProgressInfo pi;
+  char *emsg;
+  char *uris;
+  uint32_t state;
+
+  uc = GNUNET_new (struct GNUNET_FS_UnindexContext);
+  uc->h = h;
+  uc->serialization = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  uris = NULL;
+  if ((GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int64 (rh, "file size",
+                                           (int64_t *) &uc->file_size)) ||
+      (GNUNET_OK != read_start_time (rh, &uc->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "state",
+                                           (int32_t *) &state)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read (rh, "uri", &uc->chk, sizeof(struct ContentHashKey))) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "unindex-kskuri", &uris, 10 * 1024)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "ksk offset",
+                                           (int32_t *) &uc->ksk_offset)))
+  {
+    GNUNET_free (uris);
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (NULL != uris)
+  {
+    uc->ksk_uri = GNUNET_FS_uri_parse (uris, &emsg);
+    GNUNET_free (uris);
+    if (NULL == uc->ksk_uri)
+    {
+      GNUNET_break (0);
+      GNUNET_free (emsg);
+      goto cleanup;
+    }
+  }
+  if ((uc->ksk_offset > 0) &&
+      ((NULL == uc->ksk_uri) ||
+       (uc->ksk_offset > uc->ksk_uri->data.ksk.keywordCount)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  uc->state = (enum UnindexState) state;
+  switch (state)
+  {
+  case UNINDEX_STATE_HASHING:
+    break;
+
+  case UNINDEX_STATE_FS_NOTIFY:
+    if (GNUNET_OK != GNUNET_BIO_read (rh,
+                                      "unindex-hash",
+                                      &uc->file_id,
+                                      sizeof(struct GNUNET_HashCode)))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    break;
+
+  case UNINDEX_STATE_DS_REMOVE:
+  case UNINDEX_STATE_EXTRACT_KEYWORDS:
+  case UNINDEX_STATE_DS_REMOVE_KBLOCKS:
+    break;
+
+  case UNINDEX_STATE_COMPLETE:
+    break;
+
+  case UNINDEX_STATE_ERROR:
+    if (GNUNET_OK !=
+        GNUNET_BIO_read_string (rh, "unindex-emsg", &uc->emsg, 10 * 1024))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+    break;
+
+  default:
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc);
+  pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME;
+  pi.value.unindex.specifics.resume.message = uc->emsg;
+  GNUNET_FS_unindex_make_status_ (&pi,
+                                  uc,
+                                  (uc->state == UNINDEX_STATE_COMPLETE)
+                                  ? uc->file_size
+                                  : 0);
+  switch (uc->state)
+  {
+  case UNINDEX_STATE_HASHING:
+    uc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                       uc->filename,
+                                       HASHING_BLOCKSIZE,
+                                       &GNUNET_FS_unindex_process_hash_,
+                                       uc);
+    break;
+
+  case UNINDEX_STATE_FS_NOTIFY:
+    uc->state = UNINDEX_STATE_HASHING;
+    GNUNET_FS_unindex_process_hash_ (uc, &uc->file_id);
+    break;
+
+  case UNINDEX_STATE_DS_REMOVE:
+    GNUNET_FS_unindex_do_remove_ (uc);
+    break;
+
+  case UNINDEX_STATE_EXTRACT_KEYWORDS:
+    GNUNET_FS_unindex_do_extract_keywords_ (uc);
+    break;
+
+  case UNINDEX_STATE_DS_REMOVE_KBLOCKS:
+    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+    break;
+
+  case UNINDEX_STATE_COMPLETE:
+  case UNINDEX_STATE_ERROR:
+    /* no need to resume any operation, we were done */
+    break;
+
+  default:
+    break;
+  }
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failure while resuming unindexing operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  return GNUNET_OK;
+cleanup:
+  GNUNET_free (uc->filename);
+  if ((NULL != rh) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to resume unindexing operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  if (NULL != uc->serialization)
+    GNUNET_FS_remove_sync_file_ (h,
+                                 GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                                 uc->serialization);
+  GNUNET_free (uc->serialization);
+  GNUNET_free (uc);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Deserialize a download.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param parent parent download
+ * @param search associated search
+ * @param serialization name under which the search was serialized
+ */
+static void
+deserialize_download (struct GNUNET_FS_Handle *h,
+                      struct GNUNET_BIO_ReadHandle *rh,
+                      struct GNUNET_FS_DownloadContext *parent,
+                      struct GNUNET_FS_SearchResult *search,
+                      const char *serialization);
+
+
+/**
+ * Deserialize a search.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param psearch_result parent search result
+ * @param serialization name under which the search was serialized
+ */
+static struct GNUNET_FS_SearchContext *
+deserialize_search (struct GNUNET_FS_Handle *h,
+                    struct GNUNET_BIO_ReadHandle *rh,
+                    struct GNUNET_FS_SearchResult *psearch_result,
+                    const char *serialization);
+
+
+/**
+ * Function called with a filename of serialized search result
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_SearchContext *`
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_search_result (void *cls, const char *filename)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  char *serialized;
+  char *uris;
+  char *emsg;
+  char *download;
+  char *update_srch;
+  struct GNUNET_BIO_ReadHandle *rh;
+  struct GNUNET_BIO_ReadHandle *drh;
+  struct GNUNET_FS_SearchResult *sr;
+
+  serialized = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    if (NULL != serialized)
+    {
+      remove_sync_file_in_dir (sc->h,
+                               (NULL == sc->psearch_result)
+                               ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                               : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                               sc->serialization,
+                               serialized);
+      GNUNET_free (serialized);
+    }
+    return GNUNET_OK;
+  }
+  emsg = NULL;
+  uris = NULL;
+  download = NULL;
+  update_srch = NULL;
+  sr = GNUNET_new (struct GNUNET_FS_SearchResult);
+  sr->h = sc->h;
+  sr->sc = sc;
+  sr->serialization = serialized;
+  if ((GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "result-uri", &uris, 10 * 1024)) ||
+      (NULL == (sr->uri = GNUNET_FS_uri_parse (uris, &emsg))) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "download-lnk", &download, 16)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "search-lnk", &update_srch, 16)) ||
+      (GNUNET_OK != GNUNET_FS_read_meta_data (rh, "result-meta", &sr->meta)) ||
+      (GNUNET_OK != GNUNET_BIO_read (rh,
+                                     "result-key",
+                                     &sr->key,
+                                     sizeof(struct GNUNET_HashCode))) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "mandatory missing",
+         (int32_t *) &sr->mandatory_missing)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "optional support",
+         (int32_t *) &sr->optional_support)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "availability success",
+         (int32_t *) &sr->availability_success)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (
+         rh,
+         "availability trials",
+         (int32_t *) &sr->availability_trials)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  if (GNUNET_FS_URI_KSK == sr->sc->uri->type)
+  {
+    sr->keyword_bitmap = GNUNET_malloc (
+      (sr->sc->uri->data.ksk.keywordCount + 7) / 8);   /* round up, count bits */
+    if (GNUNET_OK !=
+        GNUNET_BIO_read (rh,
+                         "keyword-bitmap",
+                         sr->keyword_bitmap,
+                         (sr->sc->uri->data.ksk.keywordCount + 7) / 8))
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+  }
+  GNUNET_free (uris);
+  if (NULL != download)
+  {
+    drh = get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, download);
+    if (NULL != drh)
+    {
+      deserialize_download (sc->h, drh, NULL, sr, download);
+      if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _ ("Failed to resume sub-download `%s': %s\n"),
+                    download,
+                    emsg);
+        GNUNET_free (emsg);
+      }
+    }
+    GNUNET_free (download);
+  }
+  if (NULL != update_srch)
+  {
+    drh =
+      get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_SEARCH, update_srch);
+    if (NULL != drh)
+    {
+      deserialize_search (sc->h, drh, sr, update_srch);
+      if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _ ("Failed to resume sub-search `%s': %s\n"),
+                    update_srch,
+                    emsg);
+        GNUNET_free (emsg);
+      }
+    }
+    GNUNET_free (update_srch);
+  }
+  GNUNET_break (GNUNET_YES == GNUNET_CONTAINER_multihashmap_put (
+                  sc->master_result_map,
+                  &sr->key,
+                  sr,
+                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failure while resuming search operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  return GNUNET_OK;
+cleanup:
+  GNUNET_free (download);
+  GNUNET_free (emsg);
+  GNUNET_free (uris);
+  GNUNET_free (update_srch);
+  if (NULL != sr->uri)
+    GNUNET_FS_uri_destroy (sr->uri);
+  if (NULL != sr->meta)
+    GNUNET_FS_meta_data_destroy (sr->meta);
+  GNUNET_free (sr->serialization);
+  GNUNET_free (sr);
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failure while resuming search operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Send the 'resume' signal to the callback; also actually
+ * resume the download (put it in the queue).  Does this
+ * recursively for the top-level download and all child
+ * downloads.
+ *
+ * @param dc download to resume
+ */
+static void
+signal_download_resume (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_FS_DownloadContext *dcc;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME;
+  pi.value.download.specifics.resume.meta = dc->meta;
+  pi.value.download.specifics.resume.message = dc->emsg;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+  dcc = dc->child_head;
+  while (NULL != dcc)
+  {
+    signal_download_resume (dcc);
+    dcc = dcc->next;
+  }
+}
+
+
+/**
+ * Signal resuming of a search to our clients (for the
+ * top level search and all sub-searches).
+ *
+ * @param sc search being resumed
+ */
+static void
+signal_search_resume (struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * Iterator over search results signaling resume to the client for
+ * each result.
+ *
+ * @param cls closure, the `struct GNUNET_FS_SearchContext *`
+ * @param key current key code
+ * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *`
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+signal_result_resume (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (0 == sr->mandatory_missing)
+  {
+    pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT;
+    pi.value.search.specifics.resume_result.meta = sr->meta;
+    pi.value.search.specifics.resume_result.uri = sr->uri;
+    pi.value.search.specifics.resume_result.result = sr;
+    pi.value.search.specifics.resume_result.availability_rank =
+      2 * sr->availability_success - sr->availability_trials;
+    pi.value.search.specifics.resume_result.availability_certainty =
+      sr->availability_trials;
+    pi.value.search.specifics.resume_result.applicability_rank =
+      sr->optional_support;
+    sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  }
+  if (NULL != sr->download)
+  {
+    signal_download_resume (sr->download);
+  }
+  else
+  {
+    GNUNET_FS_search_start_probe_ (sr);
+  }
+  if (NULL != sr->update_search)
+    signal_search_resume (sr->update_search);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Free memory allocated by the search context and its children
+ *
+ * @param sc search context to free
+ */
+static void
+free_search_context (struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * Iterator over search results freeing each.
+ *
+ * @param cls closure, the `struct GNUNET_FS_SearchContext *`
+ * @param key current key code
+ * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *`
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+free_result (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (NULL != sr->update_search)
+  {
+    free_search_context (sr->update_search);
+    GNUNET_assert (NULL == sr->update_search);
+  }
+  GNUNET_FS_meta_data_destroy (sr->meta);
+  GNUNET_FS_uri_destroy (sr->uri);
+  GNUNET_free (sr);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Free memory allocated by the search context and its children
+ *
+ * @param sc search context to free
+ */
+static void
+free_search_context (struct GNUNET_FS_SearchContext *sc)
+{
+  if (NULL != sc->serialization)
+  {
+    GNUNET_FS_remove_sync_file_ (sc->h,
+                                 (sc->psearch_result == NULL)
+                                 ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                                 : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                                 sc->serialization);
+    GNUNET_FS_remove_sync_dir_ (sc->h,
+                                (sc->psearch_result == NULL)
+                                ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                                : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                                sc->serialization);
+  }
+  GNUNET_free (sc->serialization);
+  GNUNET_free (sc->emsg);
+  if (NULL != sc->uri)
+    GNUNET_FS_uri_destroy (sc->uri);
+  if (NULL != sc->master_result_map)
+  {
+    GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                           &free_result,
+                                           sc);
+    GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
+  }
+  GNUNET_free (sc);
+}
+
+
+/**
+ * Function called with a filename of serialized sub-download
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_DownloadContext *` (parent)
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_subdownload (void *cls, const char *filename)
+{
+  struct GNUNET_FS_DownloadContext *parent = cls;
+  char *serialized;
+  char *emsg;
+  struct GNUNET_BIO_ReadHandle *rh;
+
+  serialized = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Failed to resume sub-download `%s': could not open file `%s'\n"),
+                serialized,
+                filename);
+    GNUNET_free (serialized);
+    return GNUNET_OK;
+  }
+  deserialize_download (parent->h, rh, parent, NULL, serialized);
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to resume sub-download `%s': %s\n"),
+                serialized,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  GNUNET_free (serialized);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Free this download context and all of its descendants.
+ * (only works during deserialization since not all possible
+ * state it taken care of).
+ *
+ * @param dc context to free
+ */
+static void
+free_download_context (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_FS_DownloadContext *dcc;
+
+  if (NULL != dc->meta)
+    GNUNET_FS_meta_data_destroy (dc->meta);
+  if (NULL != dc->uri)
+    GNUNET_FS_uri_destroy (dc->uri);
+  GNUNET_free (dc->temp_filename);
+  GNUNET_free (dc->emsg);
+  GNUNET_free (dc->filename);
+  GNUNET_free (dc->serialization);
+  while (NULL != (dcc = dc->child_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (dc->child_head, dc->child_tail, dcc);
+    free_download_context (dcc);
+  }
+  GNUNET_FS_free_download_request_ (dc->top_request);
+  if (NULL != dc->active)
+    GNUNET_CONTAINER_multihashmap_destroy (dc->active);
+  GNUNET_free (dc);
+}
+
+
+/**
+ * Deserialize a download.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param parent parent download
+ * @param search associated search
+ * @param serialization name under which the search was serialized
+ */
+static void
+deserialize_download (struct GNUNET_FS_Handle *h,
+                      struct GNUNET_BIO_ReadHandle *rh,
+                      struct GNUNET_FS_DownloadContext *parent,
+                      struct GNUNET_FS_SearchResult *search,
+                      const char *serialization)
+{
+  struct GNUNET_FS_DownloadContext *dc;
+  char *emsg;
+  char *uris;
+  char *dn;
+  uint32_t options;
+  uint32_t status;
+
+  uris = NULL;
+  emsg = NULL;
+  dc = GNUNET_new (struct GNUNET_FS_DownloadContext);
+  dc->parent = parent;
+  dc->h = h;
+  dc->serialization = GNUNET_strdup (serialization);
+  struct GNUNET_BIO_ReadSpec rs[] = {
+    GNUNET_FS_read_spec_meta_data ("download-meta", &dc->meta),
+    GNUNET_BIO_read_spec_string ("download-emsg", &dc->emsg, 10 * 1024),
+    GNUNET_BIO_read_spec_string ("download-fn", &dc->filename, 10 * 1024),
+    GNUNET_BIO_read_spec_string ("download-tfn",
+                                 &dc->temp_filename, 10 * 1024),
+    GNUNET_BIO_read_spec_int64 ("old file size",
+                                (int64_t *) &dc->old_file_size),
+    GNUNET_BIO_read_spec_int64 ("offset",
+                                (int64_t *) &dc->offset),
+    GNUNET_BIO_read_spec_int64 ("length",
+                                (int64_t *) &dc->length),
+    GNUNET_BIO_read_spec_int64 ("completed",
+                                (int64_t *) &dc->completed),
+    GNUNET_BIO_read_spec_end (),
+  };
+  if ((GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "download-uri", &uris, 10 * 1024)) ||
+      (NULL == (dc->uri = GNUNET_FS_uri_parse (uris, &emsg))) ||
+      ((GNUNET_YES != GNUNET_FS_uri_test_chk (dc->uri)) &&
+       (GNUNET_YES != GNUNET_FS_uri_test_loc (dc->uri))) ||
+      (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs)) ||
+      (GNUNET_OK != read_start_time (rh, &dc->start_time)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "anonymity",
+                                           (int32_t *) &dc->anonymity)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "options",
+                                           (int32_t *) &options)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "status",
+                                           (int32_t *) &status)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  dc->options = (enum GNUNET_FS_DownloadOptions) options;
+  dc->active =
+    GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE),
+                                          GNUNET_NO);
+  dc->has_finished = (int) status;
+  dc->treedepth =
+    GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri));
+  if (GNUNET_FS_uri_test_loc (dc->uri))
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target));
+  if (NULL == dc->emsg)
+  {
+    dc->top_request = read_download_request (rh);
+    if (NULL == dc->top_request)
+    {
+      GNUNET_break (0);
+      goto cleanup;
+    }
+  }
+  dn = get_download_sync_filename (dc, dc->serialization, ".dir");
+  if (NULL != dn)
+  {
+    if (GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES))
+      GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc);
+    GNUNET_free (dn);
+  }
+  if (NULL != parent)
+  {
+    GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc);
+  }
+  if (NULL != search)
+  {
+    dc->search = search;
+    search->download = dc;
+  }
+  if ((NULL == parent) && (NULL == search))
+  {
+    dc->top =
+      GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc);
+    signal_download_resume (dc);
+  }
+  GNUNET_free (uris);
+  GNUNET_assert (NULL == dc->job_queue);
+  dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc);
+  return;
+cleanup:
+  GNUNET_free (uris);
+  GNUNET_free (emsg);
+  free_download_context (dc);
+}
+
+
+/**
+ * Signal resuming of a search to our clients (for the
+ * top level search and all sub-searches).
+ *
+ * @param sc search being resumed
+ */
+static void
+signal_search_resume (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_SEARCH_RESUME;
+  pi.value.search.specifics.resume.message = sc->emsg;
+  pi.value.search.specifics.resume.is_paused =
+    (NULL == sc->mq) ? GNUNET_YES : GNUNET_NO;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &signal_result_resume,
+                                         sc);
+}
+
+
+/**
+ * Deserialize a search.
+ *
+ * @param h overall context
+ * @param rh file to deserialize from
+ * @param psearch_result parent search result
+ * @param serialization name under which the search was serialized
+ */
+static struct GNUNET_FS_SearchContext *
+deserialize_search (struct GNUNET_FS_Handle *h,
+                    struct GNUNET_BIO_ReadHandle *rh,
+                    struct GNUNET_FS_SearchResult *psearch_result,
+                    const char *serialization)
+{
+  struct GNUNET_FS_SearchContext *sc;
+  char *emsg;
+  char *uris;
+  char *dn;
+  uint32_t options;
+  char in_pause;
+
+  if ((NULL != psearch_result) && (NULL != psearch_result->update_search))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  uris = NULL;
+  emsg = NULL;
+  sc = GNUNET_new (struct GNUNET_FS_SearchContext);
+  if (NULL != psearch_result)
+  {
+    sc->psearch_result = psearch_result;
+    psearch_result->update_search = sc;
+  }
+  sc->h = h;
+  sc->serialization = GNUNET_strdup (serialization);
+  if ((GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "search-uri", &uris, 10 * 1024)) ||
+      (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) ||
+      ((GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) &&
+       (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri))) ||
+      (GNUNET_OK != read_start_time (rh, &sc->start_time)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10 * 1024)) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "options",
+                                           (int32_t *) &options)) ||
+      (GNUNET_OK !=
+       GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof(in_pause))) ||
+      (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "anonymity",
+                                           (int32_t *) &sc->anonymity)))
+  {
+    GNUNET_break (0);
+    goto cleanup;
+  }
+  sc->options = (enum GNUNET_FS_SearchOptions) options;
+  sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO);
+  dn = get_serialization_file_name_in_dir (h,
+                                           (NULL == sc->psearch_result)
+                                           ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH
+                                           : GNUNET_FS_SYNC_PATH_CHILD_SEARCH,
+                                           sc->serialization,
+                                           "");
+  if (NULL != dn)
+  {
+    if (GNUNET_YES == GNUNET_DISK_directory_test (dn, GNUNET_YES))
+      GNUNET_DISK_directory_scan (dn, &deserialize_search_result, sc);
+    GNUNET_free (dn);
+  }
+  if (('\0' == in_pause) &&
+      (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc)))
+  {
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_WARNING,
+      _ ("Could not resume running search, will resume as paused search\n"));
+  }
+  signal_search_resume (sc);
+  GNUNET_free (uris);
+  return sc;
+cleanup:
+  GNUNET_free (emsg);
+  free_search_context (sc);
+  GNUNET_free (uris);
+  return NULL;
+}
+
+
+/**
+ * Function called with a filename of serialized search operation
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_Handle *`
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_search_file (void *cls, const char *filename)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  char *set;
+  char *emsg;
+  struct GNUNET_BIO_ReadHandle *rh;
+  struct GNUNET_FS_SearchContext *sc;
+  struct stat buf;
+
+  if (0 != stat (filename, &buf))
+  {
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
+    return GNUNET_OK;
+  }
+  if (S_ISDIR (buf.st_mode))
+    return GNUNET_OK; /* skip directories */
+  set = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    if (NULL != set)
+    {
+      GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_SEARCH, set);
+      GNUNET_free (set);
+    }
+    return GNUNET_OK;
+  }
+  sc = deserialize_search (h, rh, NULL, set);
+  if (NULL != sc)
+    sc->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, sc);
+  GNUNET_free (set);
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failure while resuming search operation `%s': %s\n"),
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called with a filename of serialized download operation
+ * to deserialize.
+ *
+ * @param cls the `struct GNUNET_FS_Handle *`
+ * @param filename complete filename (absolute path)
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+deserialize_download_file (void *cls, const char *filename)
+{
+  struct GNUNET_FS_Handle *h = cls;
+  char *set;
+  char *emsg;
+  struct GNUNET_BIO_ReadHandle *rh;
+
+  set = get_serialization_short_name (filename);
+  rh = GNUNET_BIO_read_open_file (filename);
+  if (NULL == rh)
+  {
+    if (0 != unlink (filename))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
+                                filename);
+    GNUNET_free (set);
+    return GNUNET_OK;
+  }
+  deserialize_download (h, rh, NULL, NULL, set);
+  GNUNET_free (set);
+  if (GNUNET_OK !=
+      GNUNET_BIO_read_close (rh,
+                             &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failure while resuming download operation `%s': %s\n",
+                filename,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Deserialize information about pending operations.
+ *
+ * @param master_path which master directory should be scanned
+ * @param proc function to call for each entry (will get @a h for 'cls')
+ * @param h the `struct GNUNET_FS_Handle *`
+ */
+static void
+deserialization_master (const char *master_path,
+                        GNUNET_FileNameCallback proc,
+                        struct GNUNET_FS_Handle *h)
+{
+  char *dn;
+
+  dn = get_serialization_file_name (h, master_path, "");
+  if (NULL == dn)
+    return;
+  if (GNUNET_YES ==
+      GNUNET_DISK_directory_test (dn,
+                                  GNUNET_YES))
+    GNUNET_DISK_directory_scan (dn,
+                                proc,
+                                h);
+  GNUNET_free (dn);
+}
+
+
+struct GNUNET_FS_Handle *
+GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                 const char *client_name,
+                 GNUNET_FS_ProgressCallback upcb,
+                 void *upcb_cls,
+                 enum GNUNET_FS_Flags flags,
+                 ...)
+{
+  struct GNUNET_FS_Handle *ret;
+  enum GNUNET_FS_OPTIONS opt;
+  va_list ap;
+
+  ret = GNUNET_new (struct GNUNET_FS_Handle);
+  ret->cfg = cfg;
+  ret->client_name = GNUNET_strdup (client_name);
+  ret->upcb = upcb;
+  ret->upcb_cls = upcb_cls;
+  ret->flags = flags;
+  ret->max_parallel_downloads = DEFAULT_MAX_PARALLEL_DOWNLOADS;
+  ret->max_parallel_requests = DEFAULT_MAX_PARALLEL_REQUESTS;
+  ret->avg_block_latency =
+    GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */
+  va_start (ap, flags);
+  while (GNUNET_FS_OPTIONS_END !=
+         (opt = ((enum GNUNET_FS_OPTIONS) va_arg (ap, int))))
+  {
+    switch (opt)
+    {
+    case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM:
+      ret->max_parallel_downloads = va_arg (ap, unsigned int);
+
+      break;
+
+    case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM:
+      ret->max_parallel_requests = va_arg (ap, unsigned int);
+
+      break;
+
+    default:
+      GNUNET_break (0);
+      GNUNET_free (ret->client_name);
+      GNUNET_free (ret);
+      va_end (ap);
+      return NULL;
+    }
+  }
+  va_end (ap);
+  if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags))
+  {
+    deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_PUBLISH,
+                            &deserialize_publish_file,
+                            ret);
+    deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                            &deserialize_search_file,
+                            ret);
+    deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
+                            &deserialize_download_file,
+                            ret);
+    deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                            &deserialize_unindex_file,
+                            ret);
+  }
+  return ret;
+}
+
+
+void
+GNUNET_FS_stop (struct GNUNET_FS_Handle *h)
+{
+  while (NULL != h->top_head)
+    h->top_head->ssf (h->top_head->ssf_cls);
+  if (NULL != h->queue_job)
+    GNUNET_SCHEDULER_cancel (h->queue_job);
+  GNUNET_free (h->client_name);
+  GNUNET_free (h);
+}
+
+
+/* end of fs_api.c */
diff --git a/src/service/fs/fs_api.h b/src/service/fs/fs_api.h
new file mode 100644
index 000000000..fdda91928
--- /dev/null
+++ b/src/service/fs/fs_api.h
@@ -0,0 +1,1941 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_api.h
+ * @brief shared definitions for the FS library
+ * @author Igor Wronsky, Christian Grothoff
+ */
+#ifndef FS_API_H
+#define FS_API_H
+
+#include "gnunet_constants.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_dht_service.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_block_lib.h"
+#include "block_fs.h"
+#include "fs.h"
+
+/**
+ * Pick a multiple of 2 here to achieve 8-byte alignment!  We also
+ * probably want DBlocks to have (roughly) the same size as IBlocks.
+ * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128
+ * byte = 2 * 512 bits).  DO NOT CHANGE!
+ */
+#define CHK_PER_INODE 256
+
+/**
+ * Maximum size for a file to be considered for inlining in a
+ * directory.
+ */
+#define MAX_INLINE_SIZE 65536
+
+/**
+ * Name of the directory with top-level searches.
+ */
+#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search"
+
+/**
+ * Name of the directory with sub-searches (namespace-updates).
+ */
+#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child"
+
+/**
+ * Name of the directory with master downloads (not associated
+ * with search or part of another download).
+ */
+#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download"
+
+/**
+ * Name of the directory with downloads that are part of another
+ * download or a search.
+ */
+#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child"
+
+/**
+ * Name of the directory with publishing operations.
+ */
+#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish"
+
+/**
+ * Name of the directory with files that are being published
+ */
+#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file"
+
+/**
+ * Name of the directory with unindex operations.
+ */
+#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex"
+
+
+/**
+ * @brief complete information needed
+ * to download a file.
+ */
+struct FileIdentifier
+{
+  /**
+   * Total size of the file in bytes. (network byte order (!))
+   */
+  uint64_t file_length;
+
+  /**
+   * Query and key of the top GNUNET_EC_IBlock.
+   */
+  struct ContentHashKey chk;
+};
+
+
+/**
+ * Information about a file and its location
+ * (peer claiming to share the file).
+ */
+struct Location
+{
+  /**
+   * Information about the shared file.
+   */
+  struct FileIdentifier fi;
+
+  /**
+   * Identity of the peer sharing the file.
+   */
+  struct GNUNET_PeerIdentity peer;
+
+  /**
+   * Time when this location URI expires.
+   */
+  struct GNUNET_TIME_Absolute expirationTime;
+
+  /**
+   * Signature over the GNUNET_EC_FileIdentifier,
+   * peer identity and expiration time.
+   */
+  struct GNUNET_CRYPTO_EddsaSignature contentSignature;
+};
+
+/**
+ * Types of URIs.
+ */
+enum GNUNET_FS_UriType
+{
+  /**
+   * Content-hash-key (simple file).
+   */
+  GNUNET_FS_URI_CHK,
+
+  /**
+   * Signed key space (file in namespace).
+   */
+  GNUNET_FS_URI_SKS,
+
+  /**
+   * Keyword search key (query with keywords).
+   */
+  GNUNET_FS_URI_KSK,
+
+  /**
+   * Location (chk with identity of hosting peer).
+   */
+  GNUNET_FS_URI_LOC
+};
+
+
+/**
+ * A Universal Resource Identifier (URI), opaque.
+ */
+struct GNUNET_FS_Uri
+{
+  /**
+   * Type of the URI.
+   */
+  enum GNUNET_FS_UriType type;
+
+  union
+  {
+    struct
+    {
+      /**
+       * Keywords start with a '+' if they are mandatory (in which
+       * case the '+' is NOT part of the keyword) and with a simple
+       * space if they are optional (in which case the space is ALSO
+       * not part of the actual keyword).
+       *
+       * Double-quotes to protect spaces and %-encoding are NOT used
+       * internally (only in URI-strings).
+       */
+      char **keywords;
+
+      /**
+       * Size of the keywords array.
+       */
+      unsigned int keywordCount;
+    } ksk;
+
+    struct
+    {
+      /**
+       * Identifier of the namespace.
+       */
+      struct GNUNET_CRYPTO_EcdsaPublicKey ns;
+
+      /**
+       * Human-readable identifier chosen for this entry in the
+       * namespace.
+       */
+      char *identifier;
+    } sks;
+
+    /**
+     * Information needed to retrieve a file (content-hash-key
+     * plus file size).
+     */
+    struct FileIdentifier chk;
+
+    /**
+     * Information needed to retrieve a file including signed
+     * location (identity of a peer) of the content.
+     */
+    struct Location loc;
+  } data;
+};
+
+
+/**
+ * Information for a file or directory that is
+ * about to be published.
+ */
+struct GNUNET_FS_FileInformation
+{
+  /**
+   * Files in a directory are kept as a linked list.
+   */
+  struct GNUNET_FS_FileInformation *next;
+
+  /**
+   * If this is a file in a directory, "dir" refers to
+   * the directory; otherwise NULL.
+   */
+  struct GNUNET_FS_FileInformation *dir;
+
+  /**
+   * Handle to the master context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Pointer kept for the client.
+   */
+  void *client_info;
+
+  /**
+   * Metadata to use for the file.
+   */
+  struct GNUNET_FS_MetaData *meta;
+
+  /**
+   * Keywords to use for KBlocks.
+   */
+  struct GNUNET_FS_Uri *keywords;
+
+  /**
+   * CHK for this file or directory. NULL if
+   * we have not yet computed it.
+   */
+  struct GNUNET_FS_Uri *chk_uri;
+
+  /**
+   * SKS URI for this file or directory. NULL if
+   * we have not yet computed it.
+   */
+  struct GNUNET_FS_Uri *sks_uri;
+
+  /**
+   * Block options for the file.
+   */
+  struct GNUNET_FS_BlockOptions bo;
+
+  /**
+   * At what time did we start this upload?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * Under what filename is this struct serialized
+   * (for operational persistence).  Should be determined
+   * using 'mktemp'.
+   */
+  char *serialization;
+
+  /**
+   * Encoder being used to publish this file.
+   */
+  struct GNUNET_FS_TreeEncoder *te;
+
+  /**
+   * Error message (non-NULL if this operation failed).
+   */
+  char *emsg;
+
+  /**
+   * Name of the file or directory (must be an absolute path).
+   */
+  char *filename;
+
+  /**
+   * Data describing either the file or the directory.
+   */
+  union
+  {
+    /**
+     * Data for a file.
+     */
+    struct
+    {
+      /**
+       * Function that can be used to read the data for the file.
+       */
+      GNUNET_FS_DataReader reader;
+
+      /**
+       * Closure for reader.
+       */
+      void *reader_cls;
+
+      /**
+       * If this file is being indexed, this value is set to the hash
+       * over the entire file (when the indexing process is started).
+       * Otherwise this field is not used.
+       */
+      struct GNUNET_HashCode file_id;
+
+      /**
+       * Size of the file (in bytes).
+       */
+      uint64_t file_size;
+
+      /**
+       * Should the file be indexed or inserted?
+       */
+      int do_index;
+
+      /**
+       * Is "file_id" already valid?  Set to #GNUNET_YES once the hash
+       * has been calculated.
+       */
+      int have_hash;
+
+      /**
+       * Has the service confirmed our INDEX_START request?
+       * #GNUNET_YES if this step has been completed.
+       */
+      int index_start_confirmed;
+    } file;
+
+    /**
+     * Data for a directory.
+     */
+    struct
+    {
+      /**
+       * Linked list of entries in the directory.
+       */
+      struct GNUNET_FS_FileInformation *entries;
+
+      /**
+       * Size of the directory itself (in bytes); 0 if the
+       * size has not yet been calculated.
+       */
+      size_t dir_size;
+
+      /**
+       * Pointer to the data for the directory (or NULL if not
+       * available).
+       */
+      void *dir_data;
+
+      /**
+       * How much of the directory have we published (relative to @e contents_size).
+       */
+      uint64_t contents_completed;
+
+      /**
+       * Sum of all of the sizes of all of the files in the directory.
+       */
+      uint64_t contents_size;
+    } dir;
+  } data;
+
+  /**
+   * Is this struct for a file or directory?
+   */
+  int is_directory;
+
+  /**
+   * Are we done publishing this file?
+   */
+  int is_published;
+};
+
+
+/**
+ * Priorities for the queue.
+ */
+enum GNUNET_FS_QueuePriority
+{
+  /**
+   * This is a probe (low priority).
+   */
+  GNUNET_FS_QUEUE_PRIORITY_PROBE,
+
+  /**
+   * Default priority.
+   */
+  GNUNET_FS_QUEUE_PRIORITY_NORMAL
+};
+
+
+/**
+ * Entry in the job queue.
+ */
+struct GNUNET_FS_QueueEntry
+{
+  /**
+   * This is a linked list.
+   */
+  struct GNUNET_FS_QueueEntry *next;
+
+  /**
+   * This is a linked list.
+   */
+  struct GNUNET_FS_QueueEntry *prev;
+
+  /**
+   * Function to call when the job is started.
+   */
+  GNUNET_SCHEDULER_TaskCallback start;
+
+  /**
+   * Function to call when the job needs to stop (or is done / dequeued).
+   */
+  GNUNET_SCHEDULER_TaskCallback stop;
+
+  /**
+   * Closure for start and stop.
+   */
+  void *cls;
+
+  /**
+   * Handle to FS primary context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Message queue handle, or NULL if job is not running.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Time the job was originally queued.
+   */
+  struct GNUNET_TIME_Absolute queue_time;
+
+  /**
+   * Time the job was started last.
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * Total amount of time the job has been running (except for the
+   * current run).
+   */
+  struct GNUNET_TIME_Relative run_time;
+
+  /**
+   * How many blocks do the active downloads have?
+   */
+  unsigned int blocks;
+
+  /**
+   * How important is this download?
+   */
+  enum GNUNET_FS_QueuePriority priority;
+
+  /**
+   * How often have we (re)started this download?
+   */
+  unsigned int start_times;
+
+  /**
+   * #GNUNET_YES if the job is active now.
+   */
+  int active;
+};
+
+
+/**
+ * Information we store for each search result.
+ */
+struct GNUNET_FS_SearchResult
+{
+  /**
+   * File-sharing context this result belongs to.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Kept in a DLL while probing.
+   */
+  struct GNUNET_FS_SearchResult *next;
+
+  /**
+   * Kept in a DLL while probing.
+   */
+  struct GNUNET_FS_SearchResult *prev;
+
+  /**
+   * Search context this result belongs to; can be NULL
+   * for probes that come from a directory result.
+   */
+  struct GNUNET_FS_SearchContext *sc;
+
+  /**
+   * URI to which this search result refers to.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Metadata for the search result.
+   */
+  struct GNUNET_FS_MetaData *meta;
+
+  /**
+   * Client info for this search result.
+   */
+  void *client_info;
+
+  /**
+   * ID of a job that is currently probing this results' availability
+   * (NULL if we are not currently probing).
+   */
+  struct GNUNET_FS_DownloadContext *probe_ctx;
+
+  /**
+   * ID of an associated download based on this search result (or
+   * NULL for none).
+   */
+  struct GNUNET_FS_DownloadContext *download;
+
+  /**
+   * If this search result triggered an update search, this field
+   * links to the update search.
+   */
+  struct GNUNET_FS_SearchContext *update_search;
+
+  /**
+   * Name under which this search result is stored on disk.
+   */
+  char *serialization;
+
+  /**
+   * Bitmap that specifies precisely which keywords have been matched already.
+   */
+  uint8_t *keyword_bitmap;
+
+  /**
+   * Key for the search result based on the URI.
+   */
+  struct GNUNET_HashCode key;
+
+  /**
+   * ID of the task that will clean up the probe_ctx should it not
+   * complete on time (and that will need to be cancelled if we clean
+   * up the search result before then).
+   */
+  struct GNUNET_SCHEDULER_Task *probe_cancel_task;
+
+  /**
+   * When did the current probe become active?
+   */
+  struct GNUNET_TIME_Absolute probe_active_time;
+
+  /**
+   * How much longer should we run the current probe before giving up?
+   */
+  struct GNUNET_TIME_Relative remaining_probe_time;
+
+  /**
+   * Anonymity level to use for probes using this search result.
+   */
+  uint32_t anonymity;
+
+  /**
+   * Number of mandatory keywords for which we have NOT yet found the
+   * search result; when this value hits zero, the search result is
+   * given to the callback.
+   */
+  uint32_t mandatory_missing;
+
+  /**
+   * Number of optional keywords under which this result was also
+   * found.
+   */
+  uint32_t optional_support;
+
+  /**
+   * Number of availability tests that have succeeded for this result.
+   */
+  uint32_t availability_success;
+
+  /**
+   * Number of availability trials that we have performed for this
+   * search result.
+   */
+  uint32_t availability_trials;
+};
+
+
+/**
+ * Add a job to the queue.
+ *
+ * @param h handle to the overall FS state
+ * @param start function to call to begin the job
+ * @param stop function to call to pause the job, or on dequeue (if the job was running)
+ * @param cls closure for start and stop
+ * @param blocks number of blocks this download has
+ * @param priority how important is this download
+ * @return queue handle
+ */
+struct GNUNET_FS_QueueEntry *
+GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h,
+                  GNUNET_SCHEDULER_TaskCallback start,
+                  GNUNET_SCHEDULER_TaskCallback stop,
+                  void *cls,
+                  unsigned int blocks,
+                  enum GNUNET_FS_QueuePriority priority);
+
+
+/**
+ * Dequeue a job from the queue.
+ *
+ * @param qe handle for the job
+ */
+void
+GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qe);
+
+
+/**
+ * Function that provides data by reading from a file.
+ *
+ * @param cls closure (points to the file information)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be
+ *            copied to @a buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually "max", 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_file_ (void *cls,
+                             uint64_t offset,
+                             size_t max,
+                             void *buf,
+                             char **emsg);
+
+
+/**
+ * Create the closure for the #GNUNET_FS_data_reader_file_() callback.
+ *
+ * @param filename file to read
+ * @return closure to use, NULL on error
+ */
+void *
+GNUNET_FS_make_file_reader_context_ (const char *filename);
+
+
+/**
+ * Function that provides data by copying from a buffer.
+ *
+ * @param cls closure (points to the buffer)
+ * @param offset offset to read from; it is possible
+ *            that the caller might need to go backwards
+ *            a bit at times
+ * @param max maximum number of bytes that should be
+ *            copied to @a buf; readers are not allowed
+ *            to provide less data unless there is an error;
+ *            a value of "0" will be used at the end to allow
+ *            the reader to clean up its internal state
+ * @param buf where the reader should write the data
+ * @param emsg location for the reader to store an error message
+ * @return number of bytes written, usually @a max, 0 on error
+ */
+size_t
+GNUNET_FS_data_reader_copy_ (void *cls,
+                             uint64_t offset,
+                             size_t max,
+                             void *buf,
+                             char **emsg);
+
+
+/**
+ * Notification of FS that a search probe has made progress.
+ * This function is used INSTEAD of the client's event handler
+ * for downloads where the #GNUNET_FS_DOWNLOAD_IS_PROBE flag is set.
+ *
+ * @param cls closure, always NULL (!), actual closure
+ *        is in the client-context of the info struct
+ * @param info details about the event, specifying the event type
+ *        and various bits about the event
+ * @return client-context (for the next progress call
+ *         for this operation; should be set to NULL for
+ *         SUSPEND and STOPPED events).  The value returned
+ *         will be passed to future callbacks in the respective
+ *         field in the `struct GNUNET_FS_ProgressInfo`.
+ */
+void *
+GNUNET_FS_search_probe_progress_ (void *cls,
+                                  const struct GNUNET_FS_ProgressInfo *info);
+
+
+/**
+ * Main function that performs the upload.
+ *
+ * @param cls `struct GNUNET_FS_PublishContext` identifies the upload
+ */
+void
+GNUNET_FS_publish_main_ (void *cls);
+
+
+/**
+ * Function called once the hash of the file
+ * that is being unindexed has been computed.
+ *
+ * @param cls closure, unindex context
+ * @param file_id computed hash, NULL on error
+ */
+void
+GNUNET_FS_unindex_process_hash_ (void *cls,
+                                 const struct GNUNET_HashCode *file_id);
+
+
+/**
+ * Extract the keywords for KBlock removal
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc);
+
+
+/**
+ * If necessary, connect to the datastore and remove the KBlocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc);
+
+
+/**
+ * Fill in all of the generic fields for a publish event and call the
+ * callback.
+ *
+ * @param pi structure to fill in
+ * @param pc overall publishing context
+ * @param p file information for the file being published
+ * @param offset where in the file are we so far
+ * @return value returned from callback
+ */
+void *
+GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_PublishContext *pc,
+                                const struct GNUNET_FS_FileInformation *p,
+                                uint64_t offset);
+
+
+/**
+ * Fill in all of the generic fields for a download event and call the
+ * callback.
+ *
+ * @param pi structure to fill in
+ * @param dc overall download context
+ */
+void
+GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                 struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * Task that creates the initial (top-level) download
+ * request for the file.
+ *
+ * @param cls the 'struct GNUNET_FS_DownloadContext'
+ */
+void
+GNUNET_FS_download_start_task_ (void *cls);
+
+
+/**
+ * Fill in all of the generic fields for
+ * an unindex event and call the callback.
+ *
+ * @param pi structure to fill in
+ * @param uc overall unindex context
+ * @param offset where we are in the file (for progress)
+ */
+void
+GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_UnindexContext *uc,
+                                uint64_t offset);
+
+/**
+ * Fill in all of the generic fields for a search event and
+ * call the callback.
+ *
+ * @param pi structure to fill in
+ * @param h file-sharing handle
+ * @param sc overall search context
+ * @return value returned by the callback
+ */
+void *
+GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                               struct GNUNET_FS_Handle *h,
+                               struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * Connect to the datastore and remove the blocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc);
+
+/**
+ * Build the request and actually initiate the search using the
+ * GNUnet FS service.
+ *
+ * @param sc search context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc);
+
+/**
+ * Start the downloading process (by entering the queue).
+ *
+ * @param dc our download context
+ */
+void
+GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * Start download probes for the given search result.
+ *
+ * @param sr the search result
+ */
+void
+GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr);
+
+
+/**
+ * Remove serialization/deserialization file from disk.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param ent entity identifier
+ */
+void
+GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h,
+                             const char *ext,
+                             const char *ent);
+
+
+/**
+ * Remove serialization/deserialization directory from disk.
+ *
+ * @param h master context
+ * @param ext component of the path
+ * @param uni unique name of parent
+ */
+void
+GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h,
+                            const char *ext,
+                            const char *uni);
+
+
+/**
+ * Synchronize this file-information struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * file information data should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param fi the struct to sync
+ */
+void
+GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f);
+
+
+/**
+ * Synchronize this publishing struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param pc the struct to sync
+ */
+void
+GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc);
+
+
+/**
+ * Synchronize this unindex struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param uc the struct to sync
+ */
+void
+GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc);
+
+
+/**
+ * Synchronize this search struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param sc the struct to sync
+ */
+void
+GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * Synchronize this search result with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param sr the struct to sync
+ */
+void
+GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr);
+
+
+/**
+ * Synchronize this download struct with its mirror
+ * on disk.  Note that all internal FS-operations that change
+ * publishing structs should already call "sync" internally,
+ * so this function is likely not useful for clients.
+ *
+ * @param dc the struct to sync
+ */
+void
+GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * Create SUSPEND event for the given publish operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext` to signal for
+ */
+void
+GNUNET_FS_publish_signal_suspend_ (void *cls);
+
+
+/**
+ * Create SUSPEND event for the given search operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the 'struct GNUNET_FS_SearchContext' to signal for
+ */
+void
+GNUNET_FS_search_signal_suspend_ (void *cls);
+
+
+/**
+ * Create SUSPEND event for the given download operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_DownloadContext` to signal for
+ */
+void
+GNUNET_FS_download_signal_suspend_ (void *cls);
+
+
+/**
+ * Create SUSPEND event for the given unindex operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_UnindexContext` to signal for
+ */
+void
+GNUNET_FS_unindex_signal_suspend_ (void *cls);
+
+
+/**
+ * Function signature of the functions that can be called
+ * to trigger suspend signals and clean-up for top-level
+ * activities.
+ *
+ * @param cls closure
+ */
+typedef void (*SuspendSignalFunction) (void *cls);
+
+/**
+ * We track all of the top-level activities of FS
+ * so that we can signal 'suspend' on shutdown.
+ */
+struct TopLevelActivity
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct TopLevelActivity *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct TopLevelActivity *prev;
+
+  /**
+   * Function to call for suspend-signalling and clean up.
+   */
+  SuspendSignalFunction ssf;
+
+  /**
+   * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*)
+   */
+  void *ssf_cls;
+};
+
+
+/**
+ * Create a top-level activity entry.
+ *
+ * @param h global fs handle
+ * @param ssf suspend signal function to use
+ * @param ssf_cls closure for @a ssf
+ * @return fresh top-level activity handle
+ */
+struct TopLevelActivity *
+GNUNET_FS_make_top (struct GNUNET_FS_Handle *h,
+                    SuspendSignalFunction ssf,
+                    void *ssf_cls);
+
+
+/**
+ * Destroy a top-level activity entry.
+ *
+ * @param h global fs handle
+ * @param top top level activity entry
+ */
+void
+GNUNET_FS_end_top (struct GNUNET_FS_Handle *h,
+                   struct TopLevelActivity *top);
+
+
+/**
+ * Master context for most FS operations.
+ */
+struct GNUNET_FS_Handle
+{
+  /**
+   * Configuration to use.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Name of our client.
+   */
+  char *client_name;
+
+  /**
+   * Function to call with updates on our progress.
+   */
+  GNUNET_FS_ProgressCallback upcb;
+
+  /**
+   * Closure for upcb.
+   */
+  void *upcb_cls;
+
+  /**
+   * Head of DLL of top-level activities.
+   */
+  struct TopLevelActivity *top_head;
+
+  /**
+   * Tail of DLL of top-level activities.
+   */
+  struct TopLevelActivity *top_tail;
+
+  /**
+   * Head of DLL of running jobs.
+   */
+  struct GNUNET_FS_QueueEntry *running_head;
+
+  /**
+   * Tail of DLL of running jobs.
+   */
+  struct GNUNET_FS_QueueEntry *running_tail;
+
+  /**
+   * Head of DLL of pending jobs.
+   */
+  struct GNUNET_FS_QueueEntry *pending_head;
+
+  /**
+   * Tail of DLL of pending jobs.
+   */
+  struct GNUNET_FS_QueueEntry *pending_tail;
+
+  /**
+   * Head of active probes.
+   */
+  struct GNUNET_FS_SearchResult *probes_head;
+
+  /**
+   * Tail of active probes.
+   */
+  struct GNUNET_FS_SearchResult *probes_tail;
+
+  /**
+   * Task that processes the jobs in the running and pending queues
+   * (and moves jobs around as needed).
+   */
+  struct GNUNET_SCHEDULER_Task *queue_job;
+
+  /**
+   * Task we use to report periodically to the application that
+   * certain search probes (from @e probes_head) are still running.
+   */
+  struct GNUNET_SCHEDULER_Task *probe_ping_task;
+
+  /**
+   * Average time we take for a single request to be satisfied.
+   * FIXME: not yet calculated properly...
+   */
+  struct GNUNET_TIME_Relative avg_block_latency;
+
+  /**
+   * How many actual downloads do we have running right now?
+   */
+  unsigned int active_downloads;
+
+  /**
+   * How many blocks do the active downloads have?
+   */
+  unsigned int active_blocks;
+
+  /**
+   * General flags.
+   */
+  enum GNUNET_FS_Flags flags;
+
+  /**
+   * Maximum number of parallel downloads.
+   */
+  unsigned int max_parallel_downloads;
+
+  /**
+   * Maximum number of parallel requests.
+   */
+  unsigned int max_parallel_requests;
+};
+
+
+/**
+ * Handle for controlling a publication process.
+ */
+struct GNUNET_FS_PublishContext
+{
+  /**
+   * Handle to the global fs context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Our top-level activity entry (if we are top-level, otherwise NULL).
+   */
+  struct TopLevelActivity *top;
+
+  /**
+   * File-structure that is being shared.
+   */
+  struct GNUNET_FS_FileInformation *fi;
+
+  /**
+   * Namespace that we are publishing in, NULL if we have no namespace.
+   */
+  struct GNUNET_CRYPTO_EcdsaPrivateKey *ns;
+
+  /**
+   * ID of the content in the namespace, NULL if we have no namespace.
+   */
+  char *nid;
+
+  /**
+   * ID for future updates, NULL if we have no namespace or no updates.
+   */
+  char *nuid;
+
+  /**
+   * Filename used for serializing information about this operation
+   * (should be determined using 'mktemp').
+   */
+  char *serialization;
+
+  /**
+   * Our own message queue for the FS service; only briefly used when
+   * we start to index a file, otherwise NULL.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Current position in the file-tree for the upload.
+   */
+  struct GNUNET_FS_FileInformation *fi_pos;
+
+  /**
+   * Non-null if we are currently hashing a file.
+   */
+  struct GNUNET_CRYPTO_FileHashContext *fhc;
+
+  /**
+   * Connection to the datastore service.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * Queue entry for reservation/unreservation.
+   */
+  struct GNUNET_DATASTORE_QueueEntry *qre;
+
+  /**
+   * Context for SKS publishing operation that is part of this publishing operation
+   * (NULL if not active).
+   */
+  struct GNUNET_FS_PublishSksContext *sks_pc;
+
+  /**
+   * Context for KSK publishing operation that is part of this publishing operation
+   * (NULL if not active).
+   */
+  struct GNUNET_FS_PublishKskContext *ksk_pc;
+
+  /**
+   * ID of the task performing the upload. NO_TASK if the upload has
+   * completed.
+   */
+  struct GNUNET_SCHEDULER_Task *upload_task;
+
+  /**
+   * Storage space to reserve for the operation.
+   */
+  uint64_t reserve_space;
+
+  /**
+   * Overall number of entries to reserve for the
+   * publish operation.
+   */
+  uint32_t reserve_entries;
+
+  /**
+   * Options for publishing.
+   */
+  enum GNUNET_FS_PublishOptions options;
+
+  /**
+   * Space reservation ID with datastore service
+   * for this upload.
+   */
+  int rid;
+
+  /**
+   * Set to #GNUNET_YES if we were able to publish any block.
+   * (and thus unindexing on error might make sense).
+   */
+  int any_done;
+
+  /**
+   * Set to #GNUNET_YES if all processing has completed.
+   */
+  int all_done;
+
+  /**
+   * Flag set to #GNUNET_YES if the next callback from
+   * #GNUNET_FS_file_information_inspect should be skipped because it
+   * is for the directory which was already processed with the parent.
+   */
+  int skip_next_fi_callback;
+};
+
+
+/**
+ * Phases of unindex processing (state machine).
+ */
+enum UnindexState
+{
+  /**
+   * We're currently hashing the file.
+   */
+  UNINDEX_STATE_HASHING = 0,
+
+  /**
+   * We're telling the datastore to delete
+   * the respective DBlocks and IBlocks.
+   */
+  UNINDEX_STATE_DS_REMOVE = 1,
+
+  /**
+   * Find out which keywords apply.
+   */
+  UNINDEX_STATE_EXTRACT_KEYWORDS = 2,
+
+  /**
+   * We're telling the datastore to remove KBlocks.
+   */
+  UNINDEX_STATE_DS_REMOVE_KBLOCKS = 3,
+
+  /**
+   * We're notifying the FS service about
+   * the unindexing.
+   */
+  UNINDEX_STATE_FS_NOTIFY = 4,
+
+  /**
+   * We're done.
+   */
+  UNINDEX_STATE_COMPLETE = 5,
+
+  /**
+   * We've encountered a fatal error.
+   */
+  UNINDEX_STATE_ERROR = 6
+};
+
+
+/**
+ * Handle for controlling an unindexing operation.
+ */
+struct GNUNET_FS_UnindexContext
+{
+  /**
+   * The content hash key of the last block we processed, will in the
+   * end be set to the CHK from the URI.  Used to remove the KBlocks.
+   */
+  struct ContentHashKey chk;
+
+  /**
+   * Global FS context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Our top-level activity entry.
+   */
+  struct TopLevelActivity *top;
+
+  /**
+   * Directory scanner to find keywords (KBlock removal).
+   */
+  struct GNUNET_FS_DirScanner *dscan;
+
+  /**
+   * Keywords found (telling us which KBlocks to remove).
+   */
+  struct GNUNET_FS_Uri *ksk_uri;
+
+  /**
+   * Current offset in KSK removal.
+   */
+  uint32_t ksk_offset;
+
+  /**
+   * Name of the file that we are unindexing.
+   */
+  char *filename;
+
+  /**
+   * Short name under which we are serializing the state of this operation.
+   */
+  char *serialization;
+
+  /**
+   * Connection to the FS service, only valid during the
+   * #UNINDEX_STATE_FS_NOTIFY phase.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Connection to the datastore service, only valid during the
+   * UNINDEX_STATE_DS_NOTIFY phase.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * Pointer kept for the client.
+   */
+  void *client_info;
+
+  /**
+   * Merkle-ish tree encoder context.
+   */
+  struct GNUNET_FS_TreeEncoder *tc;
+
+  /**
+   * Handle used to read the file.
+   */
+  struct GNUNET_DISK_FileHandle *fh;
+
+  /**
+   * Handle to datastore 'get_key' operation issued for
+   * obtaining KBlocks.
+   */
+  struct GNUNET_DATASTORE_QueueEntry *dqe;
+
+  /**
+   * Current key for decrypting UBLocks from 'get_key' operation.
+   */
+  struct GNUNET_HashCode ukey;
+
+  /**
+   * Current query of 'get_key' operation.
+   */
+  struct GNUNET_HashCode uquery;
+
+  /**
+   * Error message, NULL on success.
+   */
+  char *emsg;
+
+  /**
+   * Context for hashing of the file.
+   */
+  struct GNUNET_CRYPTO_FileHashContext *fhc;
+
+  /**
+   * Overall size of the file.
+   */
+  uint64_t file_size;
+
+  /**
+   * When did we start?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * Hash of the file's contents (once computed).
+   */
+  struct GNUNET_HashCode file_id;
+
+  /**
+   * Current operatinonal phase.
+   */
+  enum UnindexState state;
+};
+
+
+/**
+ * Information we keep for each keyword in a keyword search.
+ */
+struct SearchRequestEntry
+{
+  /**
+   * Hash of the public key, also known as the query.
+   */
+  struct GNUNET_HashCode uquery;
+
+  /**
+   * Derived public key, hashes to 'uquery'.
+   */
+  struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
+
+  /**
+   * The original keyword, used to derive the
+   * key (for decrypting the UBlock).
+   */
+  char *keyword;
+
+  /**
+   * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
+   * was found under this keyword.  Note that the entries will point
+   * to the same locations as those in the master result map (in
+   * "struct GNUNET_FS_SearchContext"), so they should not be freed.
+   * The key for each entry is the XOR of the key and query in the CHK
+   * URI (as a unique identifier for the search result).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *results;
+
+  /**
+   * Is this keyword a mandatory keyword
+   * (started with '+')?
+   */
+  int mandatory;
+};
+
+
+/**
+ * Handle for controlling a search.
+ */
+struct GNUNET_FS_SearchContext
+{
+  /**
+   * Handle to the global FS context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Our top-level activity entry (if we are top-level, otherwise NULL).
+   */
+  struct TopLevelActivity *top;
+
+  /**
+   * List of keywords that we're looking for.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * For update-searches, link to the search result that triggered
+   * the update search; otherwise NULL.
+   */
+  struct GNUNET_FS_SearchResult *psearch_result;
+
+  /**
+   * Connection to the FS service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Pointer we keep for the client.
+   */
+  void *client_info;
+
+  /**
+   * Name of the file on disk we use for persistence.
+   */
+  char *serialization;
+
+  /**
+   * Error message (non-NULL if this operation failed).
+   */
+  char *emsg;
+
+  /**
+   * Map that contains a `struct GNUNET_FS_SearchResult` for each result that
+   * was found in the search.  The key for each entry is the XOR of
+   * the key and query in the CHK URI (as a unique identifier for the
+   * search result).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *master_result_map;
+
+  /**
+   * Per-keyword information for a keyword search.  This array will
+   * have exactly as many entries as there were keywords.
+   */
+  struct SearchRequestEntry *requests;
+
+  /**
+   * When did we start?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * How long to wait before we try to reconnect to FS service?
+   */
+  struct GNUNET_TIME_Relative reconnect_backoff;
+
+  /**
+   * ID of a task that is using this struct and that must be cancelled
+   * when the search is being stopped (if not
+   * NULL).  Used for the task that adds some
+   * artificial delay when trying to reconnect to the FS service.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Anonymity level for the search.
+   */
+  uint32_t anonymity;
+
+  /**
+   * Number of mandatory keywords in this query.
+   */
+  uint32_t mandatory_count;
+
+  /**
+   * Options for the search.
+   */
+  enum GNUNET_FS_SearchOptions options;
+};
+
+
+/**
+ * FSM for possible states a block can go through.  The typical
+ * order of progression is linear through the states, alternatives
+ * are documented in the comments.
+ */
+enum BlockRequestState
+{
+  /**
+   * Initial state, block has only been allocated (since it is
+   * relevant to the overall download request).
+   */
+  BRS_INIT = 0,
+
+  /**
+   * We've checked the block on the path down the tree, and the
+   * content on disk did match the desired CHK, but not all
+   * the way down, so at the bottom some blocks will still
+   * need to be reconstructed).
+   */
+  BRS_RECONSTRUCT_DOWN = 1,
+
+  /**
+   * We've calculated the CHK bottom-up based on the meta data.
+   * This may work, but if it did we have to write the meta data to
+   * disk at the end (and we still need to check against the
+   * CHK set on top).
+   */
+  BRS_RECONSTRUCT_META_UP = 2,
+
+  /**
+   * We've calculated the CHK bottom-up based on what we have on
+   * disk, which may not be what the desired CHK is.  If the
+   * reconstructed CHKs match whatever comes from above, we're
+   * done with the respective subtree.
+   */
+  BRS_RECONSTRUCT_UP = 3,
+
+  /**
+   * We've determined the real, desired CHK for this block
+   * (full tree reconstruction failed), request is now pending.
+   * If the CHK that bubbled up through reconstruction did match
+   * the top-level request, the state machine for the subtree
+   * would have moved to BRS_DOWNLOAD_UP.
+   */
+  BRS_CHK_SET = 4,
+
+  /**
+   * We've successfully downloaded this block, but the children
+   * still need to be either downloaded or verified (download
+   * request propagates down).  If the download fails, the
+   * state machine for this block may move to
+   * BRS_DOWNLOAD_ERROR instead.
+   */
+  BRS_DOWNLOAD_DOWN = 5,
+
+  /**
+   * This block and all of its children have been downloaded
+   * successfully (full completion propagates up).
+   */
+  BRS_DOWNLOAD_UP = 6,
+
+  /**
+   * We got a block back that matched the query but did not hash to
+   * the key (malicious publisher or hash collision); this block
+   * can never be downloaded (error propagates up).
+   */
+  BRS_ERROR = 7
+};
+
+
+/**
+ * Information about an active download request.
+ */
+struct DownloadRequest
+{
+  /**
+   * Parent in the CHK-tree.
+   */
+  struct DownloadRequest *parent;
+
+  /**
+   * Array (!) of child-requests, or NULL for the bottom of the tree.
+   */
+  struct DownloadRequest **children;
+
+  /**
+   * CHK for the request for this block (set during reconstruction
+   * to what we have on disk, later to what we want to have).
+   */
+  struct ContentHashKey chk;
+
+  /**
+   * Offset of the corresponding block.  Specifically, first (!) byte of
+   * the first DBLOCK in the subtree induced by block represented by
+   * this request.
+   */
+  uint64_t offset;
+
+  /**
+   * Number of entries in @e children array.
+   */
+  unsigned int num_children;
+
+  /**
+   * Depth of the corresponding block in the tree.  0==DBLOCKs.
+   */
+  unsigned int depth;
+
+  /**
+   * Offset of the CHK for this block in the parent block
+   */
+  unsigned int chk_idx;
+
+  /**
+   * State in the FSM.
+   */
+  enum BlockRequestState state;
+};
+
+
+/**
+ * (recursively) free download request structure
+ *
+ * @param dr request to free
+ */
+void
+GNUNET_FS_free_download_request_ (struct DownloadRequest *dr);
+
+
+/**
+ * Stop the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+void
+GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr);
+
+
+/**
+ * Context for controlling a download.
+ */
+struct GNUNET_FS_DownloadContext
+{
+  /**
+   * Global FS context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Our top-level activity entry (if we are top-level, otherwise NULL).
+   */
+  struct TopLevelActivity *top;
+
+  /**
+   * Connection to the FS service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Parent download (used when downloading files
+   * in directories).
+   */
+  struct GNUNET_FS_DownloadContext *parent;
+
+  /**
+   * Associated search (used when downloading files
+   * based on search results), or NULL for none.
+   */
+  struct GNUNET_FS_SearchResult *search;
+
+  /**
+   * Head of list of child downloads.
+   */
+  struct GNUNET_FS_DownloadContext *child_head;
+
+  /**
+   * Tail of list of child downloads.
+   */
+  struct GNUNET_FS_DownloadContext *child_tail;
+
+  /**
+   * Previous download belonging to the same parent.
+   */
+  struct GNUNET_FS_DownloadContext *prev;
+
+  /**
+   * Next download belonging to the same parent.
+   */
+  struct GNUNET_FS_DownloadContext *next;
+
+  /**
+   * Context kept for the client.
+   */
+  void *client_info;
+
+  /**
+   * URI that identifies the file that we are downloading.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Known meta-data for the file (can be NULL).
+   */
+  struct GNUNET_FS_MetaData *meta;
+
+  /**
+   * Error message, NULL if we're doing OK.
+   */
+  char *emsg;
+
+  /**
+   * Random portion of filename we use for syncing state of this
+   * download.
+   */
+  char *serialization;
+
+  /**
+   * Where are we writing the data (name of the
+   * file, can be NULL!).
+   */
+  char *filename;
+
+  /**
+   * Where are we writing the data temporarily (name of the
+   * file, can be NULL!); used if we do not have a permanent
+   * name and we are a directory and we do a recursive download.
+   */
+  char *temp_filename;
+
+  /**
+   * Our entry in the job queue.
+   */
+  struct GNUNET_FS_QueueEntry *job_queue;
+
+  /**
+   * Tree encoder used for the reconstruction.
+   */
+  struct GNUNET_FS_TreeEncoder *te;
+
+  /**
+   * File handle for reading data from an existing file
+   * (to pass to tree encoder).
+   */
+  struct GNUNET_DISK_FileHandle *rfh;
+
+  /**
+   * Map of active requests (those waiting for a response).  The key
+   * is the hash of the encryped block (aka query).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *active;
+
+  /**
+   * Top-level download request.
+   */
+  struct DownloadRequest *top_request;
+
+  /**
+   * Identity of the peer having the content, or all-zeros
+   * if we don't know of such a peer.
+   */
+  struct GNUNET_PeerIdentity target;
+
+  /**
+   * ID of a task that is using this struct and that must be cancelled
+   * when the download is being stopped (if not
+   * NULL).  Used for the task that adds some
+   * artificial delay when trying to reconnect to the FS service or
+   * the task processing incrementally the data on disk, or the
+   * task requesting blocks, etc.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * What is the first offset that we're interested
+   * in?
+   */
+  uint64_t offset;
+
+  /**
+   * How many bytes starting from offset are desired?
+   * This is NOT the overall length of the file!
+   */
+  uint64_t length;
+
+  /**
+   * How many bytes have we already received within
+   * the specified range (DBlocks only).
+   */
+  uint64_t completed;
+
+  /**
+   * What was the size of the file on disk that we're downloading
+   * before we started?  Used to detect if there is a point in
+   * checking an existing block on disk for matching the desired
+   * content.  0 if the file did not exist already.
+   */
+  uint64_t old_file_size;
+
+  /**
+   * Time download was started.
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * How long to wait before we try to reconnect to FS service?
+   */
+  struct GNUNET_TIME_Relative reconnect_backoff;
+
+  /**
+   * Desired level of anonymity.
+   */
+  uint32_t anonymity;
+
+  /**
+   * The depth of the file-tree.
+   */
+  unsigned int treedepth;
+
+  /**
+   * Options for the download.
+   */
+  enum GNUNET_FS_DownloadOptions options;
+
+  /**
+   * Flag set upon transitive completion (includes child downloads).
+   * This flag is only set to #GNUNET_YES for directories where all
+   * child-downloads have also completed (and signalled completion).
+   */
+  int has_finished;
+
+  /**
+   * Are we ready to issue requests (reconstructions are finished)?
+   */
+  int issue_requests;
+};
+
+
+#endif
+
+/* end of fs_api.h */
diff --git a/src/service/fs/fs_directory.c b/src/service/fs/fs_directory.c
new file mode 100644
index 000000000..c693f9216
--- /dev/null
+++ b/src/service/fs/fs_directory.c
@@ -0,0 +1,676 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_directory.c
+ * @brief Helper functions for building directories.
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - modify directory builder API to support incremental
+ *   generation of directories (to allow directories that
+ *   would not fit into memory to be created)
+ * - modify directory processor API to support incremental
+ *   iteration over FULL directories (without missing entries)
+ *   to allow access to directories that do not fit entirely
+ *   into memory
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+
+/**
+ * String that is used to indicate that a file
+ * is a GNUnet directory.
+ */
+#define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
+
+
+/**
+ * Does the meta-data claim that this is a directory?
+ * Checks if the mime-type is that of a GNUnet directory.
+ *
+ * @return #GNUNET_YES if it is, #GNUNET_NO if it is not, #GNUNET_SYSERR if
+ *  we have no mime-type information (treat as #GNUNET_NO)
+ */
+int
+GNUNET_FS_meta_data_test_for_directory (const struct
+                                        GNUNET_FS_MetaData *md)
+{
+  char *mime;
+  int ret;
+
+  if (NULL == md)
+    return GNUNET_SYSERR;
+  mime = GNUNET_FS_meta_data_get_by_type (md,
+                                          EXTRACTOR_METATYPE_MIMETYPE);
+  if (NULL == mime)
+    return GNUNET_SYSERR;
+  ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES :
+        GNUNET_NO;
+  GNUNET_free (mime);
+  return ret;
+}
+
+
+/**
+ * Set the MIMETYPE information for the given
+ * metadata to "application/gnunet-directory".
+ *
+ * @param md metadata to add mimetype to
+ */
+void
+GNUNET_FS_meta_data_make_directory (struct GNUNET_FS_MetaData *md)
+{
+  char *mime;
+
+  mime =
+    GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
+  if (mime != NULL)
+  {
+    GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME));
+    GNUNET_free (mime);
+    return;
+  }
+  GNUNET_FS_meta_data_insert (md, "<gnunet>",
+                                     EXTRACTOR_METATYPE_MIMETYPE,
+                                     EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                     GNUNET_FS_DIRECTORY_MIME,
+                                     strlen (GNUNET_FS_DIRECTORY_MIME) + 1);
+}
+
+
+/**
+ * Closure for 'find_full_data'.
+ */
+struct GetFullDataClosure
+{
+  /**
+   * Extracted binary meta data.
+   */
+  void *data;
+
+  /**
+   * Number of bytes stored in data.
+   */
+  size_t size;
+};
+
+
+/**
+ * Type of a function that libextractor calls for each
+ * meta data item found.
+ *
+ * @param cls closure (user-defined)
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return 0 to continue extracting, 1 to abort
+ */
+static int
+find_full_data (void *cls, const char *plugin_name,
+                enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
+                const char *data_mime_type, const char *data, size_t data_len)
+{
+  struct GetFullDataClosure *gfdc = cls;
+
+  if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
+  {
+    gfdc->size = data_len;
+    if (data_len > 0)
+    {
+      gfdc->data = GNUNET_malloc (data_len);
+      GNUNET_memcpy (gfdc->data, data, data_len);
+    }
+    return 1;
+  }
+  return 0;
+}
+
+
+/**
+ * Iterate over all entries in a directory.  Note that directories
+ * are structured such that it is possible to iterate over the
+ * individual blocks as well as over the entire directory.  Thus
+ * a client can call this function on the buffer in the
+ * GNUNET_FS_ProgressCallback.  Also, directories can optionally
+ * include the contents of (small) files embedded in the directory
+ * itself; for those files, the processor may be given the
+ * contents of the file directly by this function.
+ * <p>
+ *
+ * Note that this function maybe called on parts of directories.  Thus
+ * parser errors should not be reported _at all_ (with GNUNET_break).
+ * Still, if some entries can be recovered despite these parsing
+ * errors, the function should try to do this.
+ *
+ * @param size number of bytes in data
+ * @param data pointer to the beginning of the directory
+ * @param offset offset of data in the directory
+ * @param dep function to call on each entry
+ * @param dep_cls closure for @a dep
+ * @return #GNUNET_OK if this could be a block in a directory,
+ *         #GNUNET_NO if this could be part of a directory (but not 100% OK)
+ *         #GNUNET_SYSERR if @a data does not represent a directory
+ */
+int
+GNUNET_FS_directory_list_contents (size_t size,
+                                   const void *data,
+                                   uint64_t offset,
+                                   GNUNET_FS_DirectoryEntryProcessor dep,
+                                   void *dep_cls)
+{
+  struct GetFullDataClosure full_data;
+  const char *cdata = data;
+  char *emsg;
+  uint64_t pos;
+  uint64_t align;
+  uint32_t mdSize;
+  uint64_t epos;
+  struct GNUNET_FS_Uri *uri;
+  struct GNUNET_FS_MetaData *md;
+  char *filename;
+
+  if ((offset == 0) &&
+      ((size < 8 + sizeof(uint32_t)) ||
+       (0 != memcmp (cdata,
+                     GNUNET_FS_DIRECTORY_MAGIC,
+                     8))))
+    return GNUNET_SYSERR;
+  pos = offset;
+  if (offset == 0)
+  {
+    GNUNET_memcpy (&mdSize,
+                   &cdata[8],
+                   sizeof(uint32_t));
+    mdSize = ntohl (mdSize);
+    if (mdSize > size - 8 - sizeof(uint32_t))
+    {
+      /* invalid size */
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _ ("MAGIC mismatch.  This is not a GNUnet directory.\n"));
+      return GNUNET_SYSERR;
+    }
+    md = GNUNET_FS_meta_data_deserialize (&cdata[8 + sizeof(uint32_t)],
+                                          mdSize);
+    if (md == NULL)
+    {
+      GNUNET_break (0);
+      return GNUNET_SYSERR;     /* malformed ! */
+    }
+    dep (dep_cls,
+         NULL,
+         NULL,
+         md,
+         0,
+         NULL);
+    GNUNET_FS_meta_data_destroy (md);
+    pos = 8 + sizeof(uint32_t) + mdSize;
+  }
+  while (pos < size)
+  {
+    /* find end of URI */
+    if (cdata[pos] == '\0')
+    {
+      /* URI is never empty, must be end of block,
+       * skip to next alignment */
+      align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
+      if (align == pos)
+      {
+        /* if we were already aligned, still skip a block! */
+        align += DBLOCK_SIZE;
+      }
+      pos = align;
+      if (pos >= size)
+      {
+        /* malformed - or partial download... */
+        break;
+      }
+    }
+    epos = pos;
+    while ((epos < size) && (cdata[epos] != '\0'))
+      epos++;
+    if (epos >= size)
+      return GNUNET_NO;         /* malformed - or partial download */
+
+    uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
+    pos = epos + 1;
+    if (NULL == uri)
+    {
+      GNUNET_free (emsg);
+      pos--;                    /* go back to '\0' to force going to next alignment */
+      continue;
+    }
+    if (GNUNET_FS_uri_test_ksk (uri))
+    {
+      GNUNET_FS_uri_destroy (uri);
+      GNUNET_break (0);
+      return GNUNET_NO;         /* illegal in directory! */
+    }
+
+    GNUNET_memcpy (&mdSize,
+                   &cdata[pos],
+                   sizeof(uint32_t));
+    mdSize = ntohl (mdSize);
+    pos += sizeof(uint32_t);
+    if (pos + mdSize > size)
+    {
+      GNUNET_FS_uri_destroy (uri);
+      return GNUNET_NO;         /* malformed - or partial download */
+    }
+
+    md = GNUNET_FS_meta_data_deserialize (&cdata[pos],
+                                                 mdSize);
+    if (NULL == md)
+    {
+      GNUNET_FS_uri_destroy (uri);
+      GNUNET_break (0);
+      return GNUNET_NO;         /* malformed ! */
+    }
+    pos += mdSize;
+    filename =
+      GNUNET_FS_meta_data_get_by_type (md,
+                                              EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
+    full_data.size = 0;
+    full_data.data = NULL;
+    GNUNET_FS_meta_data_iterate (md,
+                                        &find_full_data,
+                                        &full_data);
+    if (NULL != dep)
+    {
+      dep (dep_cls,
+           filename,
+           uri,
+           md,
+           full_data.size,
+           full_data.data);
+    }
+    GNUNET_free (full_data.data);
+    GNUNET_free (filename);
+    GNUNET_FS_meta_data_destroy (md);
+    GNUNET_FS_uri_destroy (uri);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Entries in the directory (builder).
+ */
+struct BuilderEntry
+{
+  /**
+   * This is a linked list.
+   */
+  struct BuilderEntry *next;
+
+  /**
+   * Length of this entry.
+   */
+  size_t len;
+};
+
+/**
+ * Internal state of a directory builder.
+ */
+struct GNUNET_FS_DirectoryBuilder
+{
+  /**
+   * Meta-data for the directory itself.
+   */
+  struct GNUNET_FS_MetaData *meta;
+
+  /**
+   * Head of linked list of entries.
+   */
+  struct BuilderEntry *head;
+
+  /**
+   * Number of entries in the directory.
+   */
+  unsigned int count;
+};
+
+
+/**
+ * Create a directory builder.
+ *
+ * @param mdir metadata for the directory
+ */
+struct GNUNET_FS_DirectoryBuilder *
+GNUNET_FS_directory_builder_create (const struct GNUNET_FS_MetaData
+                                    *mdir)
+{
+  struct GNUNET_FS_DirectoryBuilder *ret;
+
+  ret = GNUNET_new (struct GNUNET_FS_DirectoryBuilder);
+  if (mdir != NULL)
+    ret->meta = GNUNET_FS_meta_data_duplicate (mdir);
+  else
+    ret->meta = GNUNET_FS_meta_data_create ();
+  GNUNET_FS_meta_data_make_directory (ret->meta);
+  return ret;
+}
+
+
+/**
+ * Add an entry to a directory.
+ *
+ * @param bld directory to extend
+ * @param uri uri of the entry (must not be a KSK)
+ * @param md metadata of the entry
+ * @param data raw data of the entry, can be NULL, otherwise
+ *        data must point to exactly the number of bytes specified
+ *        by the uri which must be of type LOC or CHK
+ */
+void
+GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
+                                 const struct GNUNET_FS_Uri *uri,
+                                 const struct GNUNET_FS_MetaData *md,
+                                 const void *data)
+{
+  struct GNUNET_FS_Uri *curi;
+  struct BuilderEntry *e;
+  uint64_t fsize;
+  uint32_t big;
+  ssize_t ret;
+  size_t mds;
+  size_t mdxs;
+  char *uris;
+  char *serialized;
+  char *sptr;
+  size_t slen;
+  struct GNUNET_FS_MetaData *meta;
+  const struct GNUNET_FS_MetaData *meta_use;
+
+  GNUNET_assert (! GNUNET_FS_uri_test_ksk (uri));
+  if (NULL != data)
+  {
+    GNUNET_assert (! GNUNET_FS_uri_test_sks (uri));
+    if (GNUNET_FS_uri_test_chk (uri))
+    {
+      fsize = GNUNET_FS_uri_chk_get_file_size (uri);
+    }
+    else
+    {
+      curi = GNUNET_FS_uri_loc_get_uri (uri);
+      GNUNET_assert (NULL != curi);
+      fsize = GNUNET_FS_uri_chk_get_file_size (curi);
+      GNUNET_FS_uri_destroy (curi);
+    }
+  }
+  else
+  {
+    fsize = 0;                  /* not given */
+  }
+  if (fsize > MAX_INLINE_SIZE)
+    fsize = 0;                  /* too large */
+  uris = GNUNET_FS_uri_to_string (uri);
+  slen = strlen (uris) + 1;
+  mds = GNUNET_FS_meta_data_get_serialized_size (md);
+  meta_use = md;
+  meta = NULL;
+  if (fsize > 0)
+  {
+    meta = GNUNET_FS_meta_data_duplicate (md);
+    GNUNET_FS_meta_data_insert (meta, "<gnunet>",
+                                       EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
+                                       EXTRACTOR_METAFORMAT_BINARY, NULL, data,
+                                       fsize);
+    mdxs = GNUNET_FS_meta_data_get_serialized_size (meta);
+    if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
+        (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE)
+    {
+      /* adding full data would not cause us to cross
+       * additional blocks, so add it! */
+      meta_use = meta;
+      mds = mdxs;
+    }
+  }
+
+  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
+    mds = GNUNET_MAX_MALLOC_CHECKED / 2;
+  e = GNUNET_malloc (sizeof(struct BuilderEntry) + slen + mds
+                     + sizeof(uint32_t));
+  serialized = (char *) &e[1];
+  GNUNET_memcpy (serialized, uris, slen);
+  GNUNET_free (uris);
+  sptr = &serialized[slen + sizeof(uint32_t)];
+  ret =
+    GNUNET_FS_meta_data_serialize (meta_use, &sptr, mds,
+                                          GNUNET_FS_META_DATA_SERIALIZE_PART);
+  if (NULL != meta)
+    GNUNET_FS_meta_data_destroy (meta);
+  if (ret == -1)
+    mds = 0;
+  else
+    mds = ret;
+  big = htonl (mds);
+  GNUNET_memcpy (&serialized[slen], &big, sizeof(uint32_t));
+  e->len = slen + sizeof(uint32_t) + mds;
+  e->next = bld->head;
+  bld->head = e;
+  bld->count++;
+}
+
+
+/**
+ * Given the start and end position of a block of
+ * data, return the end position of that data
+ * after alignment to the DBLOCK_SIZE.
+ */
+static size_t
+do_align (size_t start_position, size_t end_position)
+{
+  size_t align;
+
+  align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
+  if ((start_position < align) && (end_position > align))
+    return align + end_position - start_position;
+  return end_position;
+}
+
+
+/**
+ * Compute a permutation of the blocks to
+ * minimize the cost of alignment.  Greedy packer.
+ *
+ * @param start starting position for the first block
+ * @param count size of the two arrays
+ * @param sizes the sizes of the individual blocks
+ * @param perm the permutation of the blocks (updated)
+ */
+static void
+block_align (size_t start, unsigned int count, const size_t *sizes,
+             unsigned int *perm)
+{
+  unsigned int i;
+  unsigned int j;
+  unsigned int tmp;
+  unsigned int best;
+  ssize_t badness;
+  size_t cpos;
+  size_t cend;
+  ssize_t cbad;
+  unsigned int cval;
+
+  cpos = start;
+  for (i = 0; i < count; i++)
+  {
+    start = cpos;
+    badness = 0x7FFFFFFF;
+    best = -1;
+    for (j = i; j < count; j++)
+    {
+      cval = perm[j];
+      cend = cpos + sizes[cval];
+      if (cpos % DBLOCK_SIZE == 0)
+      {
+        /* prefer placing the largest blocks first */
+        cbad = -(cend % DBLOCK_SIZE);
+      }
+      else
+      {
+        if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
+        {
+          /* Data fits into the same block! Prefer small left-overs! */
+          cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
+        }
+        else
+        {
+          /* Would have to waste space to re-align, add big factor, this
+           * case is a real loss (proportional to space wasted)! */
+          cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
+        }
+      }
+      if (cbad < badness)
+      {
+        best = j;
+        badness = cbad;
+      }
+    }
+    GNUNET_assert (best != -1);
+    tmp = perm[i];
+    perm[i] = perm[best];
+    perm[best] = tmp;
+    cpos += sizes[perm[i]];
+    cpos = do_align (start, cpos);
+  }
+}
+
+
+/**
+ * Finish building the directory.  Frees the
+ * builder context and returns the directory
+ * in-memory.
+ *
+ * @param bld directory to finish
+ * @param rsize set to the number of bytes needed
+ * @param rdata set to the encoded directory
+ * @return #GNUNET_OK on success
+ */
+int
+GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
+                                    size_t *rsize,
+                                    void **rdata)
+{
+  char *data;
+  char *sptr;
+  size_t *sizes;
+  unsigned int *perm;
+  unsigned int i;
+  unsigned int j;
+  struct BuilderEntry *pos;
+  struct BuilderEntry **bes;
+  size_t size;
+  size_t psize;
+  size_t off;
+  ssize_t ret;
+  uint32_t big;
+
+  size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t);
+  size += GNUNET_FS_meta_data_get_serialized_size (bld->meta);
+  sizes = NULL;
+  perm = NULL;
+  bes = NULL;
+  if (0 < bld->count)
+  {
+    sizes = GNUNET_new_array (bld->count,
+                              size_t);
+    perm = GNUNET_new_array (bld->count,
+                             unsigned int);
+    bes = GNUNET_new_array (bld->count,
+                            struct BuilderEntry *);
+    pos = bld->head;
+    for (i = 0; i < bld->count; i++)
+    {
+      perm[i] = i;
+      bes[i] = pos;
+      sizes[i] = pos->len;
+      pos = pos->next;
+    }
+    block_align (size, bld->count, sizes, perm);
+    /* compute final size with alignment */
+    for (i = 0; i < bld->count; i++)
+    {
+      psize = size;
+      size += sizes[perm[i]];
+      size = do_align (psize, size);
+    }
+  }
+  *rsize = size;
+  data = GNUNET_malloc_large (size);
+  if (data == NULL)
+  {
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                         "malloc");
+    *rsize = 0;
+    *rdata = NULL;
+    GNUNET_free (sizes);
+    GNUNET_free (perm);
+    GNUNET_free (bes);
+    return GNUNET_SYSERR;
+  }
+  *rdata = data;
+  GNUNET_memcpy (data,
+                 GNUNET_DIRECTORY_MAGIC,
+                 strlen (GNUNET_DIRECTORY_MAGIC));
+  off = strlen (GNUNET_DIRECTORY_MAGIC);
+
+  sptr = &data[off + sizeof(uint32_t)];
+  ret =
+    GNUNET_FS_meta_data_serialize (bld->meta,
+                                          &sptr,
+                                          size - off - sizeof(uint32_t),
+                                          GNUNET_FS_META_DATA_SERIALIZE_FULL);
+  GNUNET_assert (ret != -1);
+  big = htonl (ret);
+  GNUNET_memcpy (&data[off],
+                 &big,
+                 sizeof(uint32_t));
+  off += sizeof(uint32_t) + ret;
+  for (j = 0; j < bld->count; j++)
+  {
+    i = perm[j];
+    psize = off;
+    off += sizes[i];
+    off = do_align (psize, off);
+    GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
+    GNUNET_free (bes[i]);
+  }
+  GNUNET_free (sizes);
+  GNUNET_free (perm);
+  GNUNET_free (bes);
+  GNUNET_assert (off == size);
+  GNUNET_FS_meta_data_destroy (bld->meta);
+  GNUNET_free (bld);
+  return GNUNET_OK;
+}
+
+
+/* end of fs_directory.c */
diff --git a/src/service/fs/fs_dirmetascan.c b/src/service/fs/fs_dirmetascan.c
new file mode 100644
index 000000000..2379e29ce
--- /dev/null
+++ b/src/service/fs/fs_dirmetascan.c
@@ -0,0 +1,496 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2005-2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_dirmetascan.c
+ * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem'
+ *        from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.
+ * @author LRN
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_scheduler_lib.h"
+#include <pthread.h>
+
+
+/**
+ * An opaque structure a pointer to which is returned to the
+ * caller to be used to control the scanner.
+ */
+struct GNUNET_FS_DirScanner
+{
+  /**
+   * Helper process.
+   */
+  struct GNUNET_HELPER_Handle *helper;
+
+  /**
+   * Expanded filename (as given by the scan initiator).
+   * The scanner thread stores a copy here, and frees it when it finishes.
+   */
+  char *filename_expanded;
+
+  /**
+   * Second argument to helper process.
+   */
+  char *ex_arg;
+
+  /**
+   * The function that will be called every time there's a progress
+   * message.
+   */
+  GNUNET_FS_DirScannerProgressCallback progress_callback;
+
+  /**
+   * A closure for progress_callback.
+   */
+  void *progress_callback_cls;
+
+  /**
+   * After the scan is finished, it will contain a pointer to the
+   * top-level directory entry in the directory tree built by the
+   * scanner.
+   */
+  struct GNUNET_FS_ShareTreeItem *toplevel;
+
+  /**
+   * Current position during processing.
+   */
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  /**
+   * Task scheduled when we are done.
+   */
+  struct GNUNET_SCHEDULER_Task *stop_task;
+
+  /**
+   * Arguments for helper.
+   */
+  char *args[4];
+};
+
+
+/**
+ * Abort the scan.  Must not be called from within the progress_callback
+ * function.
+ *
+ * @param ds directory scanner structure
+ */
+void
+GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
+{
+  /* terminate helper */
+  if (NULL != ds->helper)
+    GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
+
+  /* free resources */
+  if (NULL != ds->toplevel)
+    GNUNET_FS_share_tree_free (ds->toplevel);
+  if (NULL != ds->stop_task)
+    GNUNET_SCHEDULER_cancel (ds->stop_task);
+  GNUNET_free (ds->ex_arg);
+  GNUNET_free (ds->filename_expanded);
+  GNUNET_free (ds);
+}
+
+
+struct GNUNET_FS_ShareTreeItem *
+GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
+{
+  struct GNUNET_FS_ShareTreeItem *result;
+
+  /* check that we're actually done */
+  GNUNET_assert (NULL == ds->helper);
+  /* preserve result */
+  result = ds->toplevel;
+  ds->toplevel = NULL;
+  GNUNET_FS_directory_scan_abort (ds);
+  return result;
+}
+
+
+/**
+ * Move in the directory from the given position to the next file
+ * in DFS traversal.
+ *
+ * @param pos current position
+ * @return next file, NULL for none
+ */
+static struct GNUNET_FS_ShareTreeItem *
+advance (struct GNUNET_FS_ShareTreeItem *pos)
+{
+  int moved;
+
+  GNUNET_assert (NULL != pos);
+  moved = 0; /* must not terminate, even on file, otherwise "normal" */
+  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
+  {
+    if ((moved != -1) && (NULL != pos->children_head))
+    {
+      pos = pos->children_head;
+      moved = 1;     /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->next)
+    {
+      pos = pos->next;
+      moved = 1;     /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->parent)
+    {
+      pos = pos->parent;
+      moved = -1;     /* force move to 'next' or 'parent' */
+      continue;
+    }
+    /* no more options, end of traversal */
+    return NULL;
+  }
+  return pos;
+}
+
+
+/**
+ * Add another child node to the tree.
+ *
+ * @param parent parent of the child, NULL for top level
+ * @param filename name of the file or directory
+ * @param is_directory GNUNET_YES for directories
+ * @return new entry that was just created
+ */
+static struct GNUNET_FS_ShareTreeItem *
+expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
+             const char *filename,
+             int is_directory)
+{
+  struct GNUNET_FS_ShareTreeItem *chld;
+  size_t slen;
+
+  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
+  chld->parent = parent;
+  chld->filename = GNUNET_strdup (filename);
+  GNUNET_asprintf (&chld->short_filename,
+                   "%s%s",
+                   GNUNET_STRINGS_get_short_name (filename),
+                   is_directory == GNUNET_YES ? "/" : "");
+  /* make sure we do not end with '//' */
+  slen = strlen (chld->short_filename);
+  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
+      (chld->short_filename[slen - 2] == '/'))
+    chld->short_filename[slen - 1] = '\0';
+  chld->is_directory = is_directory;
+  if (NULL != parent)
+    GNUNET_CONTAINER_DLL_insert (parent->children_head,
+                                 parent->children_tail,
+                                 chld);
+  return chld;
+}
+
+
+/**
+ * Task run last to shut everything down.
+ *
+ * @param cls the 'struct GNUNET_FS_DirScanner'
+ */
+static void
+finish_scan (void *cls)
+{
+  struct GNUNET_FS_DirScanner *ds = cls;
+
+  ds->stop_task = NULL;
+  if (NULL != ds->helper)
+  {
+    GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
+    ds->helper = NULL;
+  }
+  ds->progress_callback (ds->progress_callback_cls,
+                         NULL,
+                         GNUNET_SYSERR,
+                         GNUNET_FS_DIRSCANNER_FINISHED);
+}
+
+
+/**
+ * Called every time there is data to read from the scanner.
+ * Calls the scanner progress handler.
+ *
+ * @param cls the closure (directory scanner object)
+ * @param msg message from the helper process
+ * @return #GNUNET_OK on success,
+ *    #GNUNET_NO to stop further processing (no error)
+ *    #GNUNET_SYSERR to stop further processing with error
+ */
+static int
+process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_DirScanner *ds = cls;
+  const char *filename;
+  size_t left;
+
+#if 0
+  fprintf (stderr,
+           "DMS parses %u-byte message of type %u\n",
+           (unsigned int) ntohs (msg->size),
+           (unsigned int) ntohs (msg->type));
+#endif
+  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
+  filename = (const char *) &msg[1];
+  switch (ntohs (msg->type))
+  {
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
+    if (filename[left - 1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls,
+                           filename,
+                           GNUNET_NO,
+                           GNUNET_FS_DIRSCANNER_FILE_START);
+    if (NULL == ds->toplevel)
+    {
+      ds->toplevel = expand_tree (ds->pos, filename, GNUNET_NO);
+    }
+    else
+    {
+      GNUNET_assert (NULL != ds->pos);
+      (void) expand_tree (ds->pos, filename, GNUNET_NO);
+    }
+    return GNUNET_OK;
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
+    if (filename[left - 1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 == strcmp ("..", filename))
+    {
+      if (NULL == ds->pos)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      ds->pos = ds->pos->parent;
+      return GNUNET_OK;
+    }
+    ds->progress_callback (ds->progress_callback_cls,
+                           filename,
+                           GNUNET_YES,
+                           GNUNET_FS_DIRSCANNER_FILE_START);
+    ds->pos = expand_tree (ds->pos, filename, GNUNET_YES);
+    if (NULL == ds->toplevel)
+      ds->toplevel = ds->pos;
+    return GNUNET_OK;
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
+    break;
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
+    if ('\0' != filename[left - 1])
+      break;
+    ds->progress_callback (ds->progress_callback_cls,
+                           filename,
+                           GNUNET_SYSERR,
+                           GNUNET_FS_DIRSCANNER_FILE_IGNORED);
+    return GNUNET_OK;
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (NULL == ds->toplevel)
+      break;
+    ds->progress_callback (ds->progress_callback_cls,
+                           NULL,
+                           GNUNET_SYSERR,
+                           GNUNET_FS_DIRSCANNER_ALL_COUNTED);
+    ds->pos = ds->toplevel;
+    if (GNUNET_YES == ds->pos->is_directory)
+      ds->pos = advance (ds->pos);
+    return GNUNET_OK;
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: {
+      size_t nlen;
+      const char *end;
+
+      if (NULL == ds->pos)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      end = memchr (filename, 0, left);
+      if (NULL == end)
+      {
+        GNUNET_break (0);
+        break;
+      }
+      end++;
+      nlen = end - filename;
+      left -= nlen;
+      if (0 != strcmp (filename, ds->pos->filename))
+      {
+        GNUNET_break (0);
+        break;
+      }
+      ds->progress_callback (ds->progress_callback_cls,
+                             filename,
+                             GNUNET_YES,
+                             GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
+      if (0 < left)
+      {
+        ds->pos->meta = GNUNET_FS_meta_data_deserialize (end, left);
+        if (NULL == ds->pos->meta)
+        {
+          GNUNET_break (0);
+          break;
+        }
+        /* having full filenames is too dangerous; always make sure we clean them up */
+        GNUNET_FS_meta_data_delete (ds->pos->meta,
+                                    EXTRACTOR_METATYPE_FILENAME,
+                                    NULL,
+                                    0);
+        /* instead, put in our 'safer' original filename */
+        GNUNET_FS_meta_data_insert (ds->pos->meta,
+                                    "<libgnunetfs>",
+                                    EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+                                    EXTRACTOR_METAFORMAT_UTF8,
+                                    "text/plain",
+                                    ds->pos->short_filename,
+                                    strlen (ds->pos->short_filename)
+                                    + 1);
+      }
+      ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (
+        ds->pos->meta);
+      ds->pos = advance (ds->pos);
+      return GNUNET_OK;
+    }
+
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
+    if (NULL != ds->pos)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (NULL == ds->toplevel)
+      break;
+    ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, ds);
+    return GNUNET_OK;
+
+  default:
+    GNUNET_break (0);
+    break;
+  }
+  ds->progress_callback (ds->progress_callback_cls,
+                         NULL,
+                         GNUNET_SYSERR,
+                         GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called if our helper process died.
+ *
+ * @param cls the 'struct GNUNET_FS_DirScanner' callback.
+ */
+static void
+helper_died_cb (void *cls)
+{
+  struct GNUNET_FS_DirScanner *ds = cls;
+
+  ds->helper = NULL;
+  if (NULL != ds->stop_task)
+    return; /* normal death, was finished */
+  ds->progress_callback (ds->progress_callback_cls,
+                         NULL,
+                         GNUNET_SYSERR,
+                         GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+}
+
+
+/**
+ * Start a directory scanner thread.
+ *
+ * @param filename name of the directory to scan
+ * @param disable_extractor #GNUNET_YES to not run libextractor on files (only
+ *        build a tree)
+ * @param ex if not NULL, must be a list of extra plugins for extractor
+ * @param cb the callback to call when there are scanning progress messages
+ * @param cb_cls closure for 'cb'
+ * @return directory scanner object to be used for controlling the scanner
+ */
+struct GNUNET_FS_DirScanner *
+GNUNET_FS_directory_scan_start (const char *filename,
+                                int disable_extractor,
+                                const char *ex,
+                                GNUNET_FS_DirScannerProgressCallback cb,
+                                void *cb_cls)
+{
+  struct stat sbuf;
+  char *filename_expanded;
+  struct GNUNET_FS_DirScanner *ds;
+
+  if (0 != stat (filename, &sbuf))
+    return NULL;
+  filename_expanded = GNUNET_STRINGS_filename_expand (filename);
+  if (NULL == filename_expanded)
+    return NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting to scan directory `%s'\n",
+              filename_expanded);
+  ds = GNUNET_new (struct GNUNET_FS_DirScanner);
+  ds->progress_callback = cb;
+  ds->progress_callback_cls = cb_cls;
+  ds->filename_expanded = filename_expanded;
+  if (disable_extractor)
+    ds->ex_arg = GNUNET_strdup ("-");
+  else
+    ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
+  ds->args[0] = "gnunet-helper-fs-publish";
+  ds->args[1] = ds->filename_expanded;
+  ds->args[2] = ds->ex_arg;
+  ds->args[3] = NULL;
+  ds->helper = GNUNET_HELPER_start (GNUNET_NO,
+                                    "gnunet-helper-fs-publish",
+                                    ds->args,
+                                    &process_helper_msgs,
+                                    &helper_died_cb,
+                                    ds);
+  if (NULL == ds->helper)
+  {
+    GNUNET_free (filename_expanded);
+    GNUNET_free (ds);
+    return NULL;
+  }
+  return ds;
+}
+
+
+/* end of fs_dirmetascan.c */
diff --git a/src/service/fs/fs_download.c b/src/service/fs/fs_download.c
new file mode 100644
index 000000000..cee64466d
--- /dev/null
+++ b/src/service/fs/fs_download.c
@@ -0,0 +1,2338 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/fs_download.c
+ * @brief download methods
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+
+
+/**
+ * Determine if the given download (options and meta data) should cause
+ * use to try to do a recursive download.
+ */
+static int
+is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
+{
+  return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
+         ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
+          ((NULL == dc->meta) &&
+           ((NULL == dc->filename) ||
+            ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
+             (NULL != strstr (dc->filename + strlen (dc->filename)
+                              - strlen (GNUNET_FS_DIRECTORY_EXT),
+                              GNUNET_FS_DIRECTORY_EXT))))));
+}
+
+
+/**
+ * We're storing the IBLOCKS after the DBLOCKS on disk (so that we
+ * only have to truncate the file once we're done).
+ *
+ * Given the offset of a block (with respect to the DBLOCKS) and its
+ * depth, return the offset where we would store this block in the
+ * file.
+ *
+ * @param fsize overall file size
+ * @param off offset of the block in the file
+ * @param depth depth of the block in the tree, 0 for DBLOCK
+ * @return off for DBLOCKS (depth == treedepth),
+ *         otherwise an offset past the end
+ *         of the file that does not overlap
+ *         with the range for any other block
+ */
+static uint64_t
+compute_disk_offset (uint64_t fsize, uint64_t off, unsigned int depth)
+{
+  unsigned int i;
+  uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
+  uint64_t loff; /* where do IBlocks for depth "i" start? */
+  unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
+
+  if (0 == depth)
+    return off;
+  /* first IBlocks start at the end of file, rounded up
+   * to full DBLOCK_SIZE */
+  loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
+  lsize =
+    ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof(struct ContentHashKey);
+  GNUNET_assert (0 == (off % DBLOCK_SIZE));
+  ioff = (off / DBLOCK_SIZE);
+  for (i = 1; i < depth; i++)
+  {
+    loff += lsize;
+    lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
+    GNUNET_assert (lsize > 0);
+    GNUNET_assert (0 == (ioff % CHK_PER_INODE));
+    ioff /= CHK_PER_INODE;
+  }
+  return loff + ioff * sizeof(struct ContentHashKey);
+}
+
+
+/**
+ * Fill in all of the generic fields for a download event and call the
+ * callback.
+ *
+ * @param pi structure to fill in
+ * @param dc overall download context
+ */
+void
+GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                 struct GNUNET_FS_DownloadContext *dc)
+{
+  pi->value.download.dc = dc;
+  pi->value.download.cctx = dc->client_info;
+  pi->value.download.pctx =
+    (NULL == dc->parent) ? NULL : dc->parent->client_info;
+  pi->value.download.sctx =
+    (NULL == dc->search) ? NULL : dc->search->client_info;
+  pi->value.download.uri = dc->uri;
+  pi->value.download.filename = dc->filename;
+  pi->value.download.size = dc->length;
+  /* FIXME: Fix duration calculation to account for pauses */
+  pi->value.download.duration =
+    GNUNET_TIME_absolute_get_duration (dc->start_time);
+  pi->value.download.completed = dc->completed;
+  pi->value.download.anonymity = dc->anonymity;
+  pi->value.download.eta =
+    GNUNET_TIME_calculate_eta (dc->start_time, dc->completed, dc->length);
+  pi->value.download.is_active = (NULL == dc->mq) ? GNUNET_NO : GNUNET_YES;
+  pi->fsh = dc->h;
+  if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
+    dc->client_info = dc->h->upcb (dc->h->upcb_cls, pi);
+  else
+    dc->client_info = GNUNET_FS_search_probe_progress_ (NULL, pi);
+}
+
+
+/**
+ * Closure for iterator processing results.
+ */
+struct ProcessResultClosure
+{
+  /**
+   * Hash of data.
+   */
+  struct GNUNET_HashCode query;
+
+  /**
+   * Data found in P2P network.
+   */
+  const void *data;
+
+  /**
+   * Our download context.
+   */
+  struct GNUNET_FS_DownloadContext *dc;
+
+  /**
+   * When did we last transmit the request?
+   */
+  struct GNUNET_TIME_Absolute last_transmission;
+
+  /**
+   * Number of bytes in data.
+   */
+  size_t size;
+
+  /**
+   * Type of data.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Flag to indicate if this block should be stored on disk.
+   */
+  int do_store;
+
+  /**
+   * how much respect did we offer to get this reply?
+   */
+  uint32_t respect_offered;
+
+  /**
+   * how often did we transmit the query?
+   */
+  uint32_t num_transmissions;
+};
+
+
+/**
+ * Iterator over entries in the pending requests in the 'active' map for the
+ * reply that we just got.
+ *
+ * @param cls closure (our `struct ProcessResultClosure`)
+ * @param key query for the given value / request
+ * @param value value in the hash map (a `struct DownloadRequest`)
+ * @return #GNUNET_YES (we should continue to iterate); unless serious error
+ */
+static int
+process_result_with_request (void *cls,
+                             const struct GNUNET_HashCode *key,
+                             void *value);
+
+
+/**
+ * We've found a matching block without downloading it.
+ * Encrypt it and pass it to our "receive" function as
+ * if we had received it from the network.
+ *
+ * @param dc download in question
+ * @param chk request this relates to
+ * @param dr request details
+ * @param block plaintext data matching request
+ * @param len number of bytes in block
+ * @param do_store should we still store the block on disk?
+ * @return GNUNET_OK on success
+ */
+static int
+encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc,
+                        const struct ContentHashKey *chk,
+                        struct DownloadRequest *dr,
+                        const char *block,
+                        size_t len,
+                        int do_store)
+{
+  struct ProcessResultClosure prc;
+  char enc[len];
+  struct GNUNET_CRYPTO_SymmetricSessionKey sk;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_HashCode query;
+
+  GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
+  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (block, len, &sk, &iv, enc))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_CRYPTO_hash (enc, len, &query);
+  if (0 != memcmp (&query, &chk->query, sizeof(struct GNUNET_HashCode)))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "Matching %u byte block for `%s' at offset %llu already present, no need for download!\n",
+    (unsigned int) len,
+    dc->filename,
+    (unsigned long long) dr->offset);
+  /* already got it! */
+  prc.dc = dc;
+  prc.data = enc;
+  prc.size = len;
+  prc.type = (0 == dr->depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK
+             : GNUNET_BLOCK_TYPE_FS_IBLOCK;
+  prc.query = chk->query;
+  prc.do_store = do_store;
+  prc.last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS;
+  process_result_with_request (&prc, &chk->key, dr);
+  return GNUNET_OK;
+}
+
+
+/**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * We found an entry in a directory.  Check if the respective child
+ * already exists and if not create the respective child download.
+ *
+ * @param cls the parent download
+ * @param filename name of the file in the directory
+ * @param uri URI of the file (CHK or LOC)
+ * @param meta meta data of the file
+ * @param length number of bytes in data
+ * @param data contents of the file (or NULL if they were not inlined)
+ */
+static void
+trigger_recursive_download (void *cls,
+                            const char *filename,
+                            const struct GNUNET_FS_Uri *uri,
+                            const struct GNUNET_FS_MetaData *meta,
+                            size_t length,
+                            const void *data);
+
+
+/**
+ * We're done downloading a directory.  Open the file and
+ * trigger all of the (remaining) child downloads.
+ *
+ * @param dc context of download that just completed
+ */
+static void
+full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
+{
+  size_t size;
+  uint64_t size64;
+  void *data;
+  struct GNUNET_DISK_FileHandle *h;
+  struct GNUNET_DISK_MapHandle *m;
+
+  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
+  size = (size_t) size64;
+  if (size64 != (uint64_t) size)
+  {
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_ERROR,
+      _ (
+        "Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
+    return;
+  }
+  if (NULL != dc->filename)
+  {
+    h = GNUNET_DISK_file_open (dc->filename,
+                               GNUNET_DISK_OPEN_READ,
+                               GNUNET_DISK_PERM_NONE);
+  }
+  else
+  {
+    GNUNET_assert (NULL != dc->temp_filename);
+    h = GNUNET_DISK_file_open (dc->temp_filename,
+                               GNUNET_DISK_OPEN_READ,
+                               GNUNET_DISK_PERM_NONE);
+  }
+  if (NULL == h)
+    return; /* oops */
+  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
+  if (NULL == data)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Directory too large for system address space\n"));
+  }
+  else
+  {
+    if (GNUNET_OK !=
+        GNUNET_FS_directory_list_contents (size,
+                                           data,
+                                           0,
+                                           &trigger_recursive_download,
+                                           dc))
+    {
+      GNUNET_log (
+        GNUNET_ERROR_TYPE_WARNING,
+        _ (
+          "Failed to access full directory contents of `%s' for recursive download\n"),
+        dc->filename);
+    }
+    GNUNET_DISK_file_unmap (m);
+  }
+  GNUNET_DISK_file_close (h);
+  if (NULL == dc->filename)
+  {
+    if (0 != unlink (dc->temp_filename))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
+                                dc->temp_filename);
+    GNUNET_free (dc->temp_filename);
+    dc->temp_filename = NULL;
+  }
+}
+
+
+/**
+ * Check if all child-downloads have completed (or trigger them if
+ * necessary) and once we're completely done, signal completion (and
+ * possibly recurse to parent).  This function MUST be called when the
+ * download of a file itself is done or when the download of a file is
+ * done and then later a direct child download has completed (and
+ * hence this download may complete itself).
+ *
+ * @param dc download to check for completion of children
+ */
+static void
+check_completed (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  struct GNUNET_FS_DownloadContext *pos;
+
+  /* first, check if we need to download children */
+  if (is_recursive_download (dc))
+    full_recursive_download (dc);
+  /* then, check if children are done already */
+  for (pos = dc->child_head; NULL != pos; pos = pos->next)
+  {
+    if ((NULL == pos->emsg) && (pos->completed < pos->length))
+      return;   /* not done yet */
+    if ((NULL != pos->child_head) && (pos->has_finished != GNUNET_YES))
+      return;   /* not transitively done yet */
+  }
+  /* All of our children are done, so mark this download done */
+  dc->has_finished = GNUNET_YES;
+  if (NULL != dc->job_queue)
+  {
+    GNUNET_FS_dequeue_ (dc->job_queue);
+    dc->job_queue = NULL;
+  }
+  if (NULL != dc->task)
+  {
+    GNUNET_SCHEDULER_cancel (dc->task);
+    dc->task = NULL;
+  }
+  if (NULL != dc->rfh)
+  {
+    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
+    dc->rfh = NULL;
+  }
+  GNUNET_FS_download_sync_ (dc);
+
+  /* signal completion */
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+
+  /* let parent know */
+  if (NULL != dc->parent)
+    check_completed (dc->parent);
+}
+
+
+/**
+ * We got a block of plaintext data (from the meta data).
+ * Try it for upward reconstruction of the data.  On success,
+ * the top-level block will move to state BRS_DOWNLOAD_UP.
+ *
+ * @param dc context for the download
+ * @param dr download request to match against
+ * @param data plaintext data, starting from the beginning of the file
+ * @param data_len number of bytes in data
+ */
+static void
+try_match_block (struct GNUNET_FS_DownloadContext *dc,
+                 struct DownloadRequest *dr,
+                 const char *data,
+                 size_t data_len)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  unsigned int i;
+  char enc[DBLOCK_SIZE];
+  struct ContentHashKey chks[CHK_PER_INODE];
+  struct ContentHashKey in_chk;
+  struct GNUNET_CRYPTO_SymmetricSessionKey sk;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  size_t dlen;
+  struct DownloadRequest *drc;
+  struct GNUNET_DISK_FileHandle *fh;
+  int complete;
+  const char *fn;
+  const char *odata;
+  size_t odata_len;
+
+  odata = data;
+  odata_len = data_len;
+  if (BRS_DOWNLOAD_UP == dr->state)
+    return;
+  if (dr->depth > 0)
+  {
+    if ((dc->offset > 0) ||
+        (dc->length < GNUNET_ntohll (dc->uri->data.chk.file_length)))
+    {
+      /* NOTE: this test is not tight, but should suffice; the issue
+         here is that 'dr->num_children' may inherently only specify a
+         smaller range than what is in the original file;
+         thus, reconstruction of (some) inner blocks will fail.
+         FIXME: we might eventually want to write a tighter test to
+         maximize the circumstances under which we do succeed with
+         IBlock reconstruction. (need good tests though). */return;
+    }
+    complete = GNUNET_YES;
+    for (i = 0; i < dr->num_children; i++)
+    {
+      drc = dr->children[i];
+      try_match_block (dc, drc, data, data_len);
+      if (drc->state != BRS_RECONSTRUCT_META_UP)
+        complete = GNUNET_NO;
+      else
+        chks[i] = drc->chk;
+    }
+    if (GNUNET_YES != complete)
+      return;
+    data = (const char *) chks;
+    dlen = dr->num_children * sizeof(struct ContentHashKey);
+  }
+  else
+  {
+    if (dr->offset > data_len)
+      return;   /* oops */
+    dlen = GNUNET_MIN (data_len - dr->offset, DBLOCK_SIZE);
+  }
+  GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key);
+  GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
+  if (-1 ==
+      GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc))
+  {
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
+  switch (dr->state)
+  {
+  case BRS_INIT:
+    dr->chk = in_chk;
+    dr->state = BRS_RECONSTRUCT_META_UP;
+    break;
+
+  case BRS_CHK_SET:
+    if (0 != memcmp (&in_chk, &dr->chk, sizeof(struct ContentHashKey)))
+    {
+      /* other peer provided bogus meta data */
+      GNUNET_break_op (0);
+      break;
+    }
+    /* write block to disk */
+    fn = (NULL != dc->filename) ? dc->filename : dc->temp_filename;
+    if (NULL != fn)
+    {
+      fh = GNUNET_DISK_file_open (fn,
+                                  GNUNET_DISK_OPEN_READWRITE
+                                  | GNUNET_DISK_OPEN_CREATE
+                                  | GNUNET_DISK_OPEN_TRUNCATE,
+                                  GNUNET_DISK_PERM_USER_READ
+                                  | GNUNET_DISK_PERM_USER_WRITE
+                                  | GNUNET_DISK_PERM_GROUP_READ
+                                  | GNUNET_DISK_PERM_OTHER_READ);
+      if (NULL == fh)
+      {
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", fn);
+        GNUNET_asprintf (&dc->emsg,
+                         _ ("Failed to open file `%s' for writing"),
+                         fn);
+        GNUNET_DISK_file_close (fh);
+        dr->state = BRS_ERROR;
+        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+        pi.value.download.specifics.error.message = dc->emsg;
+        GNUNET_FS_download_make_status_ (&pi, dc);
+        return;
+      }
+      if (data_len != GNUNET_DISK_file_write (fh, odata, odata_len))
+      {
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "write", fn);
+        GNUNET_asprintf (&dc->emsg,
+                         _ ("Failed to open file `%s' for writing"),
+                         fn);
+        GNUNET_DISK_file_close (fh);
+        dr->state = BRS_ERROR;
+        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+        pi.value.download.specifics.error.message = dc->emsg;
+        GNUNET_FS_download_make_status_ (&pi, dc);
+        return;
+      }
+      GNUNET_DISK_file_close (fh);
+    }
+    /* signal success */
+    dr->state = BRS_DOWNLOAD_UP;
+    dc->completed = dc->length;
+    GNUNET_FS_download_sync_ (dc);
+    pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
+    pi.value.download.specifics.progress.data = data;
+    pi.value.download.specifics.progress.offset = 0;
+    pi.value.download.specifics.progress.data_len = dlen;
+    pi.value.download.specifics.progress.depth = 0;
+    pi.value.download.specifics.progress.respect_offered = 0;
+    pi.value.download.specifics.progress.block_download_duration =
+      GNUNET_TIME_UNIT_ZERO;
+    GNUNET_FS_download_make_status_ (&pi, dc);
+    if ((NULL != dc->filename) &&
+        (0 != truncate (dc->filename,
+                        GNUNET_ntohll (dc->uri->data.chk.file_length))))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "truncate",
+                                dc->filename);
+    check_completed (dc);
+    break;
+
+  default:
+    /* how did we get here? */
+    GNUNET_break (0);
+    break;
+  }
+}
+
+
+/**
+ * Type of a function that libextractor calls for each
+ * meta data item found.  If we find full data meta data,
+ * call 'try_match_block' on it.
+ *
+ * @param cls our 'struct GNUNET_FS_DownloadContext*'
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return 0 to continue extracting, 1 to abort
+ */
+static int
+match_full_data (void *cls,
+                 const char *plugin_name,
+                 enum EXTRACTOR_MetaType type,
+                 enum EXTRACTOR_MetaFormat format,
+                 const char *data_mime_type,
+                 const char *data,
+                 size_t data_len)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+
+  if (EXTRACTOR_METATYPE_GNUNET_FULL_DATA != type)
+    return 0;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Found %u bytes of FD!\n",
+              (unsigned int) data_len);
+  if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
+  {
+    GNUNET_break_op (0);
+    return 1;   /* bogus meta data */
+  }
+  try_match_block (dc, dc->top_request, data, data_len);
+  return 1;
+}
+
+
+/**
+ * Set the state of the given download request to
+ * BRS_DOWNLOAD_UP and propagate it up the tree.
+ *
+ * @param dr download request that is done
+ */
+static void
+propagate_up (struct DownloadRequest *dr)
+{
+  unsigned int i;
+
+  do
+  {
+    dr->state = BRS_DOWNLOAD_UP;
+    dr = dr->parent;
+    if (NULL == dr)
+      break;
+    for (i = 0; i < dr->num_children; i++)
+      if (dr->children[i]->state != BRS_DOWNLOAD_UP)
+        break;
+  }
+  while (i == dr->num_children);
+}
+
+
+/**
+ * Try top-down reconstruction.  Before, the given request node
+ * must have the state BRS_CHK_SET.  Afterwards, more nodes may
+ * have that state or advanced to BRS_DOWNLOAD_DOWN or even
+ * BRS_DOWNLOAD_UP.  It is also possible to get BRS_ERROR on the
+ * top level.
+ *
+ * @param dc overall download this block belongs to
+ * @param dr block to reconstruct
+ */
+static void
+try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc,
+                             struct DownloadRequest *dr)
+{
+  uint64_t off;
+  char block[DBLOCK_SIZE];
+  struct GNUNET_HashCode key;
+  uint64_t total;
+  size_t len;
+  unsigned int i;
+  struct DownloadRequest *drc;
+  uint64_t child_block_size;
+  const struct ContentHashKey *chks;
+  int up_done;
+
+  GNUNET_assert (NULL != dc->rfh);
+  GNUNET_assert (BRS_CHK_SET == dr->state);
+  total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
+  GNUNET_assert (dr->depth < dc->treedepth);
+  len = GNUNET_FS_tree_calculate_block_size (total, dr->offset, dr->depth);
+  GNUNET_assert (len <= DBLOCK_SIZE);
+  off = compute_disk_offset (total, dr->offset, dr->depth);
+  if (dc->old_file_size < off + len)
+    return; /* failure */
+  if (off != GNUNET_DISK_file_seek (dc->rfh, off, GNUNET_DISK_SEEK_SET))
+  {
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "seek", dc->filename);
+    return;   /* failure */
+  }
+  if (len != GNUNET_DISK_file_read (dc->rfh, block, len))
+  {
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "read", dc->filename);
+    return;   /* failure */
+  }
+  GNUNET_CRYPTO_hash (block, len, &key);
+  if (0 != memcmp (&key, &dr->chk.key, sizeof(struct GNUNET_HashCode)))
+    return; /* mismatch */
+  if (GNUNET_OK !=
+      encrypt_existing_match (dc, &dr->chk, dr, block, len, GNUNET_NO))
+  {
+    /* hash matches but encrypted block does not, really bad */
+    dr->state = BRS_ERROR;
+    /* propagate up */
+    while (NULL != dr->parent)
+    {
+      dr = dr->parent;
+      dr->state = BRS_ERROR;
+    }
+    return;
+  }
+  /* block matches */
+  dr->state = BRS_DOWNLOAD_DOWN;
+
+  /* set CHKs for children */
+  up_done = GNUNET_YES;
+  chks = (const struct ContentHashKey *) block;
+  for (i = 0; i < dr->num_children; i++)
+  {
+    drc = dr->children[i];
+    GNUNET_assert (drc->offset >= dr->offset);
+    child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
+    GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
+    if (BRS_INIT == drc->state)
+    {
+      drc->state = BRS_CHK_SET;
+      drc->chk = chks[drc->chk_idx];
+      try_top_down_reconstruction (dc, drc);
+    }
+    if (BRS_DOWNLOAD_UP != drc->state)
+      up_done = GNUNET_NO;   /* children not all done */
+  }
+  if (GNUNET_YES == up_done)
+    propagate_up (dr); /* children all done (or no children...) */
+}
+
+
+/**
+ * Add entries to the message queue.
+ *
+ * @param cls our download context
+ * @param key unused
+ * @param entry entry of type `struct DownloadRequest`
+ * @return #GNUNET_OK
+ */
+static int
+retry_entry (void *cls, const struct GNUNET_HashCode *key, void *entry)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct DownloadRequest *dr = entry;
+  struct SearchMessage *sm;
+  struct GNUNET_MQ_Envelope *env;
+
+  env = GNUNET_MQ_msg (sm, GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
+  if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
+    sm->options = htonl (GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY);
+  else
+    sm->options = htonl (GNUNET_FS_SEARCH_OPTION_NONE);
+  if (0 == dr->depth)
+    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
+  else
+    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
+  sm->anonymity_level = htonl (dc->anonymity);
+  sm->target = dc->target;
+  sm->query = dr->chk.query;
+  GNUNET_MQ_send (dc->mq, env);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Schedule the download of the specified block in the tree.
+ *
+ * @param dc overall download this block belongs to
+ * @param dr request to schedule
+ */
+static void
+schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
+                         struct DownloadRequest *dr)
+{
+  unsigned int i;
+
+  switch (dr->state)
+  {
+  case BRS_INIT:
+    GNUNET_assert (0);
+    break;
+
+  case BRS_RECONSTRUCT_DOWN:
+    GNUNET_assert (0);
+    break;
+
+  case BRS_RECONSTRUCT_META_UP:
+    GNUNET_assert (0);
+    break;
+
+  case BRS_RECONSTRUCT_UP:
+    GNUNET_assert (0);
+    break;
+
+  case BRS_CHK_SET:
+    /* normal case, start download */
+    break;
+
+  case BRS_DOWNLOAD_DOWN:
+    for (i = 0; i < dr->num_children; i++)
+      schedule_block_download (dc, dr->children[i]);
+    return;
+
+  case BRS_DOWNLOAD_UP:
+    /* We're done! */
+    return;
+
+  case BRS_ERROR:
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Scheduling download at offset %llu and depth %u for `%s'\n",
+              (unsigned long long) dr->offset,
+              dr->depth,
+              GNUNET_h2s (&dr->chk.query));
+  if (GNUNET_NO != GNUNET_CONTAINER_multihashmap_contains_value (dc->active,
+                                                                 &dr->chk.query,
+                                                                 dr))
+    return; /* already active */
+  GNUNET_CONTAINER_multihashmap_put (dc->active,
+                                     &dr->chk.query,
+                                     dr,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  if (NULL == dc->mq)
+    return; /* download not active */
+  retry_entry (dc, &dr->chk.query, dr);
+}
+
+
+#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
+
+/**
+ * We found an entry in a directory.  Check if the respective child
+ * already exists and if not create the respective child download.
+ *
+ * @param cls the parent download
+ * @param filename name of the file in the directory
+ * @param uri URI of the file (CHK or LOC)
+ * @param meta meta data of the file
+ * @param length number of bytes in data
+ * @param data contents of the file (or NULL if they were not inlined)
+ */
+static void
+trigger_recursive_download (void *cls,
+                            const char *filename,
+                            const struct GNUNET_FS_Uri *uri,
+                            const struct GNUNET_FS_MetaData *meta,
+                            size_t length,
+                            const void *data)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_DownloadContext *cpos;
+  char *temp_name;
+  char *fn;
+  char *us;
+  char *ext;
+  char *dn;
+  char *pos;
+  char *full_name;
+  char *sfn;
+
+  if (NULL == uri)
+    return; /* entry for the directory itself */
+  cpos = dc->child_head;
+  while (NULL != cpos)
+  {
+    if ((GNUNET_FS_uri_test_equal (uri, cpos->uri)) ||
+        ((NULL != filename) && (0 == strcmp (cpos->filename, filename))))
+      break;
+    cpos = cpos->next;
+  }
+  if (NULL != cpos)
+    return; /* already exists */
+  fn = NULL;
+  if (NULL == filename)
+  {
+    fn = GNUNET_FS_meta_data_suggest_filename (meta);
+    if (NULL == fn)
+    {
+      us = GNUNET_FS_uri_to_string (uri);
+      fn = GNUNET_strdup (&us[strlen (GNUNET_FS_URI_CHK_PREFIX)]);
+      GNUNET_free (us);
+    }
+    else if ('.' == fn[0])
+    {
+      ext = fn;
+      us = GNUNET_FS_uri_to_string (uri);
+      GNUNET_asprintf (&fn,
+                       "%s%s",
+                       &us[strlen (GNUNET_FS_URI_CHK_PREFIX)],
+                       ext);
+      GNUNET_free (ext);
+      GNUNET_free (us);
+    }
+    /* change '\' to '/' (this should have happened
+     * during insertion, but malicious peers may
+     * not have done this) */
+    while (NULL != (pos = strstr (fn, "\\")))
+      *pos = '/';
+    /* remove '../' everywhere (again, well-behaved
+     * peers don't do this, but don't trust that
+     * we did not get something nasty) */
+    while (NULL != (pos = strstr (fn, "../")))
+    {
+      pos[0] = '_';
+      pos[1] = '_';
+      pos[2] = '_';
+    }
+    filename = fn;
+  }
+  if (NULL == dc->filename)
+  {
+    full_name = NULL;
+  }
+  else
+  {
+    dn = GNUNET_strdup (dc->filename);
+    GNUNET_break (
+      (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
+      (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
+                       GNUNET_FS_DIRECTORY_EXT)));
+    sfn = GNUNET_strdup (filename);
+    while ((strlen (sfn) > 0) && ('/' == filename[strlen (sfn) - 1]))
+      sfn[strlen (sfn) - 1] = '\0';
+    if ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
+        (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
+                         GNUNET_FS_DIRECTORY_EXT)))
+      dn[strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
+    if ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
+        ((strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
+         (NULL == strstr (filename + strlen (filename)
+                          - strlen (GNUNET_FS_DIRECTORY_EXT),
+                          GNUNET_FS_DIRECTORY_EXT))))
+    {
+      GNUNET_asprintf (&full_name,
+                       "%s%s%s%s",
+                       dn,
+                       DIR_SEPARATOR_STR,
+                       sfn,
+                       GNUNET_FS_DIRECTORY_EXT);
+    }
+    else
+    {
+      GNUNET_asprintf (&full_name, "%s%s%s", dn, DIR_SEPARATOR_STR, sfn);
+    }
+    GNUNET_free (sfn);
+    GNUNET_free (dn);
+  }
+  if ((NULL != full_name) &&
+      (GNUNET_OK != GNUNET_DISK_directory_create_for_file (full_name)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ (
+                  "Failed to create directory for recursive download of `%s'\n"),
+                full_name);
+    GNUNET_free (full_name);
+    GNUNET_free (fn);
+    return;
+  }
+
+  temp_name = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Triggering recursive download of size %llu with %u bytes MD\n",
+              (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
+              (unsigned int) GNUNET_FS_meta_data_get_serialized_size (
+                meta));
+  GNUNET_FS_download_start (dc->h,
+                            uri,
+                            meta,
+                            full_name,
+                            temp_name,
+                            0,
+                            GNUNET_FS_uri_chk_get_file_size (uri),
+                            dc->anonymity,
+                            dc->options,
+                            NULL,
+                            dc);
+  GNUNET_free (full_name);
+  GNUNET_free (temp_name);
+  GNUNET_free (fn);
+}
+
+
+/**
+ * (recursively) free download request structure
+ *
+ * @param dr request to free
+ */
+void
+GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
+{
+  if (NULL == dr)
+    return;
+  for (unsigned int i = 0; i < dr->num_children; i++)
+    GNUNET_FS_free_download_request_ (dr->children[i]);
+  GNUNET_free (dr->children);
+  GNUNET_free (dr);
+}
+
+
+static int
+process_result_with_request (void *cls,
+                             const struct GNUNET_HashCode *key,
+                             void *value)
+{
+  struct ProcessResultClosure *prc = cls;
+  struct DownloadRequest *dr = value;
+  struct GNUNET_FS_DownloadContext *dc = prc->dc;
+  struct DownloadRequest *drc;
+  struct GNUNET_DISK_FileHandle *fh = NULL;
+  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  char pt[prc->size];
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t off;
+  size_t bs;
+  size_t app;
+  int i;
+  struct ContentHashKey *chkarr;
+
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "Received %u byte block `%s' matching pending request at depth %u and offset %llu/%llu\n",
+    (unsigned int) prc->size,
+    GNUNET_h2s (key),
+    dr->depth,
+    (unsigned long long) dr->offset,
+    (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
+  bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (
+                                              dc->uri->data.chk.file_length),
+                                            dr->offset,
+                                            dr->depth);
+  if (prc->size != bs)
+  {
+    GNUNET_asprintf (
+      &dc->emsg,
+      "Internal error or bogus download URI (expected %llu bytes at depth %u and offset %llu/%llu, got %llu bytes)",
+      (unsigned long long) bs,
+      dr->depth,
+      (unsigned long long) dr->offset,
+      (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
+      (unsigned long long) prc->size);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s\n", dc->emsg);
+    while (NULL != dr->parent)
+    {
+      dr->state = BRS_ERROR;
+      dr = dr->parent;
+    }
+    dr->state = BRS_ERROR;
+    goto signal_error;
+  }
+
+  (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, &prc->query, dr);
+  GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
+  if (-1 ==
+      GNUNET_CRYPTO_symmetric_decrypt (prc->data, prc->size, &skey, &iv, pt))
+  {
+    GNUNET_break (0);
+    dc->emsg = GNUNET_strdup (_ ("internal error decrypting content"));
+    goto signal_error;
+  }
+  off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
+                             dr->offset,
+                             dr->depth);
+  /* save to disk */
+  if ((GNUNET_YES == prc->do_store) &&
+      ((NULL != dc->filename) || (is_recursive_download (dc))) &&
+      ((dr->depth == dc->treedepth) ||
+       (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES))))
+  {
+    fh = GNUNET_DISK_file_open (NULL != dc->filename ? dc->filename
+                                : dc->temp_filename,
+                                GNUNET_DISK_OPEN_READWRITE
+                                | GNUNET_DISK_OPEN_CREATE,
+                                GNUNET_DISK_PERM_USER_READ
+                                | GNUNET_DISK_PERM_USER_WRITE
+                                | GNUNET_DISK_PERM_GROUP_READ
+                                | GNUNET_DISK_PERM_OTHER_READ);
+    if (NULL == fh)
+    {
+      GNUNET_asprintf (&dc->emsg,
+                       _ ("Download failed: could not open file `%s': %s"),
+                       dc->filename,
+                       strerror (errno));
+      goto signal_error;
+    }
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Saving decrypted block to disk at offset %llu\n",
+                (unsigned long long) off);
+    if ((off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)))
+    {
+      GNUNET_asprintf (&dc->emsg,
+                       _ ("Failed to seek to offset %llu in file `%s': %s"),
+                       (unsigned long long) off,
+                       dc->filename,
+                       strerror (errno));
+      goto signal_error;
+    }
+    if (prc->size != GNUNET_DISK_file_write (fh, pt, prc->size))
+    {
+      GNUNET_asprintf (
+        &dc->emsg,
+        _ ("Failed to write block of %u bytes at offset %llu in file `%s': %s"),
+        (unsigned int) prc->size,
+        (unsigned long long) off,
+        dc->filename,
+        strerror (errno));
+      goto signal_error;
+    }
+    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
+    fh = NULL;
+  }
+
+  if (0 == dr->depth)
+  {
+    /* DBLOCK, update progress and try recursion if applicable */
+    app = prc->size;
+    if (dr->offset < dc->offset)
+    {
+      /* starting offset begins in the middle of pt,
+       * do not count first bytes as progress */
+      GNUNET_assert (app > (dc->offset - dr->offset));
+      app -= (dc->offset - dr->offset);
+    }
+    if (dr->offset + prc->size > dc->offset + dc->length)
+    {
+      /* end of block is after relevant range,
+      * do not count last bytes as progress */
+      GNUNET_assert (app >
+                     (dr->offset + prc->size) - (dc->offset + dc->length));
+      app -= (dr->offset + prc->size) - (dc->offset + dc->length);
+    }
+    dc->completed += app;
+
+    /* do recursive download if option is set and either meta data
+     * says it is a directory or if no meta data is given AND filename
+     * ends in '.gnd' (top-level case) */
+    if (is_recursive_download (dc))
+      GNUNET_FS_directory_list_contents (prc->size,
+                                         pt,
+                                         off,
+                                         &trigger_recursive_download,
+                                         dc);
+  }
+  GNUNET_assert (dc->completed <= dc->length);
+  dr->state = BRS_DOWNLOAD_DOWN;
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
+  pi.value.download.specifics.progress.data = pt;
+  pi.value.download.specifics.progress.offset = dr->offset;
+  pi.value.download.specifics.progress.data_len = prc->size;
+  pi.value.download.specifics.progress.depth = dr->depth;
+  pi.value.download.specifics.progress.respect_offered = prc->respect_offered;
+  pi.value.download.specifics.progress.num_transmissions =
+    prc->num_transmissions;
+  if (prc->last_transmission.abs_value_us !=
+      GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us)
+    pi.value.download.specifics.progress.block_download_duration =
+      GNUNET_TIME_absolute_get_duration (prc->last_transmission);
+  else
+    pi.value.download.specifics.progress.block_download_duration =
+      GNUNET_TIME_UNIT_ZERO; /* found locally */
+  GNUNET_FS_download_make_status_ (&pi, dc);
+  if (0 == dr->depth)
+    propagate_up (dr);
+
+  if (dc->completed == dc->length)
+  {
+    /* download completed, signal */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Download completed, truncating file to desired length %llu\n",
+                (unsigned long long) GNUNET_ntohll (
+                  dc->uri->data.chk.file_length));
+    /* truncate file to size (since we store IBlocks at the end) */
+    if (NULL != dc->filename)
+    {
+      if (0 != truncate (dc->filename,
+                         GNUNET_ntohll (dc->uri->data.chk.file_length)))
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                  "truncate",
+                                  dc->filename);
+    }
+    GNUNET_assert (0 == dr->depth);
+    check_completed (dc);
+  }
+  if (0 == dr->depth)
+  {
+    /* bottom of the tree, no child downloads possible, just sync */
+    GNUNET_FS_download_sync_ (dc);
+    return GNUNET_YES;
+  }
+
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
+    dr->depth,
+    (unsigned long long) dr->offset);
+  GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
+  chkarr = (struct ContentHashKey *) pt;
+  for (i = dr->num_children - 1; i >= 0; i--)
+  {
+    drc = dr->children[i];
+    switch (drc->state)
+    {
+    case BRS_INIT:
+      if ((drc->chk_idx + 1) * sizeof(struct ContentHashKey) > prc->size)
+      {
+        /* 'chkarr' does not have enough space for this chk_idx;
+           internal error! */
+        GNUNET_break (0);
+        GNUNET_assert (0);
+        dc->emsg = GNUNET_strdup (_ ("internal error decoding tree"));
+        goto signal_error;
+      }
+      drc->chk = chkarr[drc->chk_idx];
+      drc->state = BRS_CHK_SET;
+      if (GNUNET_YES == dc->issue_requests)
+        schedule_block_download (dc, drc);
+      break;
+
+    case BRS_RECONSTRUCT_DOWN:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_RECONSTRUCT_META_UP:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_RECONSTRUCT_UP:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_CHK_SET:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_DOWNLOAD_DOWN:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_DOWNLOAD_UP:
+      GNUNET_assert (0);
+      break;
+
+    case BRS_ERROR:
+      GNUNET_assert (0);
+      break;
+
+    default:
+      GNUNET_assert (0);
+      break;
+    }
+  }
+  GNUNET_FS_download_sync_ (dc);
+  return GNUNET_YES;
+
+signal_error:
+  if (NULL != fh)
+    GNUNET_DISK_file_close (fh);
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+  pi.value.download.specifics.error.message = dc->emsg;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+  GNUNET_MQ_destroy (dc->mq);
+  dc->mq = NULL;
+  GNUNET_FS_free_download_request_ (dc->top_request);
+  dc->top_request = NULL;
+  if (NULL != dc->job_queue)
+  {
+    GNUNET_FS_dequeue_ (dc->job_queue);
+    dc->job_queue = NULL;
+  }
+  GNUNET_FS_download_sync_ (dc);
+  return GNUNET_NO;
+}
+
+
+/**
+ * Type of a function to call when we check the PUT message
+ * from the service.
+ *
+ * @param cls closure
+ * @param msg message received
+ */
+static int
+check_put (void *cls, const struct ClientPutMessage *cm)
+{
+  /* any varsize length is OK */
+  return GNUNET_OK;
+}
+
+
+/**
+ * Type of a function to call when we receive a message
+ * from the service.
+ *
+ * @param cls closure
+ * @param msg message received
+ */
+static void
+handle_put (void *cls, const struct ClientPutMessage *cm)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  uint16_t msize = ntohs (cm->header.size) - sizeof(*cm);
+  struct ProcessResultClosure prc;
+
+  prc.dc = dc;
+  prc.data = &cm[1];
+  prc.last_transmission = GNUNET_TIME_absolute_ntoh (cm->last_transmission);
+  prc.size = msize;
+  prc.type = ntohl (cm->type);
+  prc.do_store = GNUNET_YES;
+  prc.respect_offered = ntohl (cm->respect_offered);
+  prc.num_transmissions = ntohl (cm->num_transmissions);
+  GNUNET_CRYPTO_hash (prc.data, msize, &prc.query);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received result for query `%s' from FS service\n",
+              GNUNET_h2s (&prc.query));
+  GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
+                                              &prc.query,
+                                              &process_result_with_request,
+                                              &prc);
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_DownloadContext *`
+ * @param error error code
+ */
+static void
+download_mq_error_handler (void *cls, enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+
+  if (NULL != dc->mq)
+  {
+    GNUNET_MQ_destroy (dc->mq);
+    dc->mq = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Transmitting download request failed, trying to reconnect\n");
+  try_reconnect (dc);
+}
+
+
+/**
+ * Reconnect to the FS service and transmit our queries NOW.
+ *
+ * @param cls our download context
+ */
+static void
+do_reconnect (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] =
+  { GNUNET_MQ_hd_var_size (put,
+                           GNUNET_MESSAGE_TYPE_FS_PUT,
+                           struct ClientPutMessage,
+                           dc),
+    GNUNET_MQ_handler_end () };
+
+  dc->task = NULL;
+  dc->mq = GNUNET_CLIENT_connect (dc->h->cfg,
+                                  "fs",
+                                  handlers,
+                                  &download_mq_error_handler,
+                                  dc);
+  if (NULL == dc->mq)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Connecting to `%s'-service failed, will try again.\n",
+                "FS");
+    try_reconnect (dc);
+    return;
+  }
+  GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc);
+}
+
+
+/**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc)
+{
+  if (NULL != dc->mq)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Moving all requests back to pending list\n");
+    GNUNET_MQ_destroy (dc->mq);
+    dc->mq = NULL;
+  }
+  if (0 == dc->reconnect_backoff.rel_value_us)
+    dc->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
+  else
+    dc->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (dc->reconnect_backoff);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Will try to reconnect in %s\n",
+              GNUNET_STRINGS_relative_time_to_string (dc->reconnect_backoff,
+                                                      GNUNET_YES));
+  GNUNET_break (NULL != dc->job_queue);
+  dc->task =
+    GNUNET_SCHEDULER_add_delayed (dc->reconnect_backoff, &do_reconnect, dc);
+}
+
+
+/**
+ * We're allowed to ask the FS service for our blocks.  Start the download.
+ *
+ * @param cls the 'struct GNUNET_FS_DownloadContext'
+ */
+static void
+activate_fs_download (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  GNUNET_assert (NULL == dc->mq);
+  GNUNET_assert (NULL != dc->active);
+  do_reconnect (dc);
+  if (NULL != dc->mq)
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download activated\n");
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+}
+
+
+/**
+ * We must stop to ask the FS service for our blocks.  Pause the download.
+ *
+ * @param cls the `struct GNUNET_FS_DownloadContext`
+ */
+static void
+deactivate_fs_download (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download deactivated\n");
+  if (NULL != dc->mq)
+  {
+    GNUNET_MQ_destroy (dc->mq);
+    dc->mq = NULL;
+  }
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+}
+
+
+/**
+ * (recursively) Create a download request structure.
+ *
+ * @param parent parent of the current entry
+ * @param chk_idx index of the chk for this block in the parent block
+ * @param depth depth of the current entry, 0 are the DBLOCKs,
+ *              top level block is 'dc->treedepth - 1'
+ * @param dr_offset offset in the original file this block maps to
+ *              (as in, offset of the first byte of the first DBLOCK
+ *               in the subtree rooted in the returned download request tree)
+ * @param file_start_offset desired starting offset for the download
+ *             in the original file; requesting tree should not contain
+ *             DBLOCKs prior to the file_start_offset
+ * @param desired_length desired number of bytes the user wanted to access
+ *        (from file_start_offset).  Resulting tree should not contain
+ *        DBLOCKs after file_start_offset + file_length.
+ * @return download request tree for the given range of DBLOCKs at
+ *         the specified depth
+ */
+static struct DownloadRequest *
+create_download_request (struct DownloadRequest *parent,
+                         unsigned int chk_idx,
+                         unsigned int depth,
+                         uint64_t dr_offset,
+                         uint64_t file_start_offset,
+                         uint64_t desired_length)
+{
+  struct DownloadRequest *dr;
+  unsigned int i;
+  unsigned int head_skip;
+  uint64_t child_block_size;
+
+  dr = GNUNET_new (struct DownloadRequest);
+  dr->parent = parent;
+  dr->depth = depth;
+  dr->offset = dr_offset;
+  dr->chk_idx = chk_idx;
+  if (0 == depth)
+    return dr;
+  child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
+
+  /* calculate how many blocks at this level are not interesting
+   * from the start (rounded down), either because of the requested
+   * file offset or because this IBlock is further along */
+  if (dr_offset < file_start_offset)
+  {
+    head_skip = (file_start_offset - dr_offset) / child_block_size;
+  }
+  else
+  {
+    head_skip = 0;
+  }
+
+  /* calculate index of last block at this level that is interesting (rounded up) */
+  dr->num_children =
+    (file_start_offset + desired_length - dr_offset) / child_block_size;
+  if (dr->num_children * child_block_size <
+      file_start_offset + desired_length - dr_offset)
+    dr->num_children++; /* round up */
+  GNUNET_assert (dr->num_children > head_skip);
+  dr->num_children -= head_skip;
+  if (dr->num_children > CHK_PER_INODE)
+    dr->num_children = CHK_PER_INODE; /* cap at max */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Block at offset %llu and depth %u has %u children\n",
+              (unsigned long long) dr_offset,
+              depth,
+              dr->num_children);
+
+  /* now we can get the total number of *interesting* children for this block */
+
+  /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
+  GNUNET_assert (dr->num_children > 0);
+
+  dr->children = GNUNET_new_array (dr->num_children, struct DownloadRequest *);
+  for (i = 0; i < dr->num_children; i++)
+  {
+    dr->children[i] =
+      create_download_request (dr,
+                               i + head_skip,
+                               depth - 1,
+                               dr_offset + (i + head_skip) * child_block_size,
+                               file_start_offset,
+                               desired_length);
+  }
+  return dr;
+}
+
+
+/**
+ * Continuation after a possible attempt to reconstruct
+ * the current IBlock from the existing file.
+ *
+ * @param cls the 'struct ReconstructContext'
+ */
+static void
+reconstruct_cont (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+
+  /* clean up state from tree encoder */
+  if (NULL != dc->task)
+  {
+    GNUNET_SCHEDULER_cancel (dc->task);
+    dc->task = NULL;
+  }
+  if (NULL != dc->rfh)
+  {
+    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
+    dc->rfh = NULL;
+  }
+  /* start "normal" download */
+  dc->issue_requests = GNUNET_YES;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting normal download\n");
+  schedule_block_download (dc, dc->top_request);
+}
+
+
+/**
+ * Task requesting the next block from the tree encoder.
+ *
+ * @param cls the 'struct GNUJNET_FS_DownloadContext' we're processing
+ */
+static void
+get_next_block (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+
+  dc->task = NULL;
+  GNUNET_FS_tree_encoder_next (dc->te);
+}
+
+
+/**
+ * Function called asking for the current (encoded)
+ * block to be processed.  After processing the
+ * client should either call "GNUNET_FS_tree_encode_next"
+ * or (on error) "GNUNET_FS_tree_encode_finish".
+ *
+ * This function checks if the content on disk matches
+ * the expected content based on the URI.
+ *
+ * @param cls closure
+ * @param chk content hash key for the block
+ * @param offset offset of the block
+ * @param depth depth of the block, 0 for DBLOCK
+ * @param type type of the block (IBLOCK or DBLOCK)
+ * @param block the (encrypted) block
+ * @param block_size size of block (in bytes)
+ */
+static void
+reconstruct_cb (void *cls,
+                const struct ContentHashKey *chk,
+                uint64_t offset,
+                unsigned int depth,
+                enum GNUNET_BLOCK_Type type,
+                const void *block,
+                uint16_t block_size)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  struct DownloadRequest *dr;
+  uint64_t blen;
+  unsigned int chld;
+
+  /* find corresponding request entry */
+  dr = dc->top_request;
+  while (dr->depth > depth)
+  {
+    GNUNET_assert (dr->num_children > 0);
+    blen = GNUNET_FS_tree_compute_tree_size (dr->depth - 1);
+    chld = (offset - dr->offset) / blen;
+    if (chld < dr->children[0]->chk_idx)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Block %u < %u irrelevant for our range\n",
+                  chld,
+                  dr->children[0]->chk_idx);
+      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+      return;     /* irrelevant block */
+    }
+    if (chld > dr->children[dr->num_children - 1]->chk_idx)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Block %u > %u irrelevant for our range\n",
+                  chld,
+                  dr->children[dr->num_children - 1]->chk_idx);
+      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+      return;     /* irrelevant block */
+    }
+    dr = dr->children[chld - dr->children[0]->chk_idx];
+  }
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "Matched TE block with request at offset %llu and depth %u in state %d\n",
+    (unsigned long long) dr->offset,
+    dr->depth,
+    dr->state);
+  /* FIXME: this code needs more testing and might
+     need to handle more states... */
+  switch (dr->state)
+  {
+  case BRS_INIT:
+    break;
+
+  case BRS_RECONSTRUCT_DOWN:
+    break;
+
+  case BRS_RECONSTRUCT_META_UP:
+    break;
+
+  case BRS_RECONSTRUCT_UP:
+    break;
+
+  case BRS_CHK_SET:
+    if (0 == memcmp (chk, &dr->chk, sizeof(struct ContentHashKey)))
+    {
+      GNUNET_log (
+        GNUNET_ERROR_TYPE_DEBUG,
+        "Reconstruction succeeded, can use block at offset %llu, depth %u\n",
+        (unsigned long long) offset,
+        depth);
+      /* block matches, hence tree below matches;
+       * this request is done! */
+      dr->state = BRS_DOWNLOAD_UP;
+      (void) GNUNET_CONTAINER_multihashmap_remove (dc->active,
+                                                   &dr->chk.query,
+                                                   dr);
+      /* calculate how many bytes of payload this block
+       * corresponds to */
+      blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
+      /* how many of those bytes are in the requested range? */
+      blen = GNUNET_MIN (blen, dc->length + dc->offset - dr->offset);
+      /* signal progress */
+      dc->completed += blen;
+      pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
+      pi.value.download.specifics.progress.data = NULL;
+      pi.value.download.specifics.progress.offset = offset;
+      pi.value.download.specifics.progress.data_len = 0;
+      pi.value.download.specifics.progress.depth = 0;
+      pi.value.download.specifics.progress.respect_offered = 0;
+      pi.value.download.specifics.progress.block_download_duration =
+        GNUNET_TIME_UNIT_ZERO;
+      GNUNET_FS_download_make_status_ (&pi, dc);
+      /* FIXME: duplicated code from 'process_result_with_request - refactor */
+      if (dc->completed == dc->length)
+      {
+        /* download completed, signal */
+        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                    "Download completed, truncating file to desired length %llu\n",
+                    (unsigned long long) GNUNET_ntohll (
+                      dc->uri->data.chk.file_length));
+        /* truncate file to size (since we store IBlocks at the end) */
+        if (NULL != dc->filename)
+        {
+          if (0 != truncate (dc->filename,
+                             GNUNET_ntohll (dc->uri->data.chk.file_length)))
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                      "truncate",
+                                      dc->filename);
+        }
+      }
+    }
+    else
+      GNUNET_log (
+        GNUNET_ERROR_TYPE_DEBUG,
+        "Reconstruction failed, need to download block at offset %llu, depth %u\n",
+        (unsigned long long) offset,
+        depth);
+    break;
+
+  case BRS_DOWNLOAD_DOWN:
+    break;
+
+  case BRS_DOWNLOAD_UP:
+    break;
+
+  case BRS_ERROR:
+    break;
+
+  default:
+    GNUNET_assert (0);
+    break;
+  }
+  dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+  if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP))
+    check_completed (dc);
+}
+
+
+/**
+ * Function called by the tree encoder to obtain a block of plaintext
+ * data (for the lowest level of the tree).
+ *
+ * @param cls our 'struct ReconstructContext'
+ * @param offset identifies which block to get
+ * @param max (maximum) number of bytes to get; returning
+ *        fewer will also cause errors
+ * @param buf where to copy the plaintext buffer
+ * @param emsg location to store an error message (on error)
+ * @return number of bytes copied to buf, 0 on error
+ */
+static size_t
+fh_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_DISK_FileHandle *fh = dc->rfh;
+  ssize_t ret;
+
+  if (NULL != emsg)
+    *emsg = NULL;
+  if (offset != GNUNET_DISK_file_seek (fh, offset, GNUNET_DISK_SEEK_SET))
+  {
+    if (NULL != emsg)
+      *emsg = GNUNET_strdup (strerror (errno));
+    return 0;
+  }
+  ret = GNUNET_DISK_file_read (fh, buf, max);
+  if (ret < 0)
+  {
+    if (NULL != emsg)
+      *emsg = GNUNET_strdup (strerror (errno));
+    return 0;
+  }
+  return ret;
+}
+
+
+/**
+ * Task that creates the initial (top-level) download
+ * request for the file.
+ *
+ * @param cls the 'struct GNUNET_FS_DownloadContext'
+ */
+void
+GNUNET_FS_download_start_task_ (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  struct GNUNET_DISK_FileHandle *fh;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start task running...\n");
+  dc->task = NULL;
+  if (0 == dc->length)
+  {
+    /* no bytes required! */
+    if (NULL != dc->filename)
+    {
+      fh = GNUNET_DISK_file_open (dc->filename,
+                                  GNUNET_DISK_OPEN_READWRITE
+                                  | GNUNET_DISK_OPEN_CREATE
+                                  | ((0 ==
+                                      GNUNET_FS_uri_chk_get_file_size (dc->uri))
+                                     ? GNUNET_DISK_OPEN_TRUNCATE
+                                     : 0),
+                                  GNUNET_DISK_PERM_USER_READ
+                                  | GNUNET_DISK_PERM_USER_WRITE
+                                  | GNUNET_DISK_PERM_GROUP_READ
+                                  | GNUNET_DISK_PERM_OTHER_READ);
+      GNUNET_DISK_file_close (fh);
+    }
+    GNUNET_FS_download_sync_ (dc);
+    pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
+    pi.value.download.specifics.start.meta = dc->meta;
+    GNUNET_FS_download_make_status_ (&pi, dc);
+    check_completed (dc);
+    return;
+  }
+  if (NULL != dc->emsg)
+    return;
+  if (NULL == dc->top_request)
+  {
+    dc->top_request = create_download_request (NULL,
+                                               0,
+                                               dc->treedepth - 1,
+                                               0,
+                                               dc->offset,
+                                               dc->length);
+    dc->top_request->state = BRS_CHK_SET;
+    dc->top_request->chk = (dc->uri->type == GNUNET_FS_URI_CHK)
+                           ? dc->uri->data.chk.chk
+                           : dc->uri->data.loc.fi.chk;
+    /* signal start */
+    GNUNET_FS_download_sync_ (dc);
+    if (NULL != dc->search)
+      GNUNET_FS_search_result_sync_ (dc->search);
+    pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
+    pi.value.download.specifics.start.meta = dc->meta;
+    GNUNET_FS_download_make_status_ (&pi, dc);
+  }
+  GNUNET_FS_download_start_downloading_ (dc);
+  /* attempt reconstruction from disk */
+  if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
+    dc->rfh = GNUNET_DISK_file_open (dc->filename,
+                                     GNUNET_DISK_OPEN_READ,
+                                     GNUNET_DISK_PERM_NONE);
+  if (dc->top_request->state == BRS_CHK_SET)
+  {
+    if (NULL != dc->rfh)
+    {
+      /* first, try top-down */
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Trying top-down reconstruction for `%s'\n",
+                  dc->filename);
+      try_top_down_reconstruction (dc, dc->top_request);
+      switch (dc->top_request->state)
+      {
+      case BRS_CHK_SET:
+        break;       /* normal */
+
+      case BRS_DOWNLOAD_DOWN:
+        break;       /* normal, some blocks already down */
+
+      case BRS_DOWNLOAD_UP:
+        /* already done entirely, party! */
+        if (NULL != dc->rfh)
+        {
+          /* avoid hanging on to file handle longer than
+           * necessary */
+          GNUNET_DISK_file_close (dc->rfh);
+          dc->rfh = NULL;
+        }
+        return;
+
+      case BRS_ERROR:
+        GNUNET_asprintf (&dc->emsg, _ ("Invalid URI"));
+        GNUNET_FS_download_sync_ (dc);
+        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
+        pi.value.download.specifics.error.message = dc->emsg;
+        GNUNET_FS_download_make_status_ (&pi, dc);
+        return;
+
+      default:
+        GNUNET_assert (0);
+        break;
+      }
+    }
+  }
+  /* attempt reconstruction from meta data */
+  if ((GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) &&
+      (NULL != dc->meta))
+  {
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_DEBUG,
+      "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
+      (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
+      (unsigned int) GNUNET_FS_meta_data_get_serialized_size (dc->meta));
+    GNUNET_FS_meta_data_iterate (dc->meta, &match_full_data, dc);
+    if (BRS_DOWNLOAD_UP == dc->top_request->state)
+    {
+      if (NULL != dc->rfh)
+      {
+        /* avoid hanging on to file handle longer than
+         * necessary */
+        GNUNET_DISK_file_close (dc->rfh);
+        dc->rfh = NULL;
+      }
+      return;     /* finished, status update was already done for us */
+    }
+  }
+  if (NULL != dc->rfh)
+  {
+    /* finally, actually run bottom-up */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Trying bottom-up reconstruction of file `%s'\n",
+                dc->filename);
+    dc->te =
+      GNUNET_FS_tree_encoder_create (dc->h,
+                                     GNUNET_FS_uri_chk_get_file_size (dc->uri),
+                                     dc,
+                                     &fh_reader,
+                                     &reconstruct_cb,
+                                     NULL,
+                                     &reconstruct_cont);
+    dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
+  }
+  else
+  {
+    /* simple, top-level download */
+    dc->issue_requests = GNUNET_YES;
+    schedule_block_download (dc, dc->top_request);
+  }
+  if (BRS_DOWNLOAD_UP == dc->top_request->state)
+    check_completed (dc);
+}
+
+
+void
+GNUNET_FS_download_signal_suspend_ (void *cls)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (NULL != dc->top)
+    GNUNET_FS_end_top (dc->h, dc->top);
+  while (NULL != dc->child_head)
+    GNUNET_FS_download_signal_suspend_ (dc->child_head);
+  if (NULL != dc->search)
+  {
+    dc->search->download = NULL;
+    dc->search = NULL;
+  }
+  if (NULL != dc->job_queue)
+  {
+    GNUNET_FS_dequeue_ (dc->job_queue);
+    dc->job_queue = NULL;
+  }
+  if (NULL != dc->parent)
+    GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
+                                 dc->parent->child_tail,
+                                 dc);
+  if (NULL != dc->task)
+  {
+    GNUNET_SCHEDULER_cancel (dc->task);
+    dc->task = NULL;
+  }
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+  if (NULL != dc->te)
+  {
+    GNUNET_FS_tree_encoder_finish (dc->te, NULL);
+    dc->te = NULL;
+  }
+  if (NULL != dc->rfh)
+  {
+    GNUNET_DISK_file_close (dc->rfh);
+    dc->rfh = NULL;
+  }
+  GNUNET_FS_free_download_request_ (dc->top_request);
+  if (NULL != dc->active)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (dc->active);
+    dc->active = NULL;
+  }
+  GNUNET_free (dc->filename);
+  GNUNET_FS_meta_data_destroy (dc->meta);
+  GNUNET_FS_uri_destroy (dc->uri);
+  GNUNET_free (dc->temp_filename);
+  GNUNET_free (dc->serialization);
+  GNUNET_assert (NULL == dc->job_queue);
+  GNUNET_free (dc);
+}
+
+
+/**
+ * Helper function to setup the download context.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param uri the URI of the file (determines what to download); CHK or LOC URI
+ * @param meta known metadata for the file (can be NULL)
+ * @param filename where to store the file, maybe NULL (then no file is
+ *        created on disk and data must be grabbed from the callbacks)
+ * @param tempname where to store temporary file data, not used if filename is non-NULL;
+ *        can be NULL (in which case we will pick a name if needed); the temporary file
+ *        may already exist, in which case we will try to use the data that is there and
+ *        if it is not what is desired, will overwrite it
+ * @param offset at what offset should we start the download (typically 0)
+ * @param length how many bytes should be downloaded starting at offset
+ * @param anonymity anonymity level to use for the download
+ * @param options various options
+ * @param cctx initial value for the client context for this download
+ * @return context that can be used to control this download
+ */
+struct GNUNET_FS_DownloadContext *
+create_download_context (struct GNUNET_FS_Handle *h,
+                         const struct GNUNET_FS_Uri *uri,
+                         const struct GNUNET_FS_MetaData *meta,
+                         const char *filename,
+                         const char *tempname,
+                         uint64_t offset,
+                         uint64_t length,
+                         uint32_t anonymity,
+                         enum GNUNET_FS_DownloadOptions options,
+                         void *cctx)
+{
+  struct GNUNET_FS_DownloadContext *dc;
+
+  GNUNET_assert (GNUNET_FS_uri_test_chk (uri) || GNUNET_FS_uri_test_loc (uri));
+  if ((offset + length < offset) ||
+      (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  dc = GNUNET_new (struct GNUNET_FS_DownloadContext);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting download %p, %u bytes at offset %llu\n",
+              dc,
+              (unsigned int) length,
+              (unsigned long long) offset);
+  dc->h = h;
+  dc->uri = GNUNET_FS_uri_dup (uri);
+  dc->meta = GNUNET_FS_meta_data_duplicate (meta);
+  dc->client_info = cctx;
+  dc->start_time = GNUNET_TIME_absolute_get ();
+  if (NULL != filename)
+  {
+    dc->filename = GNUNET_strdup (filename);
+    if (GNUNET_YES == GNUNET_DISK_file_test (filename))
+      GNUNET_break (GNUNET_OK == GNUNET_DISK_file_size (filename,
+                                                        &dc->old_file_size,
+                                                        GNUNET_YES,
+                                                        GNUNET_YES));
+  }
+  if (GNUNET_FS_uri_test_loc (dc->uri))
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target));
+  dc->offset = offset;
+  dc->length = length;
+  dc->anonymity = anonymity;
+  dc->options = options;
+  dc->active =
+    GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE),
+                                          GNUNET_NO);
+  dc->treedepth =
+    GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri));
+  if ((NULL == filename) && (is_recursive_download (dc)))
+  {
+    if (NULL != tempname)
+      dc->temp_filename = GNUNET_strdup (tempname);
+    else
+      dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting download `%s' of %llu bytes with tree depth %u\n",
+              filename,
+              (unsigned long long) length,
+              dc->treedepth);
+  GNUNET_assert (NULL == dc->job_queue);
+  dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc);
+  return dc;
+}
+
+
+struct GNUNET_FS_DownloadContext *
+GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
+                          const struct GNUNET_FS_Uri *uri,
+                          const struct GNUNET_FS_MetaData *meta,
+                          const char *filename,
+                          const char *tempname,
+                          uint64_t offset,
+                          uint64_t length,
+                          uint32_t anonymity,
+                          enum GNUNET_FS_DownloadOptions options,
+                          void *cctx,
+                          struct GNUNET_FS_DownloadContext *parent)
+{
+  struct GNUNET_FS_DownloadContext *dc;
+
+  dc = create_download_context (h,
+                                uri,
+                                meta,
+                                filename,
+                                tempname,
+                                offset,
+                                length,
+                                anonymity,
+                                options,
+                                cctx);
+  if (NULL == dc)
+    return NULL;
+  dc->parent = parent;
+  if (NULL != parent)
+    GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc);
+  else if (0 == (GNUNET_FS_DOWNLOAD_IS_PROBE & options))
+    dc->top =
+      GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc);
+  return dc;
+}
+
+
+struct GNUNET_FS_DownloadContext *
+GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
+                                      struct GNUNET_FS_SearchResult *sr,
+                                      const char *filename,
+                                      const char *tempname,
+                                      uint64_t offset,
+                                      uint64_t length,
+                                      uint32_t anonymity,
+                                      enum GNUNET_FS_DownloadOptions options,
+                                      void *cctx)
+{
+  struct GNUNET_FS_DownloadContext *dc;
+
+  if ((NULL == sr) || (NULL != sr->download))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  dc = create_download_context (h,
+                                sr->uri,
+                                sr->meta,
+                                filename,
+                                tempname,
+                                offset,
+                                length,
+                                anonymity,
+                                options,
+                                cctx);
+  if (NULL == dc)
+    return NULL;
+  dc->search = sr;
+  sr->download = dc;
+  if (NULL != sr->probe_ctx)
+  {
+    GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
+    sr->probe_ctx = NULL;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
+  }
+  return dc;
+}
+
+
+/**
+ * Start the downloading process (by entering the queue).
+ *
+ * @param dc our download context
+ */
+void
+GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
+{
+  if (dc->completed == dc->length)
+    return;
+  if (NULL != dc->mq)
+    return; /* already running */
+  GNUNET_assert (NULL == dc->job_queue);
+  GNUNET_assert (NULL == dc->task);
+  GNUNET_assert (NULL != dc->active);
+  dc->job_queue =
+    GNUNET_FS_queue_ (dc->h,
+                      &activate_fs_download,
+                      &deactivate_fs_download,
+                      dc,
+                      (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
+                      (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
+                      ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
+                      : GNUNET_FS_QUEUE_PRIORITY_PROBE);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Download %p put into queue as job %p\n",
+              dc,
+              dc->job_queue);
+}
+
+
+/**
+ * Suspend a download.
+ *
+ * @param dc handle for the download
+ */
+void
+GNUNET_FS_download_suspend (struct GNUNET_FS_DownloadContext *dc)
+{
+  deactivate_fs_download (dc);
+}
+
+
+/**
+ * Resume a suspended download.
+ *
+ * @param dc handle for the download
+ */
+void
+GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+
+  GNUNET_assert (NULL == dc->task);
+  dc->job_queue =
+    GNUNET_FS_queue_ (dc->h,
+                      &activate_fs_download,
+                      &deactivate_fs_download,
+                      dc,
+                      (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
+                      (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
+                      ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
+                      : GNUNET_FS_QUEUE_PRIORITY_PROBE);
+}
+
+
+/**
+ * Stop a download (aborts if download is incomplete).
+ *
+ * @param dc handle for the download
+ * @param do_delete delete files of incomplete downloads
+ */
+void
+GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  int have_children;
+  int search_was_null;
+
+  if (NULL != dc->top)
+    GNUNET_FS_end_top (dc->h, dc->top);
+  if (NULL != dc->task)
+  {
+    GNUNET_SCHEDULER_cancel (dc->task);
+    dc->task = NULL;
+  }
+  search_was_null = (NULL == dc->search);
+  if (NULL != dc->search)
+  {
+    dc->search->download = NULL;
+    GNUNET_FS_search_result_sync_ (dc->search);
+    dc->search = NULL;
+  }
+  if (NULL != dc->job_queue)
+  {
+    GNUNET_FS_dequeue_ (dc->job_queue);
+    dc->job_queue = NULL;
+  }
+  if (NULL != dc->te)
+  {
+    GNUNET_FS_tree_encoder_finish (dc->te, NULL);
+    dc->te = NULL;
+  }
+  have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
+  while (NULL != dc->child_head)
+    GNUNET_FS_download_stop (dc->child_head, do_delete);
+  if (NULL != dc->parent)
+    GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
+                                 dc->parent->child_tail,
+                                 dc);
+  if (NULL != dc->serialization)
+    GNUNET_FS_remove_sync_file_ (dc->h,
+                                 ((NULL != dc->parent) || (! search_was_null))
+                                 ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
+                                 : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
+                                 dc->serialization);
+  if ((GNUNET_YES == have_children) && (NULL == dc->parent))
+    GNUNET_FS_remove_sync_dir_ (dc->h,
+                                (! search_was_null)
+                                ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
+                                : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
+                                dc->serialization);
+  pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED;
+  GNUNET_FS_download_make_status_ (&pi, dc);
+  GNUNET_FS_free_download_request_ (dc->top_request);
+  dc->top_request = NULL;
+  if (NULL != dc->active)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (dc->active);
+    dc->active = NULL;
+  }
+  if (NULL != dc->filename)
+  {
+    if ((dc->completed != dc->length) && (GNUNET_YES == do_delete))
+    {
+      if ((0 != unlink (dc->filename)) && (ENOENT != errno))
+        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                  "unlink",
+                                  dc->filename);
+    }
+    GNUNET_free (dc->filename);
+  }
+  GNUNET_FS_meta_data_destroy (dc->meta);
+  GNUNET_FS_uri_destroy (dc->uri);
+  if (NULL != dc->temp_filename)
+  {
+    if (0 != unlink (dc->temp_filename))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "unlink",
+                                dc->temp_filename);
+    GNUNET_free (dc->temp_filename);
+  }
+  GNUNET_free (dc->serialization);
+  GNUNET_assert (NULL == dc->job_queue);
+  GNUNET_free (dc);
+}
+
+
+/* end of fs_download.c */
diff --git a/src/service/fs/fs_file_information.c b/src/service/fs/fs_file_information.c
new file mode 100644
index 000000000..f23b9da2a
--- /dev/null
+++ b/src/service/fs/fs_file_information.c
@@ -0,0 +1,407 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_file_information.c
+ * @brief  Manage information for publishing directory hierarchies
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#if HAVE_EXTRACTOR_H
+#include <extractor.h>
+#endif
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+
+
+/**
+ * Obtain the name under which this file information
+ * structure is stored on disk.  Only works for top-level
+ * file information structures.
+ *
+ * @param s structure to get the filename for
+ * @return NULL on error, otherwise filename that
+ *         can be used to read this fi-struct from disk.
+ */
+const char *
+GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s)
+{
+  if (NULL != s->dir)
+    return NULL;
+  return s->serialization;
+}
+
+
+/**
+ * Obtain the filename from the file information structure.
+ *
+ * @param s structure to get the filename for
+ * @return "filename" field of the structure (can be NULL)
+ */
+const char *
+GNUNET_FS_file_information_get_filename (const struct
+                                         GNUNET_FS_FileInformation *s)
+{
+  return s->filename;
+}
+
+
+/**
+ * Set the filename in the file information structure.
+ * If filename was already set, frees it before setting the new one.
+ * Makes a copy of the argument.
+ *
+ * @param s structure to get the filename for
+ * @param filename filename to set
+ */
+void
+GNUNET_FS_file_information_set_filename (struct GNUNET_FS_FileInformation *s,
+                                         const char *filename)
+{
+  GNUNET_free (s->filename);
+  if (filename)
+    s->filename = GNUNET_strdup (filename);
+  else
+    s->filename = NULL;
+}
+
+
+struct GNUNET_FS_FileInformation *
+GNUNET_FS_file_information_create_from_file (
+  struct GNUNET_FS_Handle *h,
+  void *client_info,
+  const char *filename,
+  const struct GNUNET_FS_Uri *keywords,
+  const struct GNUNET_FS_MetaData *meta,
+  int do_index,
+  const struct GNUNET_FS_BlockOptions *bo)
+{
+  struct FileInfo *fi;
+  uint64_t fsize;
+  struct GNUNET_FS_FileInformation *ret;
+  const char *fn;
+  const char *ss;
+
+  /* FIXME: should include_symbolic_links be GNUNET_NO or GNUNET_YES here? */
+  if (GNUNET_OK !=
+      GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES))
+  {
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
+    return NULL;
+  }
+  fi = GNUNET_FS_make_file_reader_context_ (filename);
+  if (NULL == fi)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  ret =
+    GNUNET_FS_file_information_create_from_reader (h,
+                                                   client_info,
+                                                   fsize,
+                                                   &GNUNET_FS_data_reader_file_,
+                                                   fi,
+                                                   keywords,
+                                                   meta,
+                                                   do_index,
+                                                   bo);
+  if (ret == NULL)
+    return NULL;
+  ret->h = h;
+  ret->filename = GNUNET_strdup (filename);
+  fn = filename;
+  while (NULL != (ss = strstr (fn, DIR_SEPARATOR_STR)))
+    fn = ss + 1;
+/* FIXME: If we assume that on other platforms CRT is UTF-8-aware, then
+ * this should be changed to EXTRACTOR_METAFORMAT_UTF8
+ */
+  GNUNET_FS_meta_data_insert (ret->meta,
+                              "<gnunet>",
+                              EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+                              EXTRACTOR_METAFORMAT_C_STRING,
+                              "text/plain",
+                              fn,
+                              strlen (fn) + 1);
+  return ret;
+}
+
+
+struct GNUNET_FS_FileInformation *
+GNUNET_FS_file_information_create_from_data (
+  struct GNUNET_FS_Handle *h,
+  void *client_info,
+  uint64_t length,
+  void *data,
+  const struct GNUNET_FS_Uri *keywords,
+  const struct GNUNET_FS_MetaData *meta,
+  int do_index,
+  const struct GNUNET_FS_BlockOptions *bo)
+{
+  if (GNUNET_YES == do_index)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  return GNUNET_FS_file_information_create_from_reader (h,
+                                                        client_info,
+                                                        length,
+                                                        &
+                                                        GNUNET_FS_data_reader_copy_,
+                                                        data,
+                                                        keywords,
+                                                        meta,
+                                                        do_index,
+                                                        bo);
+}
+
+
+struct GNUNET_FS_FileInformation *
+GNUNET_FS_file_information_create_from_reader (
+  struct GNUNET_FS_Handle *h,
+  void *client_info,
+  uint64_t length,
+  GNUNET_FS_DataReader reader,
+  void *reader_cls,
+  const struct GNUNET_FS_Uri *keywords,
+  const struct GNUNET_FS_MetaData *meta,
+  int do_index,
+  const struct GNUNET_FS_BlockOptions *bo)
+{
+  struct GNUNET_FS_FileInformation *ret;
+
+  if ((GNUNET_YES == do_index) && (reader != &GNUNET_FS_data_reader_file_))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  ret = GNUNET_new (struct GNUNET_FS_FileInformation);
+  ret->h = h;
+  ret->client_info = client_info;
+  ret->meta = GNUNET_FS_meta_data_duplicate (meta);
+  if (ret->meta == NULL)
+    ret->meta = GNUNET_FS_meta_data_create ();
+  ret->keywords = (keywords == NULL) ? NULL : GNUNET_FS_uri_dup (keywords);
+  ret->data.file.reader = reader;
+  ret->data.file.reader_cls = reader_cls;
+  ret->data.file.do_index = do_index;
+  ret->data.file.file_size = length;
+  ret->bo = *bo;
+  return ret;
+}
+
+
+/**
+ * Test if a given entry represents a directory.
+ *
+ * @param ent check if this FI represents a directory
+ * @return #GNUNET_YES if so, #GNUNET_NO if not
+ */
+int
+GNUNET_FS_file_information_is_directory (
+  const struct GNUNET_FS_FileInformation *ent)
+{
+  return ent->is_directory;
+}
+
+
+struct GNUNET_FS_FileInformation *
+GNUNET_FS_file_information_create_empty_directory (
+  struct GNUNET_FS_Handle *h,
+  void *client_info,
+  const struct GNUNET_FS_Uri *keywords,
+  const struct GNUNET_FS_MetaData *meta,
+  const struct GNUNET_FS_BlockOptions *bo,
+  const char *filename)
+{
+  struct GNUNET_FS_FileInformation *ret;
+
+  ret = GNUNET_new (struct GNUNET_FS_FileInformation);
+  ret->h = h;
+  ret->client_info = client_info;
+  ret->meta = GNUNET_FS_meta_data_duplicate (meta);
+  ret->keywords = GNUNET_FS_uri_dup (keywords);
+  ret->bo = *bo;
+  ret->is_directory = GNUNET_YES;
+  if (filename != NULL)
+    ret->filename = GNUNET_strdup (filename);
+  return ret;
+}
+
+
+int
+GNUNET_FS_file_information_add (struct GNUNET_FS_FileInformation *dir,
+                                struct GNUNET_FS_FileInformation *ent)
+{
+  if ((ent->dir != NULL) || (ent->next != NULL) ||
+      (dir->is_directory != GNUNET_YES))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  ent->dir = dir;
+  ent->next = dir->data.dir.entries;
+  dir->data.dir.entries = ent;
+  dir->data.dir.dir_size = 0;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Inspect a file or directory in a publish-structure.  Clients
+ * should never modify publish structures that were passed to
+ * #GNUNET_FS_publish_start already.  When called on a directory,
+ * this function will FIRST call @a proc with information about
+ * the directory itself and then for each of the files in the
+ * directory (but not for files in subdirectories).  When called
+ * on a file, @a proc will be called exactly once (with information
+ * about the specific file).
+ *
+ * @param dir the directory
+ * @param proc function to call on each entry
+ * @param proc_cls closure for @a proc
+ */
+void
+GNUNET_FS_file_information_inspect (struct GNUNET_FS_FileInformation *dir,
+                                    GNUNET_FS_FileInformationProcessor proc,
+                                    void *proc_cls)
+{
+  struct GNUNET_FS_FileInformation *pos;
+  int no;
+
+  no = GNUNET_NO;
+  if (GNUNET_OK !=
+      proc (proc_cls,
+            dir,
+            (dir->is_directory == GNUNET_YES) ? dir->data.dir.dir_size
+            : dir->data.file.file_size,
+            dir->meta,
+            &dir->keywords,
+            &dir->bo,
+            (dir->is_directory == GNUNET_YES) ? &no : &dir->data.file.do_index,
+            &dir->client_info))
+    return;
+  if (dir->is_directory != GNUNET_YES)
+    return;
+  pos = dir->data.dir.entries;
+  while (pos != NULL)
+  {
+    no = GNUNET_NO;
+    if (GNUNET_OK !=
+        proc (proc_cls,
+              pos,
+              (pos->is_directory == GNUNET_YES) ? pos->data.dir.dir_size
+              : pos->data.file.file_size,
+              pos->meta,
+              &pos->keywords,
+              &pos->bo,
+              (pos->is_directory == GNUNET_YES) ? &no
+              : &pos->data.file.do_index,
+              &pos->client_info))
+      break;
+    pos = pos->next;
+  }
+}
+
+
+/**
+ * Destroy publish-structure.  Clients should never destroy publish
+ * structures that were passed to #GNUNET_FS_publish_start already.
+ *
+ * @param fi structure to destroy
+ * @param cleaner function to call on each entry in the structure
+ *        (useful to clean up client_info); can be NULL; return
+ *        values are ignored
+ * @param cleaner_cls closure for @a cleaner
+ */
+void
+GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi,
+                                    GNUNET_FS_FileInformationProcessor cleaner,
+                                    void *cleaner_cls)
+{
+  struct GNUNET_FS_FileInformation *pos;
+  int no;
+
+  no = GNUNET_NO;
+  if (GNUNET_YES == fi->is_directory)
+  {
+    /* clean up directory */
+    while (NULL != (pos = fi->data.dir.entries))
+    {
+      fi->data.dir.entries = pos->next;
+      GNUNET_FS_file_information_destroy (pos, cleaner, cleaner_cls);
+    }
+    /* clean up client-info */
+    if (NULL != cleaner)
+      cleaner (cleaner_cls,
+               fi,
+               fi->data.dir.dir_size,
+               fi->meta,
+               &fi->keywords,
+               &fi->bo,
+               &no,
+               &fi->client_info);
+    GNUNET_free (fi->data.dir.dir_data);
+  }
+  else
+  {
+    /* call clean-up function of the reader */
+    if (NULL != fi->data.file.reader)
+    {
+      (void) fi->data.file.reader (fi->data.file.reader_cls, 0, 0, NULL, NULL);
+      fi->data.file.reader = NULL;
+    }
+    /* clean up client-info */
+    if (NULL != cleaner)
+      cleaner (cleaner_cls,
+               fi,
+               fi->data.file.file_size,
+               fi->meta,
+               &fi->keywords,
+               &fi->bo,
+               &fi->data.file.do_index,
+               &fi->client_info);
+  }
+  GNUNET_free (fi->filename);
+  GNUNET_free (fi->emsg);
+  if (NULL != fi->sks_uri)
+    GNUNET_FS_uri_destroy (fi->sks_uri);
+  if (NULL != fi->chk_uri)
+    GNUNET_FS_uri_destroy (fi->chk_uri);
+  /* clean up serialization */
+  if ((NULL != fi->serialization) && (0 != unlink (fi->serialization)))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                              "unlink",
+                              fi->serialization);
+  if (NULL != fi->keywords)
+    GNUNET_FS_uri_destroy (fi->keywords);
+  if (NULL != fi->meta)
+    GNUNET_FS_meta_data_destroy (fi->meta);
+  GNUNET_free (fi->serialization);
+  if (NULL != fi->te)
+  {
+    GNUNET_FS_tree_encoder_finish (fi->te, NULL);
+    fi->te = NULL;
+  }
+  GNUNET_free (fi);
+}
+
+
+/* end of fs_file_information.c */
diff --git a/src/service/fs/fs_getopt.c b/src/service/fs/fs_getopt.c
new file mode 100644
index 000000000..0135e2e05
--- /dev/null
+++ b/src/service/fs/fs_getopt.c
@@ -0,0 +1,274 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_getopt.c
+ * @brief helper functions for command-line argument processing
+ * @author Igor Wronsky, Christian Grothoff
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+
+/* ******************** command-line option parsing API ******************** */
+
+/**
+ * Command-line option parser function that allows the user
+ * to specify one or more '-k' options with keywords.  Each
+ * specified keyword will be added to the URI.  A pointer to
+ * the URI must be passed as the "scls" argument.
+ *
+ * @param ctx command line processor context
+ * @param scls must be of type "struct GNUNET_FS_Uri **"
+ * @param option name of the option (typically 'k')
+ * @param value command line argument given
+ * @return #GNUNET_OK on success
+ */
+static int
+getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                     void *scls,
+                     const char *option,
+                     const char *value)
+{
+  struct GNUNET_FS_Uri **uri = scls;
+  struct GNUNET_FS_Uri *u = *uri;
+  char *val;
+  size_t slen;
+
+  if (NULL == u)
+  {
+    u = GNUNET_new (struct GNUNET_FS_Uri);
+    *uri = u;
+    u->type = GNUNET_FS_URI_KSK;
+    u->data.ksk.keywordCount = 0;
+    u->data.ksk.keywords = NULL;
+  }
+  else
+  {
+    GNUNET_assert (GNUNET_FS_URI_KSK == u->type);
+  }
+  slen = strlen (value);
+  if (0 == slen)
+    return GNUNET_SYSERR;       /* cannot be empty */
+  if (value[0] == '+')
+  {
+    /* simply preserve the "mandatory" flag */
+    if (slen < 2)
+      return GNUNET_SYSERR;     /* empty keywords not allowed */
+    if ((value[1] == '"') && (slen > 3) && (value[slen - 1] == '"'))
+    {
+      /* remove the quotes, keep the '+' */
+      val = GNUNET_malloc (slen - 1);
+      val[0] = '+';
+      GNUNET_memcpy (&val[1],
+                     &value[2],
+                     slen - 3);
+      val[slen - 2] = '\0';
+    }
+    else
+    {
+      /* no quotes, just keep the '+' */
+      val = GNUNET_strdup (value);
+    }
+  }
+  else
+  {
+    if ((value[0] == '"') && (slen > 2) && (value[slen - 1] == '"'))
+    {
+      /* remove the quotes, add a space */
+      val = GNUNET_malloc (slen);
+      val[0] = ' ';
+      GNUNET_memcpy (&val[1],
+                     &value[1],
+                     slen - 2);
+      val[slen - 1] = '\0';
+    }
+    else
+    {
+      /* add a space to indicate "not mandatory" */
+      val = GNUNET_malloc (slen + 2);
+      strcpy (val, " ");
+      strcat (val, value);
+    }
+  }
+  GNUNET_array_append (u->data.ksk.keywords,
+                       u->data.ksk.keywordCount,
+                       val);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify keywords.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] topKeywords set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_KEYWORDS (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_Uri **topKeywords)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_keywords,
+    .scls = (void *) topKeywords
+  };
+
+  return clo;
+}
+
+
+/**
+ * Command-line option parser function that allows the user to specify
+ * one or more '-m' options with metadata.  Each specified entry of
+ * the form "type=value" will be added to the metadata.  A pointer to
+ * the metadata must be passed as the "scls" argument.
+ *
+ * @param ctx command line processor context
+ * @param scls must be of type "struct GNUNET_MetaData **"
+ * @param option name of the option (typically 'k')
+ * @param value command line argument given
+ * @return #GNUNET_OK on success
+ */
+static int
+getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                     void *scls,
+                     const char *option,
+                     const char *value)
+{
+  struct GNUNET_FS_MetaData **mm = scls;
+
+#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR
+  enum EXTRACTOR_MetaType type;
+  const char *typename;
+  const char *typename_i18n;
+#endif
+  struct GNUNET_FS_MetaData *meta;
+  char *tmp;
+
+  meta = *mm;
+  if (meta == NULL)
+  {
+    meta = GNUNET_FS_meta_data_create ();
+    *mm = meta;
+  }
+
+  /* Use GNUNET_STRINGS_get_utf8_args() in main() to acquire utf-8-encoded
+   * commandline arguments, so that the following line is not needed.
+   */
+  /*tmp = GNUNET_STRINGS_to_utf8 (value, strlen (value), locale_charset ());*/
+  tmp = GNUNET_strdup (value);
+#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR
+  type = EXTRACTOR_metatype_get_max ();
+  while (type > 0)
+  {
+    type--;
+    typename = EXTRACTOR_metatype_to_string (type);
+    typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename);
+    if ((strlen (tmp) >= strlen (typename) + 1) &&
+        (tmp[strlen (typename)] == ':') &&
+        (0 == strncmp (typename, tmp, strlen (typename))))
+    {
+      GNUNET_FS_meta_data_insert (meta, "<gnunet>", type,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain",
+                                  &tmp[strlen (typename) + 1],
+                                  strlen (&tmp[strlen (typename) + 1])
+                                  + 1);
+      GNUNET_free (tmp);
+      tmp = NULL;
+      break;
+    }
+    if ((strlen (tmp) >= strlen (typename_i18n) + 1) &&
+        (tmp[strlen (typename_i18n)] == ':') &&
+        (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n))))
+    {
+      GNUNET_FS_meta_data_insert (meta, "<gnunet>", type,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain",
+                                  &tmp[strlen (typename_i18n) + 1],
+                                  strlen (&tmp
+                                          [strlen (typename_i18n) + 1])
+                                  + 1);
+      GNUNET_free (tmp);
+      tmp = NULL;
+      break;
+    }
+  }
+#endif
+
+  if (NULL != tmp)
+  {
+    GNUNET_FS_meta_data_insert (meta, "<gnunet>",
+                                EXTRACTOR_METATYPE_UNKNOWN,
+                                EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                tmp, strlen (tmp) + 1);
+    GNUNET_free (tmp);
+    printf (_
+            (
+              "Unknown metadata type in metadata option `%s'.  Using metadata type `unknown' instead.\n"),
+            value);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify metadata.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] metadata set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_METADATA (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_MetaData **meta)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_metadata,
+    .scls = (void *) meta
+  };
+
+  return clo;
+}
+
+
+/* end of fs_getopt.c */
diff --git a/src/service/fs/fs_list_indexed.c b/src/service/fs/fs_list_indexed.c
new file mode 100644
index 000000000..78816cad1
--- /dev/null
+++ b/src/service/fs/fs_list_indexed.c
@@ -0,0 +1,219 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_list_indexed.c
+ * @author Christian Grothoff
+ * @brief provide a list of all indexed files
+ */
+
+#include "platform.h"
+#include "gnunet_constants.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_protocols.h"
+#include "fs_api.h"
+
+
+/**
+ * Context for #GNUNET_FS_get_indexed_files().
+ */
+struct GNUNET_FS_GetIndexedContext
+{
+  /**
+   * Connection to the FS service.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Function to call for each indexed file.
+   */
+  GNUNET_FS_IndexedFileProcessor iterator;
+
+  /**
+   * Closure for @e iterator.
+   */
+  void *iterator_cls;
+
+  /**
+   * Continuation to trigger at the end.
+   */
+  GNUNET_SCHEDULER_TaskCallback cont;
+
+  /**
+   * Closure for @e cont.
+   */
+  void *cont_cls;
+};
+
+
+/**
+ * Function called on each response from the FS
+ * service with information about indexed files.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`)
+ * @param msg message with indexing information
+ */
+static void
+handle_index_info_end (void *cls,
+                       const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_GetIndexedContext *gic = cls;
+
+  (void) gic->iterator (gic->iterator_cls,
+                        NULL,
+                        NULL);
+  GNUNET_FS_get_indexed_files_cancel (gic);
+}
+
+
+/**
+ * Check validity of response from the FS
+ * service with information about indexed files.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`)
+ * @param iim message with indexing information
+ */
+static int
+check_index_info (void *cls,
+                  const struct IndexInfoMessage *iim)
+{
+  uint16_t msize = ntohs (iim->header.size) - sizeof(*iim);
+  const char *filename;
+
+  filename = (const char *) &iim[1];
+  if (filename[msize - 1] != '\0')
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called on each response from the FS
+ * service with information about indexed files.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_GetIndexedContext *`)
+ * @param iim message with indexing information
+ */
+static void
+handle_index_info (void *cls,
+                   const struct IndexInfoMessage *iim)
+{
+  struct GNUNET_FS_GetIndexedContext *gic = cls;
+  const char *filename;
+
+  filename = (const char *) &iim[1];
+  if (GNUNET_OK !=
+      gic->iterator (gic->iterator_cls,
+                     filename,
+                     &iim->file_id))
+  {
+    GNUNET_FS_get_indexed_files_cancel (gic);
+    return;
+  }
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_GetIndexedContent *`
+ * @param error error code
+ */
+static void
+mq_error_handler (void *cls,
+                  enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_GetIndexedContext *gic = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Failed to receive response from `%s' service (error code is %d).\n",
+              "fs",
+              error);
+  (void) gic->iterator (gic->iterator_cls,
+                        NULL,
+                        NULL);
+  GNUNET_FS_get_indexed_files_cancel (gic);
+}
+
+
+struct GNUNET_FS_GetIndexedContext *
+GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h,
+                             GNUNET_FS_IndexedFileProcessor iterator,
+                             void *iterator_cls)
+{
+  struct GNUNET_FS_GetIndexedContext *gic
+    = GNUNET_new (struct GNUNET_FS_GetIndexedContext);
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (index_info_end,
+                             GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END,
+                             struct GNUNET_MessageHeader,
+                             gic),
+    GNUNET_MQ_hd_var_size (index_info,
+                           GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY,
+                           struct IndexInfoMessage,
+                           gic),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+
+  gic->mq = GNUNET_CLIENT_connect (h->cfg,
+                                   "fs",
+                                   handlers,
+                                   &mq_error_handler,
+                                   h);
+  if (NULL == gic->mq)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to not connect to `%s' service.\n"),
+                "fs");
+    GNUNET_free (gic);
+    return NULL;
+  }
+  gic->iterator = iterator;
+  gic->iterator_cls = iterator_cls;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET);
+  GNUNET_MQ_send (gic->mq,
+                  env);
+  return gic;
+}
+
+
+/**
+ * Cancel iteration over all indexed files.
+ *
+ * @param gic operation to cancel
+ */
+void
+GNUNET_FS_get_indexed_files_cancel (struct GNUNET_FS_GetIndexedContext *gic)
+{
+  GNUNET_MQ_destroy (gic->mq);
+  GNUNET_free (gic);
+}
+
+
+/* end of fs_list_indexed.c */
diff --git a/src/service/fs/fs_misc.c b/src/service/fs/fs_misc.c
new file mode 100644
index 000000000..a8e23f042
--- /dev/null
+++ b/src/service/fs/fs_misc.c
@@ -0,0 +1,165 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2011, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/fs_misc.c
+ * @brief misc. functions related to file-sharing in general
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+
+
+/**
+ * Suggest a filename based on given metadata.
+ *
+ * @param md given meta data
+ * @return NULL if meta data is useless for suggesting a filename
+ */
+char *
+GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_FS_MetaData
+                                      *md)
+{
+  static const char *mimeMap[][2] = {
+    { "application/bz2", ".bz2" },
+    { "application/gnunet-directory", ".gnd" },
+    { "application/java", ".class" },
+    { "application/msword", ".doc" },
+    { "application/nar", ".nar" },
+    { "application/narinfo", ".narinfo" },
+    { "application/ogg", ".ogg" },
+    { "application/pdf", ".pdf" },
+    { "application/pgp-keys", ".key" },
+    { "application/pgp-signature", ".pgp" },
+    { "application/postscript", ".ps" },
+    { "application/rar", ".rar" },
+    { "application/rtf", ".rtf" },
+    { "application/xml", ".xml" },
+    { "application/x-debian-package", ".deb" },
+    { "application/x-dvi", ".dvi" },
+    { "application/x-flac", ".flac" },
+    { "application/x-gzip", ".gz" },
+    { "application/x-java-archive", ".jar" },
+    { "application/x-java-vm", ".class" },
+    { "application/x-python-code", ".pyc" },
+    { "application/x-redhat-package-manager", ".rpm" },
+    { "application/x-rpm", ".rpm" },
+    { "application/x-tar", ".tar" },
+    { "application/x-tex-pk", ".pk" },
+    { "application/x-texinfo", ".texinfo" },
+    { "application/x-xcf", ".xcf" },
+    { "application/x-xfig", ".xfig" },
+    { "application/zip", ".zip" },
+
+    { "audio/midi", ".midi" },
+    { "audio/mpeg", ".mp3" },
+    { "audio/real", ".rm" },
+    { "audio/x-wav", ".wav" },
+
+    { "image/gif", ".gif" },
+    { "image/jpeg", ".jpg" },
+    { "image/pcx", ".pcx" },
+    { "image/png", ".png" },
+    { "image/tiff", ".tiff" },
+    { "image/x-ms-bmp", ".bmp" },
+    { "image/x-xpixmap", ".xpm" },
+
+    { "text/css", ".css" },
+    { "text/html", ".html" },
+    { "text/plain", ".txt" },
+    { "text/rtf", ".rtf" },
+    { "text/x-c++hdr", ".h++" },
+    { "text/x-c++src", ".c++" },
+    { "text/x-chdr", ".h" },
+    { "text/x-csrc", ".c" },
+    { "text/x-java", ".java" },
+    { "text/x-moc", ".moc" },
+    { "text/x-pascal", ".pas" },
+    { "text/x-perl", ".pl" },
+    { "text/x-python", ".py" },
+    { "text/x-tex", ".tex" },
+
+    { "video/avi", ".avi" },
+    { "video/mpeg", ".mpeg" },
+    { "video/quicktime", ".qt" },
+    { "video/real", ".rm" },
+    { "video/x-msvideo", ".avi" },
+    { NULL, NULL },
+  };
+  char *ret;
+  unsigned int i;
+  char *mime;
+  char *base;
+  const char *ext;
+
+  ret =
+    GNUNET_FS_meta_data_get_by_type (md,
+                                     EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
+  if (ret != NULL)
+    return ret;
+  ext = NULL;
+  mime =
+    GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
+  if (mime != NULL)
+  {
+    i = 0;
+    while ((mimeMap[i][0] != NULL) && (0 != strcmp (mime, mimeMap[i][0])))
+      i++;
+    if (mimeMap[i][1] == NULL)
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
+                  _ ("Did not find mime type `%s' in extension list.\n"), mime);
+    else
+      ext = mimeMap[i][1];
+    GNUNET_free (mime);
+  }
+  base =
+    GNUNET_FS_meta_data_get_first_by_types (md,
+                                            EXTRACTOR_METATYPE_TITLE,
+                                            EXTRACTOR_METATYPE_BOOK_TITLE,
+                                            EXTRACTOR_METATYPE_ORIGINAL_TITLE,
+                                            EXTRACTOR_METATYPE_PACKAGE_NAME,
+                                            EXTRACTOR_METATYPE_URL,
+                                            EXTRACTOR_METATYPE_URI,
+                                            EXTRACTOR_METATYPE_DESCRIPTION,
+                                            EXTRACTOR_METATYPE_ISRC,
+                                            EXTRACTOR_METATYPE_JOURNAL_NAME,
+                                            EXTRACTOR_METATYPE_AUTHOR_NAME,
+                                            EXTRACTOR_METATYPE_SUBJECT,
+                                            EXTRACTOR_METATYPE_ALBUM,
+                                            EXTRACTOR_METATYPE_ARTIST,
+                                            EXTRACTOR_METATYPE_KEYWORDS,
+                                            EXTRACTOR_METATYPE_COMMENT,
+                                            EXTRACTOR_METATYPE_UNKNOWN,
+                                            -1);
+  if ((base == NULL) && (ext == NULL))
+    return NULL;
+  if (base == NULL)
+    return GNUNET_strdup (ext);
+  if (ext == NULL)
+    return base;
+  GNUNET_asprintf (&ret, "%s%s", base, ext);
+  GNUNET_free (base);
+  return ret;
+}
+
+
+/* end of fs_misc.c */
diff --git a/src/service/fs/fs_namespace.c b/src/service/fs/fs_namespace.c
new file mode 100644
index 000000000..f8b7b91c0
--- /dev/null
+++ b/src/service/fs/fs_namespace.c
@@ -0,0 +1,804 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2003-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_namespace.c
+ * @brief publishing to namespaces, and tracking updateable entries
+ *        for our namespaces
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_publish_ublock.h"
+
+
+/**
+ * Information about an (updateable) node in the
+ * namespace.
+ */
+struct NamespaceUpdateNode
+{
+  /**
+   * Identifier for this node.
+   */
+  char *id;
+
+  /**
+   * Identifier of children of this node.
+   */
+  char *update;
+
+  /**
+   * Metadata for this entry.
+   */
+  struct GNUNET_FS_MetaData *md;
+
+  /**
+   * URI of this entry in the namespace.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Namespace update generation ID.  Used to ensure
+   * freshness of the tree_id.
+   */
+  unsigned int nug;
+
+  /**
+   * TREE this entry belongs to (if nug is current).
+   */
+  unsigned int tree_id;
+};
+
+
+/**
+ * Handle to update information for a namespace.
+ */
+struct GNUNET_FS_UpdateInformationGraph
+{
+  /**
+   * Handle to the FS service context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Array with information about nodes in the namespace.
+   */
+  struct NamespaceUpdateNode **update_nodes;
+
+  /**
+   * Private key for the namespace.
+   */
+  struct GNUNET_CRYPTO_EcdsaPrivateKey ns;
+
+  /**
+   * Hash map mapping identifiers of update nodes
+   * to the update nodes (initialized on-demand).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *update_map;
+
+  /**
+   * Size of the update nodes array.
+   */
+  unsigned int update_node_count;
+
+  /**
+   * Reference counter.
+   */
+  unsigned int rc;
+
+  /**
+   * Generator for unique nug numbers.
+   */
+  unsigned int nug_gen;
+};
+
+
+/**
+ * Return the name of the directory in which we store
+ * the update information graph for the given local namespace.
+ *
+ * @param h file-sharing handle
+ * @param ns namespace handle
+ * @return NULL on error, otherwise the name of the directory
+ */
+static char *
+get_update_information_directory (
+  struct GNUNET_FS_Handle *h,
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns)
+{
+  char *dn;
+  char *ret;
+  struct GNUNET_CRYPTO_EcdsaPublicKey pub;
+  struct GNUNET_HashCode hc;
+  struct GNUNET_CRYPTO_HashAsciiEncoded enc;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (h->cfg, "FS", "UPDATE_DIR", &dn))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "fs", "UPDATE_DIR");
+    return NULL;
+  }
+  GNUNET_CRYPTO_ecdsa_key_get_public (ns, &pub);
+  GNUNET_CRYPTO_hash (&pub, sizeof(pub), &hc);
+  GNUNET_CRYPTO_hash_to_enc (&hc, &enc);
+  GNUNET_asprintf (&ret,
+                   "%s%s%s",
+                   dn,
+                   DIR_SEPARATOR_STR,
+                   (const char *) enc.encoding);
+  GNUNET_free (dn);
+  return ret;
+}
+
+
+/**
+ * Release memory occupied by UIG datastructure.
+ *
+ * @param uig data structure to free
+ */
+static void
+free_update_information_graph (struct GNUNET_FS_UpdateInformationGraph *uig)
+{
+  unsigned int i;
+  struct NamespaceUpdateNode *nsn;
+
+  for (i = 0; i < uig->update_node_count; i++)
+  {
+    nsn = uig->update_nodes[i];
+    GNUNET_FS_meta_data_destroy (nsn->md);
+    GNUNET_FS_uri_destroy (nsn->uri);
+    GNUNET_free (nsn->id);
+    GNUNET_free (nsn->update);
+    GNUNET_free (nsn);
+  }
+  GNUNET_array_grow (uig->update_nodes, uig->update_node_count, 0);
+  if (NULL != uig->update_map)
+    GNUNET_CONTAINER_multihashmap_destroy (uig->update_map);
+  GNUNET_free (uig);
+}
+
+
+/**
+ * Write a namespace's update node graph to a file.
+ *
+ * @param uig update information graph to dump
+ */
+static void
+write_update_information_graph (struct GNUNET_FS_UpdateInformationGraph *uig)
+{
+  char *fn;
+  struct GNUNET_BIO_WriteHandle *wh;
+  unsigned int i;
+  struct NamespaceUpdateNode *n;
+  char *uris;
+
+  fn = get_update_information_directory (uig->h, &uig->ns);
+  wh = GNUNET_BIO_write_open_file (fn);
+  if (NULL == wh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to open `%s' for writing: %s\n"),
+                fn,
+                strerror (errno));
+    GNUNET_free (fn);
+    return;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write_int32 (wh,
+                                           "fs-namespace-node-count",
+                                           uig->update_node_count))
+    goto END;
+  for (i = 0; i < uig->update_node_count; i++)
+  {
+    n = uig->update_nodes[i];
+    uris = GNUNET_FS_uri_to_string (n->uri);
+    struct GNUNET_BIO_WriteSpec ws[] = {
+      GNUNET_BIO_write_spec_string ("fs-namespace-node-id", n->id),
+      GNUNET_FS_write_spec_meta_data ("fs-namespace-node-meta", n->md),
+      GNUNET_BIO_write_spec_string ("fs-namespace-node-update", n->update),
+      GNUNET_BIO_write_spec_string ("fs-namespace-uris", uris),
+      GNUNET_BIO_write_spec_end (),
+    };
+    if (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws))
+    {
+      GNUNET_free (uris);
+      break;
+    }
+    GNUNET_free (uris);
+  }
+  END:
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to write `%s': %s\n"),
+                fn,
+                strerror (errno));
+  GNUNET_free (fn);
+}
+
+
+/**
+ * Read the namespace update node graph from a file.
+ *
+ * @param h FS handle to use
+ * @param ns namespace to read
+ * @return update graph, never NULL
+ */
+static struct GNUNET_FS_UpdateInformationGraph *
+read_update_information_graph (struct GNUNET_FS_Handle *h,
+                               const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns)
+{
+  struct GNUNET_FS_UpdateInformationGraph *uig;
+  char *fn;
+  struct GNUNET_BIO_ReadHandle *rh;
+  unsigned int i;
+  struct NamespaceUpdateNode *n;
+  char *uris;
+  uint32_t count;
+  char *emsg;
+
+  uig = GNUNET_new (struct GNUNET_FS_UpdateInformationGraph);
+  uig->h = h;
+  uig->ns = *ns;
+  fn = get_update_information_directory (h, ns);
+  if (GNUNET_YES != GNUNET_DISK_file_test (fn))
+  {
+    GNUNET_free (fn);
+    return uig;
+  }
+  rh = GNUNET_BIO_read_open_file (fn);
+  if (NULL == rh)
+  {
+    GNUNET_free (fn);
+    return uig;
+  }
+  if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "fs-namespace-count",
+                                          (int32_t *) &count))
+  {
+    GNUNET_break (0);
+    goto END;
+  }
+  if (count > 1024 * 1024)
+  {
+    GNUNET_break (0);
+    goto END;
+  }
+  if (0 == count)
+    goto END;
+  uig->update_nodes =
+    GNUNET_malloc (count * sizeof(struct NamespaceUpdateNode *));
+
+  for (i = 0; i < count; i++)
+  {
+    n = GNUNET_new (struct NamespaceUpdateNode);
+    struct GNUNET_BIO_ReadSpec rs[] = {
+      GNUNET_BIO_read_spec_string ("identifier", &n->id, 1024),
+      GNUNET_FS_read_spec_meta_data ("meta", &n->md),
+      GNUNET_BIO_read_spec_string ("update-id", &n->update, 1024),
+      GNUNET_BIO_read_spec_string ("uri", &uris, 1024 * 2),
+      GNUNET_BIO_read_spec_end (),
+    };
+    if (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs))
+    {
+      GNUNET_break (0);
+      GNUNET_free (n->id);
+      GNUNET_free (n->update);
+      if (n->md != NULL)
+        GNUNET_FS_meta_data_destroy (n->md);
+      GNUNET_free (n);
+      break;
+    }
+    n->uri = GNUNET_FS_uri_parse (uris, &emsg);
+    GNUNET_free (uris);
+    if (n->uri == NULL)
+    {
+      GNUNET_break (0);
+      GNUNET_free (emsg);
+      GNUNET_free (n->id);
+      GNUNET_free (n->update);
+      GNUNET_FS_meta_data_destroy (n->md);
+      GNUNET_free (n);
+      break;
+    }
+    uig->update_nodes[i] = n;
+  }
+  uig->update_node_count = i;
+  END:
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to read `%s': %s\n"),
+                fn,
+                emsg);
+    GNUNET_free (emsg);
+  }
+  GNUNET_free (fn);
+  return uig;
+}
+
+
+/**
+ * Context for the SKS publication.
+ */
+struct GNUNET_FS_PublishSksContext
+{
+  /**
+   * URI of the new entry in the namespace.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Namespace update node to add to namespace on success (or to be
+   * deleted if publishing failed).
+   */
+  struct NamespaceUpdateNode *nsn;
+
+  /**
+   * Namespace we're publishing to.
+   */
+  struct GNUNET_CRYPTO_EcdsaPrivateKey ns;
+
+  /**
+   * Handle to the datastore.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * Handle to FS.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Function to call once we're done.
+   */
+  GNUNET_FS_PublishContinuation cont;
+
+  /**
+   * Closure for cont.
+   */
+  void *cont_cls;
+
+  /**
+   * Handle for our UBlock operation request.
+   */
+  struct GNUNET_FS_PublishUblockContext *uc;
+};
+
+
+/**
+ * Function called by the UBlock construction with
+ * the result from the PUT (UBlock) request.
+ *
+ * @param cls closure of type "struct GNUNET_FS_PublishSksContext*"
+ * @param msg error message (or NULL)
+ */
+static void
+sks_publish_cont (void *cls, const char *msg)
+{
+  struct GNUNET_FS_PublishSksContext *psc = cls;
+  struct GNUNET_FS_UpdateInformationGraph *uig;
+
+  psc->uc = NULL;
+  if (NULL != msg)
+  {
+    if (NULL != psc->cont)
+      psc->cont (psc->cont_cls, NULL, msg);
+    GNUNET_FS_publish_sks_cancel (psc);
+    return;
+  }
+  if (NULL != psc->nsn)
+  {
+    /* FIXME: this can be done much more
+     * efficiently by simply appending to the
+     * file and overwriting the 4-byte header */
+    uig = read_update_information_graph (psc->h, &psc->ns);
+    GNUNET_array_append (uig->update_nodes, uig->update_node_count, psc->nsn);
+    psc->nsn = NULL;
+    write_update_information_graph (uig);
+    free_update_information_graph (uig);
+  }
+  if (NULL != psc->cont)
+    psc->cont (psc->cont_cls, psc->uri, NULL);
+  GNUNET_FS_publish_sks_cancel (psc);
+}
+
+
+struct GNUNET_FS_PublishSksContext *
+GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
+                       const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
+                       const char *identifier,
+                       const char *update,
+                       const struct GNUNET_FS_MetaData *meta,
+                       const struct GNUNET_FS_Uri *uri,
+                       const struct GNUNET_FS_BlockOptions *bo,
+                       enum GNUNET_FS_PublishOptions options,
+                       GNUNET_FS_PublishContinuation cont,
+                       void *cont_cls)
+{
+  struct GNUNET_FS_PublishSksContext *psc;
+  struct GNUNET_FS_Uri *sks_uri;
+
+  sks_uri = GNUNET_new (struct GNUNET_FS_Uri);
+  sks_uri->type = GNUNET_FS_URI_SKS;
+  sks_uri->data.sks.identifier = GNUNET_strdup (identifier);
+  GNUNET_CRYPTO_ecdsa_key_get_public (ns, &sks_uri->data.sks.ns);
+
+  psc = GNUNET_new (struct GNUNET_FS_PublishSksContext);
+  psc->h = h;
+  psc->uri = sks_uri;
+  psc->cont = cont;
+  psc->cont_cls = cont_cls;
+  psc->ns = *ns;
+  if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
+  {
+    psc->dsh = GNUNET_DATASTORE_connect (h->cfg);
+    if (NULL == psc->dsh)
+    {
+      sks_publish_cont (psc, _ ("Failed to connect to datastore."));
+      return NULL;
+    }
+  }
+  if (NULL != update)
+  {
+    psc->nsn = GNUNET_new (struct NamespaceUpdateNode);
+    psc->nsn->id = GNUNET_strdup (identifier);
+    psc->nsn->update = GNUNET_strdup (update);
+    psc->nsn->md = GNUNET_FS_meta_data_duplicate (meta);
+    psc->nsn->uri = GNUNET_FS_uri_dup (uri);
+  }
+  psc->uc = GNUNET_FS_publish_ublock_ (h,
+                                       psc->dsh,
+                                       identifier,
+                                       update,
+                                       ns,
+                                       meta,
+                                       uri,
+                                       bo,
+                                       options,
+                                       &sks_publish_cont,
+                                       psc);
+  return psc;
+}
+
+
+/**
+ * Abort the SKS publishing operation.
+ *
+ * @param psc context of the operation to abort.
+ */
+void
+GNUNET_FS_publish_sks_cancel (struct GNUNET_FS_PublishSksContext *psc)
+{
+  if (NULL != psc->uc)
+  {
+    GNUNET_FS_publish_ublock_cancel_ (psc->uc);
+    psc->uc = NULL;
+  }
+  if (NULL != psc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO);
+    psc->dsh = NULL;
+  }
+  GNUNET_FS_uri_destroy (psc->uri);
+  if (NULL != psc->nsn)
+  {
+    GNUNET_FS_meta_data_destroy (psc->nsn->md);
+    GNUNET_FS_uri_destroy (psc->nsn->uri);
+    GNUNET_free (psc->nsn->id);
+    GNUNET_free (psc->nsn->update);
+    GNUNET_free (psc->nsn);
+  }
+  GNUNET_free (psc);
+}
+
+
+/**
+ * Closure for 'process_update_node'.
+ */
+struct ProcessUpdateClosure
+{
+  /**
+   * Function to call for each node.
+   */
+  GNUNET_FS_IdentifierProcessor ip;
+
+  /**
+   * Closure for 'ip'.
+   */
+  void *ip_cls;
+};
+
+
+/**
+ * Call the iterator in the closure for each node.
+ *
+ * @param cls closure (of type 'struct ProcessUpdateClosure *')
+ * @param key current key code
+ * @param value value in the hash map (of type 'struct NamespaceUpdateNode *')
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int
+process_update_node (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct ProcessUpdateClosure *pc = cls;
+  struct NamespaceUpdateNode *nsn = value;
+
+  pc->ip (pc->ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Closure for 'find_trees'.
+ */
+struct FindTreeClosure
+{
+  /**
+   * UIG we are operating on.
+   */
+  struct GNUNET_FS_UpdateInformationGraph *uig;
+
+  /**
+   * Array with 'head's of TREEs.
+   */
+  struct NamespaceUpdateNode **tree_array;
+
+  /**
+   * Size of 'tree_array'
+   */
+  unsigned int tree_array_size;
+
+  /**
+   * Current generational ID used.
+   */
+  unsigned int nug;
+
+  /**
+   * Identifier for the current TREE, or UINT_MAX for none yet.
+   */
+  unsigned int id;
+};
+
+
+/**
+ * Find all nodes reachable from the current node (including the
+ * current node itself).  If they are in no tree, add them to the
+ * current one.   If they are the head of another tree, merge the
+ * trees.  If they are in the middle of another tree, let them be.
+ * We can tell that a node is already in an tree by checking if
+ * its 'nug' field is set to the current 'nug' value.  It is the
+ * head of an tree if it is in the 'tree_array' under its respective
+ * 'tree_id'.
+ *
+ * In short, we're trying to find the smallest number of tree to
+ * cover a directed graph.
+ *
+ * @param cls closure (of type 'struct FindTreeClosure')
+ * @param key current key code
+ * @param value value in the hash map
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int
+find_trees (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct FindTreeClosure *fc = cls;
+  struct NamespaceUpdateNode *nsn = value;
+  struct GNUNET_HashCode hc;
+
+  if (nsn->nug == fc->nug)
+  {
+    if (UINT_MAX == nsn->tree_id)
+      return GNUNET_YES;   /* circular */
+    GNUNET_assert (nsn->tree_id < fc->tree_array_size);
+    if (fc->tree_array[nsn->tree_id] != nsn)
+      return GNUNET_YES;   /* part of "another" (directed) TREE,
+                            * and not root of it, end trace */
+    if (nsn->tree_id == fc->id)
+      return GNUNET_YES;   /* that's our own root (can this be?) */
+    /* merge existing TREE, we have a root for both */
+    fc->tree_array[nsn->tree_id] = NULL;
+    if (UINT_MAX == fc->id)
+      fc->id = nsn->tree_id;   /* take over ID */
+  }
+  else
+  {
+    nsn->nug = fc->nug;
+    nsn->tree_id = UINT_MAX;   /* mark as undef */
+    /* trace */
+    GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc);
+    GNUNET_CONTAINER_multihashmap_get_multiple (fc->uig->update_map,
+                                                &hc,
+                                                &find_trees,
+                                                fc);
+  }
+  return GNUNET_YES;
+}
+
+
+/**
+ * List all of the identifiers in the namespace for which we could
+ * produce an update.  Namespace updates form a graph where each node
+ * has a name.  Each node can have any number of URI/meta-data entries
+ * which can each be linked to other nodes.  Cycles are possible.
+ *
+ * Calling this function with "next_id" NULL will cause the library to
+ * call "ip" with a root for each strongly connected component of the
+ * graph (a root being a node from which all other nodes in the Tree
+ * are reachable).
+ *
+ * Calling this function with "next_id" being the name of a node will
+ * cause the library to call "ip" with all children of the node.  Note
+ * that cycles within the final tree are possible (including self-loops).
+ * I know, odd definition of a tree, but the GUI will display an actual
+ * tree (GtkTreeView), so that's what counts for the term here.
+ *
+ * @param h fs handle to use
+ * @param ns namespace to inspect for updateable content
+ * @param next_id ID to look for; use NULL to look for tree roots
+ * @param ip function to call on each updateable identifier
+ * @param ip_cls closure for ip
+ */
+void
+GNUNET_FS_namespace_list_updateable (
+  struct GNUNET_FS_Handle *h,
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
+  const char *next_id,
+  GNUNET_FS_IdentifierProcessor ip,
+  void *ip_cls)
+{
+  unsigned int i;
+  unsigned int nug;
+  struct GNUNET_HashCode hc;
+  struct NamespaceUpdateNode *nsn;
+  struct ProcessUpdateClosure pc;
+  struct FindTreeClosure fc;
+  struct GNUNET_FS_UpdateInformationGraph *uig;
+
+  uig = read_update_information_graph (h, ns);
+  if (NULL == uig->update_nodes)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "No updateable nodes found for ID `%s'\n",
+                next_id);
+    free_update_information_graph (uig);
+    return;   /* no nodes */
+  }
+  uig->update_map =
+    GNUNET_CONTAINER_multihashmap_create (2 + 3 * uig->update_node_count / 4,
+                                          GNUNET_NO);
+  for (i = 0; i < uig->update_node_count; i++)
+  {
+    nsn = uig->update_nodes[i];
+    GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc);
+    GNUNET_CONTAINER_multihashmap_put (
+      uig->update_map,
+      &hc,
+      nsn,
+      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  }
+  if (NULL != next_id)
+  {
+    GNUNET_CRYPTO_hash (next_id, strlen (next_id), &hc);
+    pc.ip = ip;
+    pc.ip_cls = ip_cls;
+    GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map,
+                                                &hc,
+                                                &process_update_node,
+                                                &pc);
+    free_update_information_graph (uig);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Calculating TREEs to find roots of update trees\n");
+  /* Find heads of TREEs in update graph */
+  nug = ++uig->nug_gen;
+  fc.tree_array = NULL;
+  fc.tree_array_size = 0;
+
+  for (i = 0; i < uig->update_node_count; i++)
+  {
+    nsn = uig->update_nodes[i];
+    if (nsn->nug == nug)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "TREE of node `%s' is %u\n",
+                  nsn->id,
+                  nsn->nug);
+      continue;     /* already placed in TREE */
+    }
+    GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc);
+    nsn->nug = nug;
+    nsn->tree_id = UINT_MAX;
+    fc.id = UINT_MAX;
+    fc.nug = nug;
+    fc.uig = uig;
+    GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map,
+                                                &hc,
+                                                &find_trees,
+                                                &fc);
+    if (UINT_MAX == fc.id)
+    {
+      /* start new TREE */
+      for (fc.id = 0; fc.id < fc.tree_array_size; fc.id++)
+      {
+        if (NULL == fc.tree_array[fc.id])
+        {
+          fc.tree_array[fc.id] = nsn;
+          nsn->tree_id = fc.id;
+          break;
+        }
+      }
+      if (fc.id == fc.tree_array_size)
+      {
+        GNUNET_array_append (fc.tree_array, fc.tree_array_size, nsn);
+        nsn->tree_id = fc.id;
+      }
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Starting new TREE %u with node `%s'\n",
+                  nsn->tree_id,
+                  nsn->id);
+      /* put all nodes with same identifier into this TREE */
+      GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc);
+      fc.id = nsn->tree_id;
+      fc.nug = nug;
+      fc.uig = uig;
+      GNUNET_CONTAINER_multihashmap_get_multiple (uig->update_map,
+                                                  &hc,
+                                                  &find_trees,
+                                                  &fc);
+    }
+    else
+    {
+      /* make head of TREE "id" */
+      fc.tree_array[fc.id] = nsn;
+      nsn->tree_id = fc.id;
+    }
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "TREE of node `%s' is %u\n",
+                nsn->id,
+                fc.id);
+  }
+  for (i = 0; i < fc.tree_array_size; i++)
+  {
+    nsn = fc.tree_array[i];
+    if (NULL != nsn)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Root of TREE %u is node `%s'\n",
+                  i,
+                  nsn->id);
+      ip (ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update);
+    }
+  }
+  GNUNET_array_grow (fc.tree_array, fc.tree_array_size, 0);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Done processing TREEs\n");
+  free_update_information_graph (uig);
+}
+
+
+/* end of fs_namespace.c */
diff --git a/src/service/fs/fs_publish.c b/src/service/fs/fs_publish.c
new file mode 100644
index 000000000..d1662c78b
--- /dev/null
+++ b/src/service/fs/fs_publish.c
@@ -0,0 +1,1625 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/fs_publish.c
+ * @brief publish a file or directory in GNUnet
+ * @see https://gnunet.org/encoding
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+
+
+/**
+ * Fill in all of the generic fields for
+ * a publish event and call the callback.
+ *
+ * @param pi structure to fill in
+ * @param pc overall publishing context
+ * @param p file information for the file being published
+ * @param offset where in the file are we so far
+ * @return value returned from callback
+ */
+void *
+GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_PublishContext *pc,
+                                const struct GNUNET_FS_FileInformation *p,
+                                uint64_t offset)
+{
+  pi->value.publish.pc = pc;
+  pi->value.publish.fi = p;
+  pi->value.publish.cctx = p->client_info;
+  pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info;
+  pi->value.publish.filename = p->filename;
+  pi->value.publish.size =
+    (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size :
+    p->data.file.file_size;
+  pi->value.publish.eta =
+    GNUNET_TIME_calculate_eta (p->start_time, offset,
+                               pi->value.publish.size);
+  pi->value.publish.completed = offset;
+  pi->value.publish.duration =
+    GNUNET_TIME_absolute_get_duration (p->start_time);
+  pi->value.publish.anonymity = p->bo.anonymity_level;
+  pi->fsh = pc->h;
+  return pc->h->upcb (pc->h->upcb_cls, pi);
+}
+
+
+/**
+ * Cleanup the publish context, we're done with it.
+ *
+ * @param pc struct to clean up
+ */
+static void
+publish_cleanup (struct GNUNET_FS_PublishContext *pc)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Cleaning up publish context (done!)\n");
+  if (NULL != pc->fhc)
+  {
+    GNUNET_CRYPTO_hash_file_cancel (pc->fhc);
+    pc->fhc = NULL;
+  }
+  GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL);
+  GNUNET_free (pc->nid);
+  GNUNET_free (pc->nuid);
+  GNUNET_free (pc->serialization);
+  if (NULL != pc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
+    pc->dsh = NULL;
+  }
+  if (NULL != pc->mq)
+  {
+    GNUNET_MQ_destroy (pc->mq);
+    pc->mq = NULL;
+  }
+  GNUNET_assert (NULL == pc->upload_task);
+  GNUNET_free (pc);
+}
+
+
+/**
+ * Function called by the datastore API with
+ * the result from the PUT request.
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext *`
+ * @param success #GNUNET_OK on success
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg error message (or NULL)
+ */
+static void
+ds_put_cont (void *cls,
+             int success,
+             struct GNUNET_TIME_Absolute min_expiration,
+             const char *msg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pc->qre = NULL;
+  if (GNUNET_SYSERR == success)
+  {
+    GNUNET_asprintf (&pc->fi_pos->emsg,
+                     _ ("Publishing failed: %s"),
+                     msg);
+    pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
+    pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+    pi.value.publish.specifics.error.message = pc->fi_pos->emsg;
+    pc->fi_pos->client_info =
+      GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0);
+    if ((GNUNET_YES != pc->fi_pos->is_directory) &&
+        (NULL != pc->fi_pos->filename) &&
+        (GNUNET_YES == pc->any_done) &&
+        (GNUNET_YES == pc->fi_pos->data.file.do_index))
+    {
+      /* run unindex to clean up */
+      GNUNET_FS_unindex_start (pc->h,
+                               pc->fi_pos->filename,
+                               NULL);
+    }
+    return;
+  }
+  pc->any_done = GNUNET_YES;
+  GNUNET_assert (NULL == pc->upload_task);
+  pc->upload_task =
+    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                        &GNUNET_FS_publish_main_, pc);
+}
+
+
+/**
+ * Generate the callback that signals clients
+ * that a file (or directory) has been completely
+ * published.
+ *
+ * @param p the completed upload
+ * @param pc context of the publication
+ */
+static void
+signal_publish_completion (struct GNUNET_FS_FileInformation *p,
+                           struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED;
+  pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO;
+  pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
+  pi.value.publish.specifics.completed.sks_uri = p->sks_uri;
+  p->client_info =
+    GNUNET_FS_publish_make_status_ (&pi, pc, p,
+                                    p->data.file.file_size);
+}
+
+
+/**
+ * Generate the callback that signals clients
+ * that a file (or directory) has encountered
+ * a problem during publication.
+ *
+ * @param p the upload that had trouble
+ * @param pc context of the publication
+ * @param emsg error message
+ */
+static void
+signal_publish_error (struct GNUNET_FS_FileInformation *p,
+                      struct GNUNET_FS_PublishContext *pc,
+                      const char *emsg)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  p->emsg = GNUNET_strdup (emsg);
+  pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
+  pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+  pi.value.publish.specifics.error.message = emsg;
+  p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
+  if ((p->is_directory != GNUNET_YES) &&
+      (NULL != p->filename) &&
+      (GNUNET_YES == pc->any_done) &&
+      (p->data.file.do_index == GNUNET_YES))
+  {
+    /* run unindex to clean up */
+    GNUNET_FS_unindex_start (pc->h,
+                             p->filename,
+                             NULL);
+  }
+}
+
+
+/**
+ * Datastore returns from reservation cancel request.
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext *`
+ * @param success success code (not used)
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg error message (typically NULL, not used)
+ */
+static void
+finish_release_reserve (void *cls, int success,
+                        struct GNUNET_TIME_Absolute min_expiration,
+                        const char *msg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  pc->qre = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Releasing reserve done!\n");
+  signal_publish_completion (pc->fi, pc);
+  pc->all_done = GNUNET_YES;
+  GNUNET_FS_publish_sync_ (pc);
+}
+
+
+/**
+ * We've finished publishing the SBlock as part of a larger upload.
+ * Check the result and complete the larger upload.
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext *` of the larger upload
+ * @param uri URI of the published SBlock
+ * @param emsg NULL on success, otherwise error message
+ */
+static void
+publish_sblocks_cont (void *cls,
+                      const struct GNUNET_FS_Uri *uri,
+                      const char *emsg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  pc->sks_pc = NULL;
+  if (NULL != emsg)
+  {
+    signal_publish_error (pc->fi, pc, emsg);
+    GNUNET_FS_publish_sync_ (pc);
+    return;
+  }
+  if (NULL != uri)
+  {
+    /* sks publication, remember namespace URI */
+    pc->fi->sks_uri = GNUNET_FS_uri_dup (uri);
+  }
+  GNUNET_assert (pc->qre == NULL);
+  if ((pc->dsh != NULL) && (pc->rid != 0))
+  {
+    pc->qre =
+      GNUNET_DATASTORE_release_reserve (pc->dsh, pc->rid, UINT_MAX, UINT_MAX,
+                                        &finish_release_reserve, pc);
+  }
+  else
+  {
+    finish_release_reserve (pc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
+  }
+}
+
+
+/**
+ * We are almost done publishing the structure,
+ * add SBlocks (if needed).
+ *
+ * @param pc overall upload data
+ */
+static void
+publish_sblock (struct GNUNET_FS_PublishContext *pc)
+{
+  if (NULL != pc->ns)
+    pc->sks_pc = GNUNET_FS_publish_sks (pc->h,
+                                        pc->ns,
+                                        pc->nid,
+                                        pc->nuid,
+                                        pc->fi->meta,
+                                        pc->fi->chk_uri,
+                                        &pc->fi->bo,
+                                        pc->options,
+                                        &publish_sblocks_cont, pc);
+  else
+    publish_sblocks_cont (pc, NULL, NULL);
+}
+
+
+/**
+ * We've finished publishing a KBlock as part of a larger upload.
+ * Check the result and continue the larger upload.
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext *`
+ *        of the larger upload
+ * @param uri URI of the published blocks
+ * @param emsg NULL on success, otherwise error message
+ */
+static void
+publish_kblocks_cont (void *cls,
+                      const struct GNUNET_FS_Uri *uri,
+                      const char *emsg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p = pc->fi_pos;
+
+  pc->ksk_pc = NULL;
+  if (NULL != emsg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Error uploading KSK blocks: %s\n",
+                emsg);
+    signal_publish_error (p, pc, emsg);
+    GNUNET_FS_file_information_sync_ (p);
+    GNUNET_FS_publish_sync_ (pc);
+    GNUNET_assert (NULL == pc->upload_task);
+    pc->upload_task =
+      GNUNET_SCHEDULER_add_with_priority
+        (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+        &GNUNET_FS_publish_main_,
+        pc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "KSK blocks published, moving on to next file\n");
+  if (NULL != p->dir)
+    signal_publish_completion (p, pc);
+  /* move on to next file */
+  if (NULL != p->next)
+    pc->fi_pos = p->next;
+  else
+    pc->fi_pos = p->dir;
+  GNUNET_FS_publish_sync_ (pc);
+  GNUNET_assert (NULL == pc->upload_task);
+  pc->upload_task =
+    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                        &GNUNET_FS_publish_main_, pc);
+}
+
+
+/**
+ * Function called by the tree encoder to obtain
+ * a block of plaintext data (for the lowest level
+ * of the tree).
+ *
+ * @param cls our publishing context
+ * @param offset identifies which block to get
+ * @param max (maximum) number of bytes to get; returning
+ *        fewer will also cause errors
+ * @param buf where to copy the plaintext buffer
+ * @param emsg location to store an error message (on error)
+ * @return number of bytes copied to buf, 0 on error
+ */
+static size_t
+block_reader (void *cls,
+              uint64_t offset,
+              size_t max,
+              void *buf,
+              char **emsg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+  const char *dd;
+  size_t pt_size;
+
+  p = pc->fi_pos;
+  if (GNUNET_YES == p->is_directory)
+  {
+    pt_size = GNUNET_MIN (max, p->data.dir.dir_size - offset);
+    dd = p->data.dir.dir_data;
+    GNUNET_memcpy (buf, &dd[offset], pt_size);
+  }
+  else
+  {
+    if (UINT64_MAX == offset)
+    {
+      if (&GNUNET_FS_data_reader_file_ == p->data.file.reader)
+      {
+        /* force closing the file to avoid keeping too many files open */
+        p->data.file.reader (p->data.file.reader_cls, offset, 0, NULL, NULL);
+      }
+      return 0;
+    }
+    pt_size = GNUNET_MIN (max, p->data.file.file_size - offset);
+    if (0 == pt_size)
+      return 0;                 /* calling reader with pt_size==0
+                                 * might free buf, so don't! */
+    if (pt_size !=
+        p->data.file.reader (p->data.file.reader_cls, offset, pt_size, buf,
+                             emsg))
+      return 0;
+  }
+  return pt_size;
+}
+
+
+/**
+ * The tree encoder has finished processing a
+ * file.   Call it's finish method and deal with
+ * the final result.
+ *
+ * @param cls our publishing context
+ */
+static void
+encode_cont (void *cls)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+  struct GNUNET_FS_ProgressInfo pi;
+  char *emsg;
+  uint64_t flen;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished with tree encoder\n");
+  p = pc->fi_pos;
+  p->chk_uri = GNUNET_FS_tree_encoder_get_uri (p->te);
+  GNUNET_FS_file_information_sync_ (p);
+  GNUNET_FS_tree_encoder_finish (p->te, &emsg);
+  p->te = NULL;
+  if (NULL != emsg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Error during tree walk: %s\n",
+                emsg);
+    GNUNET_asprintf (&p->emsg,
+                     _ ("Publishing failed: %s"),
+                     emsg);
+    GNUNET_free (emsg);
+    pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
+    pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+    pi.value.publish.specifics.error.message = p->emsg;
+    p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
+  }
+  else
+  {
+    /* final progress event */
+    GNUNET_assert (NULL != p->chk_uri);
+    flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri);
+    pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
+    pi.value.publish.specifics.progress.data = NULL;
+    pi.value.publish.specifics.progress.offset = flen;
+    pi.value.publish.specifics.progress.data_len = 0;
+    pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen);
+    p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen);
+  }
+  /* continue with main */  /* continue with main */
+  GNUNET_assert (NULL == pc->upload_task);
+  pc->upload_task =
+    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                        &GNUNET_FS_publish_main_, pc);
+}
+
+
+/**
+ * Function called asking for the current (encoded)
+ * block to be processed.  After processing the
+ * client should either call #GNUNET_FS_tree_encoder_next
+ * or (on error) #GNUNET_FS_tree_encoder_finish.
+ *
+ * @param cls closure
+ * @param chk content hash key for the block
+ * @param offset offset of the block in the file
+ * @param depth depth of the block in the file, 0 for DBLOCK
+ * @param type type of the block (IBLOCK or DBLOCK)
+ * @param block the (encrypted) block
+ * @param block_size size of @a block (in bytes)
+ */
+static void
+block_proc (void *cls,
+            const struct ContentHashKey *chk,
+            uint64_t offset,
+            unsigned int depth,
+            enum GNUNET_BLOCK_Type type,
+            const void *block,
+            uint16_t block_size)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+  struct OnDemandBlock odb;
+
+  p = pc->fi_pos;
+  if (NULL == pc->dsh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Waiting for datastore connection\n");
+    GNUNET_assert (NULL == pc->upload_task);
+    pc->upload_task =
+      GNUNET_SCHEDULER_add_with_priority
+        (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc);
+    return;
+  }
+
+  if ((GNUNET_YES != p->is_directory) &&
+      (GNUNET_YES == p->data.file.do_index) &&
+      (GNUNET_BLOCK_TYPE_FS_DBLOCK == type))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Indexing block `%s' for offset %llu with index size %u\n",
+                GNUNET_h2s (&chk->query),
+                (unsigned long long) offset,
+                (unsigned int) sizeof(struct OnDemandBlock));
+    odb.offset = GNUNET_htonll (offset);
+    odb.file_id = p->data.file.file_id;
+    GNUNET_assert (pc->qre == NULL);
+    pc->qre =
+      GNUNET_DATASTORE_put (pc->dsh,
+                            (p->is_directory == GNUNET_YES) ? 0 : pc->rid,
+                            &chk->query,
+                            sizeof(struct OnDemandBlock),
+                            &odb,
+                            GNUNET_BLOCK_TYPE_FS_ONDEMAND,
+                            p->bo.content_priority,
+                            p->bo.anonymity_level,
+                            p->bo.replication_level,
+                            p->bo.expiration_time,
+                            -2, 1,
+                            &ds_put_cont, pc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Publishing block `%s' for offset %llu with size %u\n",
+              GNUNET_h2s (&chk->query),
+              (unsigned long long) offset,
+              (unsigned int) block_size);
+  GNUNET_assert (pc->qre == NULL);
+  pc->qre =
+    GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 :
+                          pc->rid,
+                          &chk->query,
+                          block_size,
+                          block,
+                          type,
+                          p->bo.content_priority,
+                          p->bo.anonymity_level,
+                          p->bo.replication_level,
+                          p->bo.expiration_time,
+                          -2, 1,
+                          &ds_put_cont,
+                          pc);
+}
+
+
+/**
+ * Function called with information about our
+ * progress in computing the tree encoding.
+ *
+ * @param cls closure
+ * @param offset where are we in the file
+ * @param pt_block plaintext of the currently processed block
+ * @param pt_size size of @a pt_block
+ * @param depth depth of the block in the tree, 0 for DBLOCK
+ */
+static void
+progress_proc (void *cls, uint64_t offset,
+               const void *pt_block,
+               size_t pt_size,
+               unsigned int depth)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+  struct GNUNET_FS_FileInformation *par;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  p = pc->fi_pos;
+  pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS;
+  pi.value.publish.specifics.progress.data = pt_block;
+  pi.value.publish.specifics.progress.offset = offset;
+  pi.value.publish.specifics.progress.data_len = pt_size;
+  pi.value.publish.specifics.progress.depth = depth;
+  p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset);
+  if ((0 != depth) ||
+      (GNUNET_YES == p->is_directory))
+    return;
+  while (NULL != (par = p->dir))
+  {
+    p = par;
+    GNUNET_assert (GNUNET_YES == par->is_directory);
+    p->data.dir.contents_completed += pt_size;
+    pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY;
+    pi.value.publish.specifics.progress_directory.completed =
+      p->data.dir.contents_completed;
+    pi.value.publish.specifics.progress_directory.total =
+      p->data.dir.contents_size;
+    pi.value.publish.specifics.progress_directory.eta =
+      GNUNET_TIME_calculate_eta (p->start_time,
+                                 p
+                                 ->data.dir.contents_completed,
+                                 p
+                                 ->data.dir.contents_size);
+    p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
+  }
+}
+
+
+/**
+ * We are uploading a file or directory; load (if necessary) the next
+ * block into memory, encrypt it and send it to the FS service.  Then
+ * continue with the main task.
+ *
+ * @param pc overall upload data
+ */
+static void
+publish_content (struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_FS_FileInformation *p;
+  char *emsg;
+  struct GNUNET_FS_DirectoryBuilder *db;
+  struct GNUNET_FS_FileInformation *dirpos;
+  void *raw_data;
+  uint64_t size;
+
+  p = pc->fi_pos;
+  GNUNET_assert (NULL != p);
+  if (NULL == p->te)
+  {
+    if (GNUNET_YES == p->is_directory)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n");
+      db = GNUNET_FS_directory_builder_create (p->meta);
+      dirpos = p->data.dir.entries;
+      while (NULL != dirpos)
+      {
+        if (GNUNET_YES == dirpos->is_directory)
+        {
+          raw_data = dirpos->data.dir.dir_data;
+          dirpos->data.dir.dir_data = NULL;
+        }
+        else
+        {
+          raw_data = NULL;
+          if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
+              (dirpos->data.file.file_size > 0))
+          {
+            raw_data = GNUNET_malloc (dirpos->data.file.file_size);
+            emsg = NULL;
+            if (dirpos->data.file.file_size !=
+                dirpos->data.file.reader (dirpos->data.file.reader_cls, 0,
+                                          dirpos->data.file.file_size, raw_data,
+                                          &emsg))
+            {
+              GNUNET_free (emsg);
+              GNUNET_free (raw_data);
+              raw_data = NULL;
+            }
+            dirpos->data.file.reader (dirpos->data.file.reader_cls, UINT64_MAX,
+                                      0, 0, NULL);
+          }
+        }
+        GNUNET_FS_directory_builder_add (db, dirpos->chk_uri, dirpos->meta,
+                                         raw_data);
+        GNUNET_free (raw_data);
+        dirpos = dirpos->next;
+      }
+      GNUNET_free (p->data.dir.dir_data);
+      p->data.dir.dir_data = NULL;
+      p->data.dir.dir_size = 0;
+      GNUNET_FS_directory_builder_finish (db, &p->data.dir.dir_size,
+                                          &p->data.dir.dir_data);
+      GNUNET_FS_file_information_sync_ (p);
+    }
+    size = (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size :
+           p->data.file.file_size;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Creating tree encoder\n");
+    p->te =
+      GNUNET_FS_tree_encoder_create (pc->h, size, pc, &block_reader,
+                                     &block_proc, &progress_proc,
+                                     &encode_cont);
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Processing next block from tree\n");
+  GNUNET_FS_tree_encoder_next (p->te);
+}
+
+
+/**
+ * Check the response from the "fs" service to our 'start index'
+ * request.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param msg the response we got
+ */
+static int
+check_index_start_failed (void *cls,
+                          const struct GNUNET_MessageHeader *msg)
+{
+  size_t msize = ntohs (msg->size) - sizeof(*msg);
+  const char *emsg = (const char *) &msg[1];
+
+  if (emsg[msize - 1] != '\0')
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process the response from the "fs" service to our 'start index'
+ * request.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param msg the response we got
+ */
+static void
+handle_index_start_failed (void *cls,
+                           const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+  const char *emsg = (const char *) &msg[1];
+  char *msgtxt;
+
+  GNUNET_MQ_destroy (pc->mq);
+  pc->mq = NULL;
+  p = pc->fi_pos;
+  GNUNET_asprintf (&msgtxt,
+                   _ ("Can not index file `%s': %s.\n"),
+                   p->filename,
+                   gettext (emsg));
+  signal_publish_error (p,
+                        pc,
+                        msgtxt);
+  GNUNET_free (msgtxt);
+  GNUNET_FS_file_information_sync_ (p);
+  GNUNET_FS_publish_sync_ (pc);
+}
+
+
+/**
+ * Process the response from the "fs" service to our 'start index'
+ * request.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param msg the response we got
+ */
+static void
+handle_index_start_ok (void *cls,
+                       const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+
+  GNUNET_MQ_destroy (pc->mq);
+  pc->mq = NULL;
+  p = pc->fi_pos;
+  p->data.file.index_start_confirmed = GNUNET_YES;
+  GNUNET_FS_file_information_sync_ (p);
+  publish_content (pc);
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_PublishContext *`
+ * @param error error code
+ */
+static void
+index_mq_error_handler (void *cls,
+                        enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+
+  if (NULL != pc->mq)
+  {
+    GNUNET_MQ_destroy (pc->mq);
+    pc->mq = NULL;
+  }
+  p = pc->fi_pos;
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              _ ("Can not index file `%s': %s.  Will try to insert instead.\n"),
+              p->filename,
+              _ ("error on index-start request to `fs' service"));
+  p->data.file.do_index = GNUNET_NO;
+  GNUNET_FS_file_information_sync_ (p);
+  publish_content (pc);
+}
+
+
+/**
+ * Function called once the hash computation over an
+ * indexed file has completed.
+ *
+ * @param cls closure, our publishing context
+ * @param res resulting hash, NULL on error
+ */
+static void
+hash_for_index_cb (void *cls,
+                   const struct GNUNET_HashCode *res)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (index_start_ok,
+                             GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK,
+                             struct GNUNET_MessageHeader,
+                             pc),
+    GNUNET_MQ_hd_var_size (index_start_failed,
+                           GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED,
+                           struct GNUNET_MessageHeader,
+                           pc),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_FS_FileInformation *p;
+  struct GNUNET_MQ_Envelope *env;
+  struct IndexStartMessage *ism;
+  size_t slen;
+  uint64_t dev;
+  uint64_t ino;
+  char *fn;
+
+  pc->fhc = NULL;
+  p = pc->fi_pos;
+  if (NULL == res)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Can not index file `%s': %s.  Will try to insert instead.\n"),
+                p->filename,
+                _ ("failed to compute hash"));
+    p->data.file.do_index = GNUNET_NO;
+    GNUNET_FS_file_information_sync_ (p);
+    publish_content (pc);
+    return;
+  }
+  if (GNUNET_YES == p->data.file.index_start_confirmed)
+  {
+    publish_content (pc);
+    return;
+  }
+  fn = GNUNET_STRINGS_filename_expand (p->filename);
+  GNUNET_assert (fn != NULL);
+  slen = strlen (fn) + 1;
+  if (slen >=
+      GNUNET_MAX_MESSAGE_SIZE - sizeof(struct IndexStartMessage))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _
+                  ("Can not index file `%s': %s.  Will try to insert instead.\n"),
+                fn, _ ("filename too long"));
+    GNUNET_free (fn);
+    p->data.file.do_index = GNUNET_NO;
+    GNUNET_FS_file_information_sync_ (p);
+    publish_content (pc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hash of indexed file `%s' is `%s'\n",
+              p->filename,
+              GNUNET_h2s (res));
+  if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
+  {
+    p->data.file.file_id = *res;
+    p->data.file.have_hash = GNUNET_YES;
+    p->data.file.index_start_confirmed = GNUNET_YES;
+    GNUNET_FS_file_information_sync_ (p);
+    publish_content (pc);
+    GNUNET_free (fn);
+    return;
+  }
+  pc->mq = GNUNET_CLIENT_connect (pc->h->cfg,
+                                  "fs",
+                                  handlers,
+                                  &index_mq_error_handler,
+                                  pc);
+  if (NULL == pc->mq)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Can not index file `%s': %s.  Will try to insert instead.\n"),
+                p->filename,
+                _ ("could not connect to `fs' service"));
+    p->data.file.do_index = GNUNET_NO;
+    publish_content (pc);
+    GNUNET_free (fn);
+    return;
+  }
+  if (p->data.file.have_hash != GNUNET_YES)
+  {
+    p->data.file.file_id = *res;
+    p->data.file.have_hash = GNUNET_YES;
+    GNUNET_FS_file_information_sync_ (p);
+  }
+  env = GNUNET_MQ_msg_extra (ism,
+                             slen,
+                             GNUNET_MESSAGE_TYPE_FS_INDEX_START);
+  if (GNUNET_OK ==
+      GNUNET_DISK_file_get_identifiers (p->filename,
+                                        &dev,
+                                        &ino))
+  {
+    ism->device = GNUNET_htonll (dev);
+    ism->inode = GNUNET_htonll (ino);
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                _ ("Failed to get file identifiers for `%s'\n"),
+                p->filename);
+  }
+  ism->file_id = *res;
+  GNUNET_memcpy (&ism[1],
+                 fn,
+                 slen);
+  GNUNET_free (fn);
+  GNUNET_MQ_send (pc->mq,
+                  env);
+}
+
+
+/**
+ * We've computed the CHK/LOC URI, now publish the KSKs (if applicable).
+ *
+ * @param pc publishing context to do this for
+ */
+static void
+publish_kblocks (struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_FS_FileInformation *p;
+
+  p = pc->fi_pos;
+  /* upload of "p" complete, publish KBlocks! */
+  if (NULL != p->keywords)
+  {
+    pc->ksk_pc = GNUNET_FS_publish_ksk (pc->h,
+                                        p->keywords,
+                                        p->meta,
+                                        p->chk_uri,
+                                        &p->bo,
+                                        pc->options,
+                                        &publish_kblocks_cont,
+                                        pc);
+  }
+  else
+  {
+    publish_kblocks_cont (pc, p->chk_uri, NULL);
+  }
+}
+
+
+/**
+ * Process the response from the "fs" service to our LOC sign request.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param sig the response we got
+ */
+static void
+handle_signature_response (void *cls,
+                           const struct ResponseLocSignatureMessage *sig)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_FileInformation *p;
+
+  p = pc->fi_pos;
+  p->chk_uri->type = GNUNET_FS_URI_LOC;
+  /* p->data.loc.fi kept from CHK before */
+  p->chk_uri->data.loc.peer = sig->peer;
+  p->chk_uri->data.loc.expirationTime
+    = GNUNET_TIME_absolute_ntoh (sig->expiration_time);
+  p->chk_uri->data.loc.contentSignature = sig->signature;
+  GNUNET_FS_file_information_sync_ (p);
+  GNUNET_FS_publish_sync_ (pc);
+  publish_kblocks (pc);
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_PublishContext *`
+ * @param error error code
+ */
+static void
+loc_mq_error_handler (void *cls,
+                      enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  if (NULL != pc->mq)
+  {
+    GNUNET_MQ_destroy (pc->mq);
+    pc->mq = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              _ ("Can not create LOC URI. Will continue with CHK instead.\n"));
+  publish_kblocks (pc);
+}
+
+
+/**
+ * We're publishing without anonymity. Contact the FS service
+ * to create a signed LOC URI for further processing, then
+ * continue with KSKs.
+ *
+ * @param pc the publishing context do to this for
+ */
+static void
+create_loc_uri (struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (signature_response,
+                             GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE,
+                             struct ResponseLocSignatureMessage,
+                             pc),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MQ_Envelope *env;
+  struct RequestLocSignatureMessage *req;
+  struct GNUNET_FS_FileInformation *p;
+
+  if (NULL != pc->mq)
+    GNUNET_MQ_destroy (pc->mq);
+  pc->mq = GNUNET_CLIENT_connect (pc->h->cfg,
+                                  "fs",
+                                  handlers,
+                                  &loc_mq_error_handler,
+                                  pc);
+  if (NULL == pc->mq)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Can not create LOC URI. Will continue with CHK instead.\n"));
+    publish_kblocks (pc);
+    return;
+  }
+  p = pc->fi_pos;
+  env = GNUNET_MQ_msg (req,
+                       GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN);
+  req->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT);
+  req->expiration_time = GNUNET_TIME_absolute_hton (p->bo.expiration_time);
+  req->chk = p->chk_uri->data.chk.chk;
+  req->file_length = GNUNET_htonll (p->chk_uri->data.chk.file_length);
+  GNUNET_MQ_send (pc->mq,
+                  env);
+}
+
+
+/**
+ * Main function that performs the upload.
+ *
+ * @param cls `struct GNUNET_FS_PublishContext *` identifies the upload
+ */
+void
+GNUNET_FS_publish_main_ (void *cls)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  struct GNUNET_FS_FileInformation *p;
+  char *fn;
+
+  pc->upload_task = NULL;
+  p = pc->fi_pos;
+  if (NULL == p)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publishing complete, now publishing SKS and KSK blocks.\n");
+    /* upload of entire hierarchy complete,
+     * publish namespace entries */
+    GNUNET_FS_publish_sync_ (pc);
+    publish_sblock (pc);
+    return;
+  }
+  /* find starting position */
+  while ((GNUNET_YES == p->is_directory) &&
+         (NULL != p->data.dir.entries) &&
+         (NULL == p->emsg) &&
+         (NULL == p->data.dir.entries->chk_uri))
+  {
+    p = p->data.dir.entries;
+    pc->fi_pos = p;
+    GNUNET_FS_publish_sync_ (pc);
+  }
+  /* abort on error */
+  if (NULL != p->emsg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Error uploading: %s\n",
+                p->emsg);
+    /* error with current file, abort all
+     * related files as well! */
+    while (NULL != p->dir)
+    {
+      fn = GNUNET_FS_meta_data_get_by_type (p->meta,
+                                            EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
+      p = p->dir;
+      if (fn != NULL)
+      {
+        GNUNET_asprintf (&p->emsg,
+                         _ ("Recursive upload failed at `%s': %s"),
+                         fn,
+                         p->emsg);
+        GNUNET_free (fn);
+      }
+      else
+      {
+        GNUNET_asprintf (&p->emsg,
+                         _ ("Recursive upload failed: %s"),
+                         p->emsg);
+      }
+      pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR;
+      pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+      pi.value.publish.specifics.error.message = p->emsg;
+      p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0);
+    }
+    pc->all_done = GNUNET_YES;
+    GNUNET_FS_publish_sync_ (pc);
+    return;
+  }
+  /* handle completion */
+  if (NULL != p->chk_uri)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "File upload complete, now publishing KSK blocks.\n");
+    GNUNET_FS_publish_sync_ (pc);
+
+    if ((0 == p->bo.anonymity_level) &&
+        (GNUNET_YES !=
+         GNUNET_FS_uri_test_loc (p->chk_uri)))
+    {
+      /* zero anonymity, box CHK URI in LOC URI */
+      create_loc_uri (pc);
+    }
+    else
+    {
+      publish_kblocks (pc);
+    }
+    return;
+  }
+  if ((GNUNET_YES != p->is_directory) && (p->data.file.do_index))
+  {
+    if (NULL == p->filename)
+    {
+      p->data.file.do_index = GNUNET_NO;
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _ (
+                    "Can not index file `%s': %s.  Will try to insert instead.\n"),
+                  "<no-name>",
+                  _ ("needs to be an actual file"));
+      GNUNET_FS_file_information_sync_ (p);
+      publish_content (pc);
+      return;
+    }
+    if (p->data.file.have_hash)
+    {
+      hash_for_index_cb (pc, &p->data.file.file_id);
+    }
+    else
+    {
+      p->start_time = GNUNET_TIME_absolute_get ();
+      pc->fhc =
+        GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, p->filename,
+                                 HASHING_BLOCKSIZE, &hash_for_index_cb, pc);
+    }
+    return;
+  }
+  publish_content (pc);
+}
+
+
+/**
+ * Signal the FS's progress function that we are starting
+ * an upload.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param fi the entry in the publish-structure
+ * @param length length of the file or directory
+ * @param meta metadata for the file or directory (can be modified)
+ * @param uri pointer to the keywords that will be used for this entry (can be modified)
+ * @param bo block options
+ * @param do_index should we index?
+ * @param client_info pointer to client context set upon creation (can be modified)
+ * @return #GNUNET_OK to continue (always)
+ */
+static int
+fip_signal_start (void *cls,
+                  struct GNUNET_FS_FileInformation *fi,
+                  uint64_t length,
+                  struct GNUNET_FS_MetaData *meta,
+                  struct GNUNET_FS_Uri **uri,
+                  struct GNUNET_FS_BlockOptions *bo,
+                  int *do_index,
+                  void **client_info)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  unsigned int kc;
+  uint64_t left;
+
+  if (GNUNET_YES == pc->skip_next_fi_callback)
+  {
+    pc->skip_next_fi_callback = GNUNET_NO;
+    return GNUNET_OK;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting publish operation\n");
+  if (*do_index)
+  {
+    /* space for on-demand blocks */
+    pc->reserve_space +=
+      ((length + DBLOCK_SIZE
+        - 1) / DBLOCK_SIZE) * sizeof(struct OnDemandBlock);
+  }
+  else
+  {
+    /* space for DBlocks */
+    pc->reserve_space += length;
+  }
+  /* entries for IBlocks and DBlocks, space for IBlocks */
+  left = length;
+  while (1)
+  {
+    left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
+    pc->reserve_entries += left;
+    if (left <= 1)
+      break;
+    left = left * sizeof(struct ContentHashKey);
+    pc->reserve_space += left;
+  }
+  pc->reserve_entries++;
+  /* entries and space for keywords */
+  if (NULL != *uri)
+  {
+    kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
+    pc->reserve_entries += kc;
+    pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc;
+  }
+  pi.status = GNUNET_FS_STATUS_PUBLISH_START;
+  *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
+  GNUNET_FS_file_information_sync_ (fi);
+  if ((fi->is_directory) && (fi->dir != NULL))
+  {
+    /* We are a directory, and we are not top-level; process entries in directory */
+    pc->skip_next_fi_callback = GNUNET_YES;
+    GNUNET_FS_file_information_inspect (fi, &fip_signal_start, pc);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Actually signal the FS's progress function that we are suspending
+ * an upload.
+ *
+ * @param fi the entry in the publish-structure
+ * @param pc the publish context of which a file is being suspended
+ */
+static void
+suspend_operation (struct GNUNET_FS_FileInformation *fi,
+                   struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t off;
+
+  if (NULL != pc->ksk_pc)
+  {
+    GNUNET_FS_publish_ksk_cancel (pc->ksk_pc);
+    pc->ksk_pc = NULL;
+  }
+  if (NULL != pc->sks_pc)
+  {
+    GNUNET_FS_publish_sks_cancel (pc->sks_pc);
+    pc->sks_pc = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Suspending publish operation\n");
+  GNUNET_free (fi->serialization);
+  fi->serialization = NULL;
+  off = (NULL == fi->chk_uri) ? 0 : (GNUNET_YES == fi->is_directory) ?
+        fi->data.dir.dir_size : fi->data.file.file_size;
+  pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND;
+  GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
+  if (NULL != pc->qre)
+  {
+    GNUNET_DATASTORE_cancel (pc->qre);
+    pc->qre = NULL;
+  }
+  if (NULL != pc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO);
+    pc->dsh = NULL;
+  }
+  pc->rid = 0;
+}
+
+
+/**
+ * Signal the FS's progress function that we are suspending
+ * an upload.  Performs the recursion.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param fi the entry in the publish-structure
+ * @param length length of the file or directory
+ * @param meta metadata for the file or directory (can be modified)
+ * @param uri pointer to the keywords that will be used for this entry (can be modified)
+ * @param bo block options
+ * @param do_index should we index?
+ * @param client_info pointer to client context set upon creation (can be modified)
+ * @return #GNUNET_OK to continue (always)
+ */
+static int
+fip_signal_suspend (void *cls,
+                    struct GNUNET_FS_FileInformation *fi,
+                    uint64_t length,
+                    struct GNUNET_FS_MetaData *meta,
+                    struct GNUNET_FS_Uri **uri,
+                    struct GNUNET_FS_BlockOptions *bo,
+                    int *do_index,
+                    void **client_info)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  if (GNUNET_YES == pc->skip_next_fi_callback)
+  {
+    pc->skip_next_fi_callback = GNUNET_NO;
+    return GNUNET_OK;
+  }
+  if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta))
+  {
+    /* process entries in directory */
+    pc->skip_next_fi_callback = GNUNET_YES;
+    GNUNET_FS_file_information_inspect (fi, &fip_signal_suspend, pc);
+  }
+  suspend_operation (fi, pc);
+  *client_info = NULL;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Create SUSPEND event for the given publish operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext` to signal for
+ */
+void
+GNUNET_FS_publish_signal_suspend_ (void *cls)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  if (NULL != pc->upload_task)
+  {
+    GNUNET_SCHEDULER_cancel (pc->upload_task);
+    pc->upload_task = NULL;
+  }
+  pc->skip_next_fi_callback = GNUNET_YES;
+  GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_suspend, pc);
+  suspend_operation (pc->fi, pc);
+  GNUNET_FS_end_top (pc->h, pc->top);
+  pc->top = NULL;
+  publish_cleanup (pc);
+}
+
+
+/**
+ * We have gotten a reply for our space reservation request.
+ * Either fail (insufficient space) or start publishing for good.
+ *
+ * @param cls the `struct GNUNET_FS_PublishContext *`
+ * @param success positive reservation ID on success
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg error message on error, otherwise NULL
+ */
+static void
+finish_reserve (void *cls,
+                int success,
+                struct GNUNET_TIME_Absolute min_expiration,
+                const char *msg)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+
+  pc->qre = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Reservation complete (%d)!\n",
+              success);
+  if ((msg != NULL) || (success <= 0))
+  {
+    GNUNET_asprintf (&pc->fi->emsg,
+                     _ ("Datastore failure: %s"),
+                     msg);
+    signal_publish_error (pc->fi, pc, pc->fi->emsg);
+    return;
+  }
+  pc->rid = success;
+  GNUNET_assert (NULL == pc->upload_task);
+  pc->upload_task =
+    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND,
+                                        &GNUNET_FS_publish_main_, pc);
+}
+
+
+/**
+ * Calculate the total size of all of the files in the directory structure.
+ *
+ * @param fi file structure to traverse
+ */
+static uint64_t
+compute_contents_size (struct GNUNET_FS_FileInformation *fi)
+{
+  struct GNUNET_FS_FileInformation *ent;
+
+  if (GNUNET_YES != fi->is_directory)
+    return fi->data.file.file_size;
+  fi->data.dir.contents_size = 0;
+  for (ent = fi->data.dir.entries; NULL != ent; ent = ent->next)
+    fi->data.dir.contents_size += compute_contents_size (ent);
+  return fi->data.dir.contents_size;
+}
+
+
+/**
+ * Publish a file or directory.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param fi information about the file or directory structure to publish
+ * @param ns namespace to publish the file in, NULL for no namespace
+ * @param nid identifier to use for the published content in the namespace
+ *        (can be NULL, must be NULL if namespace is NULL)
+ * @param nuid update-identifier that will be used for future updates
+ *        (can be NULL, must be NULL if namespace or nid is NULL)
+ * @param options options for the publication
+ * @return context that can be used to control the publish operation
+ */
+struct GNUNET_FS_PublishContext *
+GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h,
+                         struct GNUNET_FS_FileInformation *fi,
+                         const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
+                         const char *nid,
+                         const char *nuid,
+                         enum GNUNET_FS_PublishOptions options)
+{
+  struct GNUNET_FS_PublishContext *ret;
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  GNUNET_assert (NULL != h);
+  compute_contents_size (fi);
+  if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
+  {
+    dsh = GNUNET_DATASTORE_connect (h->cfg);
+    if (NULL == dsh)
+      return NULL;
+  }
+  else
+  {
+    dsh = NULL;
+  }
+  ret = GNUNET_new (struct GNUNET_FS_PublishContext);
+  ret->dsh = dsh;
+  ret->h = h;
+  ret->fi = fi;
+  if (NULL != ns)
+  {
+    ret->ns = GNUNET_new (struct GNUNET_CRYPTO_EcdsaPrivateKey);
+    *ret->ns = *ns;
+    GNUNET_assert (NULL != nid);
+    ret->nid = GNUNET_strdup (nid);
+    if (NULL != nuid)
+      ret->nuid = GNUNET_strdup (nuid);
+  }
+  ret->options = options;
+  /* signal start */
+  GNUNET_FS_file_information_inspect (ret->fi, &fip_signal_start, ret);
+  ret->fi_pos = ret->fi;
+  ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret);
+  GNUNET_FS_publish_sync_ (ret);
+  if (NULL != ret->dsh)
+  {
+    GNUNET_assert (NULL == ret->qre);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _ (
+                  "Reserving space for %u entries and %llu bytes for publication\n"),
+                (unsigned int) ret->reserve_entries,
+                (unsigned long long) ret->reserve_space);
+    ret->qre =
+      GNUNET_DATASTORE_reserve (ret->dsh, ret->reserve_space,
+                                ret->reserve_entries,
+                                &finish_reserve,
+                                ret);
+  }
+  else
+  {
+    GNUNET_assert (NULL == ret->upload_task);
+    ret->upload_task =
+      GNUNET_SCHEDULER_add_with_priority
+        (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, ret);
+  }
+  return ret;
+}
+
+
+/**
+ * Signal the FS's progress function that we are stopping
+ * an upload.
+ *
+ * @param cls closure (of type `struct GNUNET_FS_PublishContext *`)
+ * @param fi the entry in the publish-structure
+ * @param length length of the file or directory
+ * @param meta metadata for the file or directory (can be modified)
+ * @param uri pointer to the keywords that will be used for this entry (can be modified)
+ * @param bo block options (can be modified)
+ * @param do_index should we index?
+ * @param client_info pointer to client context set upon creation (can be modified)
+ * @return #GNUNET_OK to continue (always)
+ */
+static int
+fip_signal_stop (void *cls,
+                 struct GNUNET_FS_FileInformation *fi,
+                 uint64_t length,
+                 struct GNUNET_FS_MetaData *meta,
+                 struct GNUNET_FS_Uri **uri,
+                 struct GNUNET_FS_BlockOptions *bo,
+                 int *do_index, void **client_info)
+{
+  struct GNUNET_FS_PublishContext *pc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t off;
+
+  if (GNUNET_YES == pc->skip_next_fi_callback)
+  {
+    pc->skip_next_fi_callback = GNUNET_NO;
+    return GNUNET_OK;
+  }
+  if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta))
+  {
+    /* process entries in directory first */
+    pc->skip_next_fi_callback = GNUNET_YES;
+    GNUNET_FS_file_information_inspect (fi, &fip_signal_stop, pc);
+  }
+  if (NULL != fi->serialization)
+  {
+    GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO,
+                                 fi->serialization);
+    GNUNET_free (fi->serialization);
+    fi->serialization = NULL;
+  }
+  off = (fi->chk_uri == NULL) ? 0 : length;
+  pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
+  GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off));
+  *client_info = NULL;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Stop an upload.  Will abort incomplete uploads (but
+ * not remove blocks that have already been published) or
+ * simply clean up the state for completed uploads.
+ * Must NOT be called from within the event callback!
+ *
+ * @param pc context for the upload to stop
+ */
+void
+GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t off;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Publish stop called\n");
+  GNUNET_FS_end_top (pc->h, pc->top);
+  if (NULL != pc->ksk_pc)
+  {
+    GNUNET_FS_publish_ksk_cancel (pc->ksk_pc);
+    pc->ksk_pc = NULL;
+  }
+  if (NULL != pc->sks_pc)
+  {
+    GNUNET_FS_publish_sks_cancel (pc->sks_pc);
+    pc->sks_pc = NULL;
+  }
+  if (NULL != pc->upload_task)
+  {
+    GNUNET_SCHEDULER_cancel (pc->upload_task);
+    pc->upload_task = NULL;
+  }
+  pc->skip_next_fi_callback = GNUNET_YES;
+  GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_stop, pc);
+
+  if (NULL != pc->fi->serialization)
+  {
+    GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO,
+                                 pc->fi->serialization);
+    GNUNET_free (pc->fi->serialization);
+    pc->fi->serialization = NULL;
+  }
+  off = (NULL == pc->fi->chk_uri) ? 0 : GNUNET_ntohll (
+    pc->fi->chk_uri->data.chk.file_length);
+
+  if (NULL != pc->serialization)
+  {
+    GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH,
+                                 pc->serialization);
+    GNUNET_free (pc->serialization);
+    pc->serialization = NULL;
+  }
+  if (NULL != pc->qre)
+  {
+    GNUNET_DATASTORE_cancel (pc->qre);
+    pc->qre = NULL;
+  }
+  pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED;
+  GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi, off));
+  publish_cleanup (pc);
+}
+
+
+/* end of fs_publish.c */
diff --git a/src/service/fs/fs_publish_ksk.c b/src/service/fs/fs_publish_ksk.c
new file mode 100644
index 000000000..3981ad335
--- /dev/null
+++ b/src/service/fs/fs_publish_ksk.c
@@ -0,0 +1,255 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_publish_ksk.c
+ * @brief publish a URI under a keyword in GNUnet
+ * @see https://gnunet.org/encoding and #2564
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+#include "fs_publish_ublock.h"
+
+/**
+ * Context for the KSK publication.
+ */
+struct GNUNET_FS_PublishKskContext
+{
+  /**
+   * Keywords to use.
+   */
+  struct GNUNET_FS_Uri *ksk_uri;
+
+  /**
+   * URI to publish.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Metadata to use.
+   */
+  struct GNUNET_FS_MetaData *meta;
+
+  /**
+   * Global FS context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * UBlock publishing operation that is active.
+   */
+  struct GNUNET_FS_PublishUblockContext *uc;
+
+  /**
+   * Handle to the datastore, NULL if we are just simulating.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * Current task.
+   */
+  struct GNUNET_SCHEDULER_Task *ksk_task;
+
+  /**
+   * Function to call once we're done.
+   */
+  GNUNET_FS_PublishContinuation cont;
+
+  /**
+   * Closure for cont.
+   */
+  void *cont_cls;
+
+  /**
+   * When should the KBlocks expire?
+   */
+  struct GNUNET_FS_BlockOptions bo;
+
+  /**
+   * Options to use.
+   */
+  enum GNUNET_FS_PublishOptions options;
+
+  /**
+   * Keyword that we are currently processing.
+   */
+  unsigned int i;
+};
+
+
+/**
+ * Continuation of #GNUNET_FS_publish_ksk() that performs
+ * the actual publishing operation (iterating over all
+ * of the keywords).
+ *
+ * @param cls closure of type `struct PublishKskContext *`
+ */
+static void
+publish_ksk_cont (void *cls);
+
+
+/**
+ * Function called by the datastore API with
+ * the result from the PUT request.
+ *
+ * @param cls closure of type `struct GNUNET_FS_PublishKskContext *`
+ * @param msg error message (or NULL)
+ */
+static void
+kb_put_cont (void *cls,
+             const char *msg)
+{
+  struct GNUNET_FS_PublishKskContext *pkc = cls;
+
+  pkc->uc = NULL;
+  if (NULL != msg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "KBlock PUT operation failed: %s\n", msg);
+    pkc->cont (pkc->cont_cls, NULL, msg);
+    GNUNET_FS_publish_ksk_cancel (pkc);
+    return;
+  }
+  pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc);
+}
+
+
+static void
+publish_ksk_cont (void *cls)
+{
+  struct GNUNET_FS_PublishKskContext *pkc = cls;
+  const char *keyword;
+
+  pkc->ksk_task = NULL;
+  if ((pkc->i == pkc->ksk_uri->data.ksk.keywordCount) ||
+      (NULL == pkc->dsh))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "KSK PUT operation complete\n");
+    pkc->cont (pkc->cont_cls, pkc->ksk_uri,
+               NULL);
+    GNUNET_FS_publish_ksk_cancel (pkc);
+    return;
+  }
+  keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++];
+  pkc->uc = GNUNET_FS_publish_ublock_ (pkc->h,
+                                       pkc->dsh,
+                                       keyword + 1 /* skip '+' */,
+                                       NULL,
+                                       GNUNET_CRYPTO_ecdsa_key_get_anonymous (),
+                                       pkc->meta,
+                                       pkc->uri,
+                                       &pkc->bo,
+                                       pkc->options,
+                                       &kb_put_cont, pkc);
+}
+
+
+/**
+ * Publish a CHK under various keywords on GNUnet.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param ksk_uri keywords to use
+ * @param meta metadata to use
+ * @param uri URI to refer to in the KBlock
+ * @param bo per-block options
+ * @param options publication options
+ * @param cont continuation
+ * @param cont_cls closure for cont
+ * @return NULL on error ('cont' will still be called)
+ */
+struct GNUNET_FS_PublishKskContext *
+GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
+                       const struct GNUNET_FS_Uri *ksk_uri,
+                       const struct GNUNET_FS_MetaData *meta,
+                       const struct GNUNET_FS_Uri *uri,
+                       const struct GNUNET_FS_BlockOptions *bo,
+                       enum GNUNET_FS_PublishOptions options,
+                       GNUNET_FS_PublishContinuation cont, void *cont_cls)
+{
+  struct GNUNET_FS_PublishKskContext *pkc;
+
+  GNUNET_assert (NULL != uri);
+  pkc = GNUNET_new (struct GNUNET_FS_PublishKskContext);
+  pkc->h = h;
+  pkc->bo = *bo;
+  pkc->options = options;
+  pkc->cont = cont;
+  pkc->cont_cls = cont_cls;
+  pkc->meta = GNUNET_FS_meta_data_duplicate (meta);
+  if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
+  {
+    pkc->dsh = GNUNET_DATASTORE_connect (h->cfg);
+    if (NULL == pkc->dsh)
+    {
+      cont (cont_cls,
+            NULL,
+            _ ("Could not connect to datastore."));
+      GNUNET_free (pkc);
+      return NULL;
+    }
+  }
+  pkc->uri = GNUNET_FS_uri_dup (uri);
+  pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri);
+  pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc);
+  return pkc;
+}
+
+
+/**
+ * Abort the KSK publishing operation.
+ *
+ * @param pkc context of the operation to abort.
+ */
+void
+GNUNET_FS_publish_ksk_cancel (struct GNUNET_FS_PublishKskContext *pkc)
+{
+  if (NULL != pkc->ksk_task)
+  {
+    GNUNET_SCHEDULER_cancel (pkc->ksk_task);
+    pkc->ksk_task = NULL;
+  }
+  if (NULL != pkc->uc)
+  {
+    GNUNET_FS_publish_ublock_cancel_ (pkc->uc);
+    pkc->uc = NULL;
+  }
+  if (NULL != pkc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO);
+    pkc->dsh = NULL;
+  }
+  GNUNET_FS_meta_data_destroy (pkc->meta);
+  GNUNET_FS_uri_destroy (pkc->ksk_uri);
+  GNUNET_FS_uri_destroy (pkc->uri);
+  GNUNET_free (pkc);
+}
+
+
+/* end of fs_publish_ksk.c */
diff --git a/src/service/fs/fs_publish_ublock.c b/src/service/fs/fs_publish_ublock.c
new file mode 100644
index 000000000..ad12d9b08
--- /dev/null
+++ b/src/service/fs/fs_publish_ublock.c
@@ -0,0 +1,299 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_publish_ublock.c
+ * @brief publish a UBLOCK in GNUnet
+ * @see https://gnunet.org/encoding and #2564
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_signatures.h"
+#include "fs_publish_ublock.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+
+
+/**
+ * Derive the key for symmetric encryption/decryption from
+ * the public key and the label.
+ *
+ * @param skey where to store symmetric key
+ * @param iv where to store the IV
+ * @param label label to use for key derivation
+ * @param pub public key to use for key derivation
+ */
+static void
+derive_ublock_encryption_key (struct GNUNET_CRYPTO_SymmetricSessionKey *skey,
+                              struct GNUNET_CRYPTO_SymmetricInitializationVector
+                              *iv,
+                              const char *label,
+                              const struct GNUNET_CRYPTO_EcdsaPublicKey *pub)
+{
+  struct GNUNET_HashCode key;
+
+  /* derive key from 'label' and public key of the namespace */
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CRYPTO_kdf (&key, sizeof(key),
+                                    "UBLOCK-ENC", strlen ("UBLOCK-ENC"),
+                                    label, strlen (label),
+                                    pub, sizeof(*pub),
+                                    NULL, 0));
+  GNUNET_CRYPTO_hash_to_aes_key (&key, skey, iv);
+}
+
+
+void
+GNUNET_FS_ublock_decrypt_ (const void *input,
+                           size_t input_len,
+                           const struct GNUNET_CRYPTO_EcdsaPublicKey *ns,
+                           const char *label,
+                           void *output)
+{
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
+
+  derive_ublock_encryption_key (&skey, &iv,
+                                label, ns);
+  GNUNET_CRYPTO_symmetric_decrypt (input, input_len,
+                                   &skey, &iv,
+                                   output);
+}
+
+
+/**
+ * Context for 'ublock_put_cont'.
+ */
+struct GNUNET_FS_PublishUblockContext
+{
+  /**
+   * Function to call when done.
+   */
+  GNUNET_FS_UBlockContinuation cont;
+
+  /**
+   * Closure of 'cont'.
+   */
+  void *cont_cls;
+
+  /**
+   * Handle for active datastore operation.
+   */
+  struct GNUNET_DATASTORE_QueueEntry *qre;
+
+  /**
+   * Task to run continuation asynchronously.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+};
+
+
+/**
+ * Continuation of #GNUNET_FS_publish_ublock_().
+ *
+ * @param cls closure of type "struct GNUNET_FS_PublishUblockContext*"
+ * @param success #GNUNET_SYSERR on failure (including timeout/queue drop)
+ *                #GNUNET_NO if content was already there
+ *                #GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority content to be stored
+ *                by the datacache at this time, zero for unknown, forever if we have no
+ *                space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+ublock_put_cont (void *cls,
+                 int32_t success,
+                 struct GNUNET_TIME_Absolute min_expiration,
+                 const char *msg)
+{
+  struct GNUNET_FS_PublishUblockContext *uc = cls;
+
+  uc->qre = NULL;
+  uc->cont (uc->cont_cls, msg);
+  GNUNET_free (uc);
+}
+
+
+/**
+ * Run the continuation.
+ *
+ * @param cls the `struct GNUNET_FS_PublishUblockContext *`
+ */
+static void
+run_cont (void *cls)
+{
+  struct GNUNET_FS_PublishUblockContext *uc = cls;
+
+  uc->task = NULL;
+  uc->cont (uc->cont_cls, NULL);
+  GNUNET_free (uc);
+}
+
+
+struct GNUNET_FS_PublishUblockContext *
+GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h,
+                           struct GNUNET_DATASTORE_Handle *dsh,
+                           const char *label,
+                           const char *ulabel,
+                           const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
+                           const struct GNUNET_FS_MetaData *meta,
+                           const struct GNUNET_FS_Uri *uri,
+                           const struct GNUNET_FS_BlockOptions *bo,
+                           enum GNUNET_FS_PublishOptions options,
+                           GNUNET_FS_UBlockContinuation cont, void *cont_cls)
+{
+  struct GNUNET_FS_PublishUblockContext *uc;
+  struct GNUNET_HashCode query;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey *nsd;
+  struct GNUNET_CRYPTO_EcdsaPublicKey pub;
+  char *uris;
+  size_t size;
+  char *kbe;
+  char *sptr;
+  ssize_t mdsize;
+  size_t slen;
+  size_t ulen;
+  struct UBlock *ub_plain;
+  struct UBlock *ub_enc;
+
+  /* compute ublock to publish */
+  if (NULL == meta)
+    mdsize = 0;
+  else
+    mdsize = GNUNET_FS_meta_data_get_serialized_size (meta);
+  GNUNET_assert (mdsize >= 0);
+  uris = GNUNET_FS_uri_to_string (uri);
+  slen = strlen (uris) + 1;
+  if (NULL == ulabel)
+    ulen = 1;
+  else
+    ulen = strlen (ulabel) + 1;
+  size = mdsize + sizeof(struct UBlock) + slen + ulen;
+  if (size > MAX_UBLOCK_SIZE)
+  {
+    size = MAX_UBLOCK_SIZE;
+    mdsize = size - sizeof(struct UBlock) - (slen + ulen);
+  }
+  ub_plain = GNUNET_malloc (size);
+  kbe = (char *) &ub_plain[1];
+  if (NULL != ulabel)
+    GNUNET_memcpy (kbe, ulabel, ulen);
+  kbe += ulen;
+  GNUNET_memcpy (kbe, uris, slen);
+  kbe += slen;
+  GNUNET_free (uris);
+  sptr = kbe;
+  if (NULL != meta)
+    mdsize =
+      GNUNET_FS_meta_data_serialize (meta, &sptr, mdsize,
+                                     GNUNET_FS_META_DATA_SERIALIZE_PART);
+  if (-1 == mdsize)
+  {
+    GNUNET_break (0);
+    GNUNET_free (ub_plain);
+    cont (cont_cls, _ ("Internal error."));
+    return NULL;
+  }
+  size = sizeof(struct UBlock) + slen + mdsize + ulen;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Publishing under identifier `%s'\n",
+              label);
+  /* get public key of the namespace */
+  GNUNET_CRYPTO_ecdsa_key_get_public (ns,
+                                      &pub);
+  derive_ublock_encryption_key (&skey, &iv,
+                                label, &pub);
+
+  /* encrypt ublock */
+  ub_enc = GNUNET_malloc (size);
+  GNUNET_CRYPTO_symmetric_encrypt (&ub_plain[1],
+                                   ulen + slen + mdsize,
+                                   &skey, &iv,
+                                   &ub_enc[1]);
+  GNUNET_free (ub_plain);
+  ub_enc->purpose.size = htonl (ulen + slen + mdsize
+                                + sizeof(struct UBlock)
+                                - sizeof(struct GNUNET_CRYPTO_EcdsaSignature));
+  ub_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_UBLOCK);
+
+  /* derive signing-key from 'label' and public key of the namespace */
+  nsd = GNUNET_CRYPTO_ecdsa_private_key_derive (ns, label, "fs-ublock");
+  GNUNET_CRYPTO_ecdsa_key_get_public (nsd,
+                                      &ub_enc->verification_key);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdsa_sign_ (nsd,
+                                            &ub_enc->purpose,
+                                            &ub_enc->signature));
+  GNUNET_CRYPTO_hash (&ub_enc->verification_key,
+                      sizeof(ub_enc->verification_key),
+                      &query);
+  GNUNET_free (nsd);
+
+  uc = GNUNET_new (struct GNUNET_FS_PublishUblockContext);
+  uc->cont = cont;
+  uc->cont_cls = cont_cls;
+  if (NULL != dsh)
+  {
+    uc->qre =
+      GNUNET_DATASTORE_put (dsh,
+                            0,
+                            &query,
+                            ulen + slen + mdsize + sizeof(struct UBlock),
+                            ub_enc,
+                            GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                            bo->content_priority,
+                            bo->anonymity_level,
+                            bo->replication_level,
+                            bo->expiration_time,
+                            -2, 1,
+                            &ublock_put_cont, uc);
+  }
+  else
+  {
+    uc->task = GNUNET_SCHEDULER_add_now (&run_cont,
+                                         uc);
+  }
+  GNUNET_free (ub_enc);
+  return uc;
+}
+
+
+/**
+ * Abort UBlock publishing operation.
+ *
+ * @param uc operation to abort.
+ */
+void
+GNUNET_FS_publish_ublock_cancel_ (struct GNUNET_FS_PublishUblockContext *uc)
+{
+  if (NULL != uc->qre)
+    GNUNET_DATASTORE_cancel (uc->qre);
+  if (NULL != uc->task)
+    GNUNET_SCHEDULER_cancel (uc->task);
+  GNUNET_free (uc);
+}
+
+
+/* end of fs_publish_ublock.c */
diff --git a/src/service/fs/fs_publish_ublock.h b/src/service/fs/fs_publish_ublock.h
new file mode 100644
index 000000000..4adffc6c1
--- /dev/null
+++ b/src/service/fs/fs_publish_ublock.h
@@ -0,0 +1,109 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010, 2012, 2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_publish_ublock.h
+ * @brief publish a UBLOCK in GNUnet
+ * @see https://gnunet.org/encoding and #2564
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ */
+#ifndef FS_PUBLISH_UBLOCK_H
+#define FS_PUBLISH_UBLOCK_H
+
+#include "gnunet_util_lib.h"
+#include "gnunet_datastore_service.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_identity_service.h"
+
+
+/**
+ * Decrypt the given UBlock, storing the result in output.
+ *
+ * @param input input data
+ * @param input_len number of bytes in @a input
+ * @param ns public key under which the UBlock was stored
+ * @param label label under which the UBlock was stored
+ * @param output where to write the result, has input_len bytes
+ */
+void
+GNUNET_FS_ublock_decrypt_ (const void *input,
+                           size_t input_len,
+                           const struct GNUNET_CRYPTO_EcdsaPublicKey *ns,
+                           const char *label,
+                           void *output);
+
+
+/**
+ * Context for 'ublock_put_cont'.
+ */
+struct GNUNET_FS_PublishUblockContext;
+
+
+/**
+ * Signature of a function called as the continuation of a UBlock
+ * publication.
+ *
+ * @param cls closure
+ * @param emsg error message, NULL on success
+ */
+typedef void (*GNUNET_FS_UBlockContinuation) (void *cls,
+                                              const char *emsg);
+
+
+/**
+ * Publish a UBlock.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param dsh datastore handle to use for storage operation
+ * @param label identifier to use
+ * @param ulabel update label to use, may be an empty string for none
+ * @param ns namespace to publish in
+ * @param meta metadata to use
+ * @param uri URI to refer to in the UBlock
+ * @param bo per-block options
+ * @param options publication options
+ * @param cont continuation
+ * @param cont_cls closure for @a cont
+ * @return NULL on error (@a cont will still be called)
+ */
+struct GNUNET_FS_PublishUblockContext *
+GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h,
+                           struct GNUNET_DATASTORE_Handle *dsh,
+                           const char *label,
+                           const char *ulabel,
+                           const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
+                           const struct GNUNET_FS_MetaData *meta,
+                           const struct GNUNET_FS_Uri *uri,
+                           const struct GNUNET_FS_BlockOptions *bo,
+                           enum GNUNET_FS_PublishOptions options,
+                           GNUNET_FS_UBlockContinuation cont, void *cont_cls);
+
+
+/**
+ * Abort UBlock publishing operation.
+ *
+ * @param uc operation to abort.
+ */
+void
+GNUNET_FS_publish_ublock_cancel_ (struct GNUNET_FS_PublishUblockContext *uc);
+
+#endif
diff --git a/src/service/fs/fs_search.c b/src/service/fs/fs_search.c
new file mode 100644
index 000000000..8b8c54c67
--- /dev/null
+++ b/src/service/fs/fs_search.c
@@ -0,0 +1,1826 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2001-2014 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/fs_search.c
+ * @brief Helper functions for searching.
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_protocols.h"
+#include "fs_api.h"
+#include "fs_publish_ublock.h"
+
+
+/**
+ * Number of availability trials we perform per search result.
+ */
+#define AVAILABILITY_TRIALS_MAX 8
+
+/**
+ * Fill in all of the generic fields for a search event and
+ * call the callback.
+ *
+ * @param pi structure to fill in
+ * @param h file-sharing handle
+ * @param sc overall search context
+ * @return value returned by the callback
+ */
+void *
+GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                               struct GNUNET_FS_Handle *h,
+                               struct GNUNET_FS_SearchContext *sc)
+{
+  void *ret;
+
+  pi->value.search.sc = sc;
+  pi->value.search.cctx = (NULL != sc) ? sc->client_info : NULL;
+  pi->value.search.pctx =
+    ((NULL == sc) || (NULL == sc->psearch_result))
+    ? NULL
+    : sc->psearch_result->client_info;
+  pi->value.search.query = (NULL != sc) ? sc->uri : NULL;
+  pi->value.search.duration = (NULL != sc)
+                              ? GNUNET_TIME_absolute_get_duration (
+    sc->start_time)
+                              : GNUNET_TIME_UNIT_ZERO;
+  pi->value.search.anonymity = (NULL != sc) ? sc->anonymity : 0;
+  pi->fsh = h;
+  ret = h->upcb (h->upcb_cls, pi);
+  return ret;
+}
+
+
+/**
+ * Check if the given result is identical to the given URI.
+ *
+ * @param cls points to the URI we check against
+ * @param key not used
+ * @param value a `struct GNUNET_FS_SearchResult` who's URI we
+ *        should compare with
+ * @return #GNUNET_SYSERR if the result is present,
+ *         #GNUNET_OK otherwise
+ */
+static int
+test_result_present (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     void *value)
+{
+  const struct GNUNET_FS_Uri *uri = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (GNUNET_FS_uri_test_equal (uri, sr->uri))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * We've found a new CHK result.  Let the client
+ * know about it.
+ *
+ * @param sc the search context
+ * @param sr the specific result
+ */
+static void
+notify_client_chk_result (struct GNUNET_FS_SearchContext *sc,
+                          struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_SEARCH_RESULT;
+  pi.value.search.specifics.result.meta = sr->meta;
+  pi.value.search.specifics.result.uri = sr->uri;
+  pi.value.search.specifics.result.result = sr;
+  pi.value.search.specifics.result.applicability_rank = sr->optional_support;
+  sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+}
+
+
+/**
+ * We've found new information about an existing CHK result.  Let the
+ * client know about it.
+ *
+ * @param sc the search context
+ * @param sr the specific result
+ */
+static void
+notify_client_chk_update (struct GNUNET_FS_SearchContext *sc,
+                          struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_SEARCH_UPDATE;
+  pi.value.search.specifics.update.cctx = sr->client_info;
+  pi.value.search.specifics.update.meta = sr->meta;
+  pi.value.search.specifics.update.uri = sr->uri;
+  pi.value.search.specifics.update.availability_rank =
+    2 * sr->availability_success - sr->availability_trials;
+  pi.value.search.specifics.update.availability_certainty =
+    sr->availability_trials;
+  pi.value.search.specifics.update.applicability_rank = sr->optional_support;
+  pi.value.search.specifics.update.current_probe_time
+    = GNUNET_TIME_absolute_get_duration (sr->probe_active_time);
+  sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+}
+
+
+/**
+ * Context for "get_result_present".
+ */
+struct GetResultContext
+{
+  /**
+   * The URI we're looking for.
+   */
+  const struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Where to store a pointer to the search
+   * result struct if we found a match.
+   */
+  struct GNUNET_FS_SearchResult *sr;
+};
+
+
+/**
+ * Check if the given result is identical to the given URI and if so
+ * return it.
+ *
+ * @param cls a `struct GetResultContext`
+ * @param key not used
+ * @param value a `struct GNUNET_FS_SearchResult` who's URI we
+ *        should compare with
+ * @return #GNUNET_OK
+ */
+static int
+get_result_present (void *cls,
+                    const struct GNUNET_HashCode *key,
+                    void *value)
+{
+  struct GetResultContext *grc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (GNUNET_FS_uri_test_equal (grc->uri, sr->uri))
+    grc->sr = sr;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Signal result of last probe to client and then schedule next
+ * probe.
+ *
+ * @param sr search result to signal for
+ */
+static void
+signal_probe_result (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_SEARCH_UPDATE;
+  pi.value.search.specifics.update.cctx = sr->client_info;
+  pi.value.search.specifics.update.meta = sr->meta;
+  pi.value.search.specifics.update.uri = sr->uri;
+  pi.value.search.specifics.update.availability_rank
+    = 2 * sr->availability_success - sr->availability_trials;
+  pi.value.search.specifics.update.availability_certainty
+    = sr->availability_trials;
+  pi.value.search.specifics.update.applicability_rank = sr->optional_support;
+  pi.value.search.specifics.update.current_probe_time
+    = GNUNET_TIME_absolute_get_duration (sr->probe_active_time);
+  sr->client_info = GNUNET_FS_search_make_status_ (&pi, sr->h, sr->sc);
+  GNUNET_FS_search_start_probe_ (sr);
+}
+
+
+/**
+ * Handle the case where we have failed to receive a response for our probe.
+ *
+ * @param cls our `struct GNUNET_FS_SearchResult *`
+ */
+static void
+probe_failure_handler (void *cls)
+{
+  struct GNUNET_FS_SearchResult *sr = cls;
+
+  sr->probe_cancel_task = NULL;
+  sr->availability_trials++;
+  GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
+  sr->probe_ctx = NULL;
+  GNUNET_FS_stop_probe_ping_task_ (sr);
+  GNUNET_FS_search_result_sync_ (sr);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Probe #%u for search result %p failed\n",
+              sr->availability_trials,
+              sr);
+  signal_probe_result (sr);
+}
+
+
+/**
+ * Handle the case where we have gotten a response for our probe.
+ *
+ * @param cls our `struct GNUNET_FS_SearchResult *`
+ */
+static void
+probe_success_handler (void *cls)
+{
+  struct GNUNET_FS_SearchResult *sr = cls;
+
+  sr->probe_cancel_task = NULL;
+  sr->availability_trials++;
+  sr->availability_success++;
+  GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
+  sr->probe_ctx = NULL;
+  GNUNET_FS_stop_probe_ping_task_ (sr);
+  GNUNET_FS_search_result_sync_ (sr);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Probe #%u for search result %p succeeded\n",
+              sr->availability_trials,
+              sr);
+  signal_probe_result (sr);
+}
+
+
+/**
+ * Notification of FS that a search probe has made progress.
+ * This function is used INSTEAD of the client's event handler
+ * for downloads where the #GNUNET_FS_DOWNLOAD_IS_PROBE flag is set.
+ *
+ * @param cls closure, always NULL (!), actual closure
+ *        is in the client-context of the info struct
+ * @param info details about the event, specifying the event type
+ *        and various bits about the event
+ * @return client-context (for the next progress call
+ *         for this operation; should be set to NULL for
+ *         SUSPEND and STOPPED events).  The value returned
+ *         will be passed to future callbacks in the respective
+ *         field in the `struct GNUNET_FS_ProgressInfo`.
+ */
+void *
+GNUNET_FS_search_probe_progress_ (void *cls,
+                                  const struct GNUNET_FS_ProgressInfo *info)
+{
+  struct GNUNET_FS_SearchResult *sr = info->value.download.cctx;
+  struct GNUNET_TIME_Relative dur;
+
+  switch (info->status)
+  {
+  case GNUNET_FS_STATUS_DOWNLOAD_START:
+    /* ignore */
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_RESUME:
+    /* probes should never be resumed */
+    GNUNET_assert (0);
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND:
+    /* probes should never be suspended */
+    GNUNET_break (0);
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS:
+    /* ignore */
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_ERROR:
+    if (NULL != sr->probe_cancel_task)
+    {
+      GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+      sr->probe_cancel_task = NULL;
+    }
+    sr->probe_cancel_task =
+      GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time,
+                                    &probe_failure_handler, sr);
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED:
+    if (NULL != sr->probe_cancel_task)
+    {
+      GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+      sr->probe_cancel_task = NULL;
+    }
+    sr->probe_cancel_task =
+      GNUNET_SCHEDULER_add_now (&probe_success_handler, sr);
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_STOPPED:
+    if (NULL != sr->probe_cancel_task)
+    {
+      GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+      sr->probe_cancel_task = NULL;
+    }
+    sr = NULL;
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE:
+    if (NULL == sr->probe_cancel_task)
+    {
+      sr->probe_active_time = GNUNET_TIME_absolute_get ();
+      sr->probe_cancel_task =
+        GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time,
+                                      &probe_failure_handler, sr);
+    }
+    break;
+  case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE:
+    if (NULL != sr->probe_cancel_task)
+    {
+      GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+      sr->probe_cancel_task = NULL;
+    }
+    dur = GNUNET_TIME_absolute_get_duration (sr->probe_active_time);
+    sr->remaining_probe_time =
+      GNUNET_TIME_relative_subtract (sr->remaining_probe_time, dur);
+    if (0 == sr->remaining_probe_time.rel_value_us)
+      sr->probe_cancel_task =
+        GNUNET_SCHEDULER_add_now (&probe_failure_handler, sr);
+    GNUNET_FS_search_result_sync_ (sr);
+    break;
+
+  default:
+    GNUNET_break (0);
+    return NULL;
+  }
+  return sr;
+}
+
+
+/**
+ * Task run periodically to remind clients that a probe is active.
+ *
+ * @param cls the `struct GNUNET_FS_SearchResult` that we are probing for
+ */
+static void
+probe_ping_task_cb (void *cls)
+{
+  struct GNUNET_FS_Handle *h = cls;
+
+  for (struct GNUNET_FS_SearchResult *sr = h->probes_head;
+       NULL != sr;
+       sr = sr->next)
+    if (NULL != sr->probe_ctx->mq)
+      signal_probe_result (sr);
+  h->probe_ping_task
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_FS_PROBE_UPDATE_FREQUENCY,
+                                    &probe_ping_task_cb,
+                                    h);
+}
+
+
+/**
+ * Start the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+static void
+start_probe_ping_task (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_Handle *h = sr->h;
+
+  GNUNET_CONTAINER_DLL_insert (h->probes_head,
+                               h->probes_tail,
+                               sr);
+  if (NULL == h->probe_ping_task)
+    h->probe_ping_task
+      = GNUNET_SCHEDULER_add_now (&probe_ping_task_cb,
+                                  h);
+}
+
+
+/**
+ * Stop the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+void
+GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_Handle *h = sr->h;
+
+  GNUNET_CONTAINER_DLL_remove (h->probes_head,
+                               h->probes_tail,
+                               sr);
+  if (NULL == h->probes_head)
+  {
+    GNUNET_SCHEDULER_cancel (h->probe_ping_task);
+    h->probe_ping_task = NULL;
+  }
+}
+
+
+/**
+ * Start download probes for the given search result.
+ *
+ * @param sr the search result
+ */
+void
+GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr)
+{
+  uint64_t off;
+  uint64_t len;
+
+  if (NULL != sr->probe_ctx)
+    return;
+  if (NULL != sr->download)
+    return;
+  if (0 == (sr->h->flags & GNUNET_FS_FLAGS_DO_PROBES))
+    return;
+  if (sr->availability_trials > AVAILABILITY_TRIALS_MAX)
+    return;
+  if ( (GNUNET_FS_URI_CHK != sr->uri->type) &&
+       (GNUNET_FS_URI_LOC != sr->uri->type) )
+    return;
+  len = GNUNET_FS_uri_chk_get_file_size (sr->uri);
+  if (0 == len)
+    return;
+  if ((len <= DBLOCK_SIZE) && (sr->availability_success > 0))
+    return;
+  off = len / DBLOCK_SIZE;
+  if (off > 0)
+    off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, off);
+  off *= DBLOCK_SIZE;
+  if (len - off < DBLOCK_SIZE)
+    len = len - off;
+  else
+    len = DBLOCK_SIZE;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting probe #%u (at offset %llu) for search result %p\n",
+              sr->availability_trials + 1,
+              (unsigned long long) off,
+              sr);
+  sr->remaining_probe_time =
+    GNUNET_TIME_relative_saturating_multiply (sr->h->avg_block_latency,
+                                              2 * (1
+                                                   + sr->availability_trials));
+  sr->probe_ctx =
+    GNUNET_FS_download_start (sr->h, sr->uri, sr->meta, NULL, NULL, off,
+                              len, sr->anonymity,
+                              GNUNET_FS_DOWNLOAD_NO_TEMPORARIES
+                              | GNUNET_FS_DOWNLOAD_IS_PROBE, sr, NULL);
+  start_probe_ping_task (sr);
+}
+
+
+/**
+ * Start download probes for the given search result.
+ *
+ * @param h file-sharing handle to use for the operation
+ * @param uri URI to probe
+ * @param meta meta data associated with the URI
+ * @param client_info client info pointer to use for associated events
+ * @param anonymity anonymity level to use for the probes
+ * @return the search result handle to access the probe activity
+ */
+struct GNUNET_FS_SearchResult *
+GNUNET_FS_probe (struct GNUNET_FS_Handle *h,
+                 const struct GNUNET_FS_Uri *uri,
+                 const struct GNUNET_FS_MetaData *meta,
+                 void *client_info,
+                 uint32_t anonymity)
+{
+  struct GNUNET_FS_SearchResult *sr;
+
+  GNUNET_assert (NULL != h);
+  GNUNET_assert (NULL != uri);
+  sr = GNUNET_new (struct GNUNET_FS_SearchResult);
+  sr->h = h;
+  sr->uri = GNUNET_FS_uri_dup (uri);
+  sr->meta = GNUNET_FS_meta_data_duplicate (meta);
+  sr->client_info = client_info;
+  sr->anonymity = anonymity;
+  GNUNET_FS_search_start_probe_ (sr);
+  return sr;
+}
+
+
+/**
+ * Stop probing activity associated with a search result.
+ *
+ * @param sr search result
+ */
+static void
+GNUNET_FS_search_stop_probe_ (struct GNUNET_FS_SearchResult *sr)
+{
+  if (NULL != sr->probe_ctx)
+  {
+    GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
+    sr->probe_ctx = NULL;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
+  }
+  if (NULL != sr->probe_cancel_task)
+  {
+    GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+    sr->probe_cancel_task = NULL;
+  }
+}
+
+
+/**
+ * Stop probe activity.  Must ONLY be used on values
+ * returned from #GNUNET_FS_probe.
+ *
+ * @param sr search result to stop probing for (freed)
+ * @return the value of the 'client_info' pointer
+ */
+void *
+GNUNET_FS_probe_stop (struct GNUNET_FS_SearchResult *sr)
+{
+  void *client_info;
+
+  GNUNET_assert (NULL == sr->sc);
+  GNUNET_FS_search_stop_probe_ (sr);
+  GNUNET_FS_uri_destroy (sr->uri);
+  GNUNET_FS_meta_data_destroy (sr->meta);
+  client_info = sr->client_info;
+  GNUNET_free (sr);
+  return client_info;
+}
+
+
+/**
+ * We have received a KSK result.  Check how it fits in with the
+ * overall query and notify the client accordingly.
+ *
+ * @param sc context for the overall query
+ * @param ent entry for the specific keyword
+ * @param uri the URI that was found
+ * @param meta metadata associated with the URI
+ *        under the @a ent keyword
+ */
+static void
+process_ksk_result (struct GNUNET_FS_SearchContext *sc,
+                    struct SearchRequestEntry *ent,
+                    const struct GNUNET_FS_Uri *uri,
+                    const struct GNUNET_FS_MetaData *meta)
+{
+  struct GNUNET_HashCode key;
+  struct GNUNET_FS_SearchResult *sr;
+  struct GetResultContext grc;
+  int is_new;
+  unsigned int koff;
+
+  /* check if new */
+  GNUNET_assert (NULL != sc);
+  if (GNUNET_OK !=
+      GNUNET_FS_uri_to_key (uri,
+                            &key))
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  if (GNUNET_SYSERR ==
+      GNUNET_CONTAINER_multihashmap_get_multiple (ent->results,
+                                                  &key,
+                                                  &test_result_present,
+                                                  (void *) uri))
+    return;                     /* duplicate result */
+  /* try to find search result in master map */
+  grc.sr = NULL;
+  grc.uri = uri;
+  GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map,
+                                              &key,
+                                              &get_result_present, &grc);
+  sr = grc.sr;
+  is_new = (NULL == sr) || (sr->mandatory_missing > 0);
+  if (NULL == sr)
+  {
+    sr = GNUNET_new (struct GNUNET_FS_SearchResult);
+    sr->h = sc->h;
+    sr->sc = sc;
+    sr->anonymity = sc->anonymity;
+    sr->uri = GNUNET_FS_uri_dup (uri);
+    sr->meta = GNUNET_FS_meta_data_duplicate (meta);
+    sr->mandatory_missing = sc->mandatory_count;
+    sr->key = key;
+    sr->keyword_bitmap = GNUNET_malloc ((sc->uri->data.ksk.keywordCount + 7)
+                                        / 8);                                       /* round up, count bits */
+    GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr,
+                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  }
+  else
+  {
+    GNUNET_FS_meta_data_merge (sr->meta, meta);
+  }
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONTAINER_multihashmap_put (ent->results,
+                                                   &sr->key,
+                                                   sr,
+                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+
+  koff = ent - sc->requests;
+  GNUNET_assert ((ent >= sc->requests) &&
+                 (koff < sc->uri->data.ksk.keywordCount));
+  sr->keyword_bitmap[koff / 8] |= (1 << (koff % 8));
+  /* check if mandatory satisfied */
+  if (1 <= GNUNET_CONTAINER_multihashmap_size (ent->results))
+  {
+    if (ent->mandatory)
+    {
+      GNUNET_break (sr->mandatory_missing > 0);
+      sr->mandatory_missing--;
+    }
+    else
+    {
+      sr->optional_support++;
+    }
+  }
+  if (0 != sr->mandatory_missing)
+  {
+    GNUNET_break (NULL == sr->client_info);
+    return;
+  }
+  if (is_new)
+    notify_client_chk_result (sc, sr);
+  else
+    notify_client_chk_update (sc, sr);
+  GNUNET_FS_search_result_sync_ (sr);
+  GNUNET_FS_search_start_probe_ (sr);
+}
+
+
+/**
+ * Start search for content, internal API.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param uri specifies the search parameters; can be
+ *        a KSK URI or an SKS URI.
+ * @param anonymity desired level of anonymity
+ * @param options options for the search
+ * @param cctx client context
+ * @param psearch parent search result (for namespace update searches)
+ * @return context that can be used to control the search
+ */
+static struct GNUNET_FS_SearchContext *
+search_start (struct GNUNET_FS_Handle *h,
+              const struct GNUNET_FS_Uri *uri,
+              uint32_t anonymity,
+              enum GNUNET_FS_SearchOptions options,
+              void *cctx,
+              struct GNUNET_FS_SearchResult *psearch);
+
+
+/**
+ * We have received an SKS result.  Start searching for updates and
+ * notify the client if it is a new result.
+ *
+ * @param sc context for the overall query
+ * @param id_update identifier for updates, NULL for none
+ * @param uri the URI that was found
+ * @param meta metadata associated with the URI
+ */
+static void
+process_sks_result (struct GNUNET_FS_SearchContext *sc,
+                    const char *id_update,
+                    const struct GNUNET_FS_Uri *uri,
+                    const struct GNUNET_FS_MetaData *meta)
+{
+  struct GNUNET_FS_Uri uu;
+  struct GNUNET_HashCode key;
+  struct GNUNET_FS_SearchResult *sr;
+
+  /* check if new */
+  GNUNET_assert (NULL != sc);
+  if (GNUNET_OK !=
+      GNUNET_FS_uri_to_key (uri,
+                            &key))
+  {
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_CRYPTO_hash_xor (&uri->data.chk.chk.key,
+                          &uri->data.chk.chk.query,
+                          &key);
+  if (GNUNET_SYSERR ==
+      GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map, &key,
+                                                  &test_result_present,
+                                                  (void *) uri))
+    return;                     /* duplicate result */
+  sr = GNUNET_new (struct GNUNET_FS_SearchResult);
+  sr->h = sc->h;
+  sr->sc = sc;
+  sr->anonymity = sc->anonymity;
+  sr->uri = GNUNET_FS_uri_dup (uri);
+  sr->meta = GNUNET_FS_meta_data_duplicate (meta);
+  sr->key = key;
+  GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  GNUNET_FS_search_result_sync_ (sr);
+  GNUNET_FS_search_start_probe_ (sr);
+  /* notify client */
+  if (0 == sr->mandatory_missing)
+    notify_client_chk_result (sc, sr);
+  else
+    GNUNET_break (NULL == sr->client_info);
+  /* search for updates */
+  if (0 == strlen (id_update))
+    return;                     /* no updates */
+  uu.type = GNUNET_FS_URI_SKS;
+  uu.data.sks.ns = sc->uri->data.sks.ns;
+  uu.data.sks.identifier = GNUNET_strdup (id_update);
+  (void) search_start (sc->h, &uu, sc->anonymity, sc->options, NULL, sr);
+  GNUNET_free (uu.data.sks.identifier);
+}
+
+
+/**
+ * Decrypt a ublock using a 'keyword' as the passphrase.  Given the
+ * KSK public key derived from the keyword, this function looks up
+ * the original keyword in the search context and decrypts the
+ * given ciphertext block.
+ *
+ * @param sc search context with the keywords
+ * @param dpub derived public key used for the search
+ * @param edata encrypted data
+ * @param edata_size number of bytes in @a edata (and @a data)
+ * @param data where to store the plaintext
+ * @return keyword index on success, #GNUNET_SYSERR on error (no such
+ *         keyword, internal error)
+ */
+static int
+decrypt_block_with_keyword (const struct GNUNET_FS_SearchContext *sc,
+                            const struct GNUNET_CRYPTO_EcdsaPublicKey *dpub,
+                            const void *edata,
+                            size_t edata_size,
+                            char *data)
+{
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
+  struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+  unsigned int i;
+
+  /* find key */
+  for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
+    if (0 == memcmp (dpub,
+                     &sc->requests[i].dpub,
+                     sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey)))
+      break;
+  if (i == sc->uri->data.ksk.keywordCount)
+  {
+    /* oops, does not match any of our keywords!? */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  /* decrypt */
+  anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub);
+  GNUNET_FS_ublock_decrypt_ (edata, edata_size,
+                             &anon_pub,
+                             sc->requests[i].keyword,
+                             data);
+  return i;
+}
+
+
+/**
+ * Process a keyword search result.  The actual type of block is
+ * a UBlock; we know it is a keyword search result because that's
+ * what we were searching for.
+ *
+ * @param sc our search context
+ * @param ub the ublock with the keyword search result
+ * @param size size of @a ub
+ */
+static void
+process_kblock (struct GNUNET_FS_SearchContext *sc,
+                const struct UBlock *ub,
+                size_t size)
+{
+  size_t j;
+  char pt[size - sizeof(struct UBlock)];
+  const char *eos;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *uri;
+  char *emsg;
+  int i;
+
+  if (-1 == (i = decrypt_block_with_keyword (sc,
+                                             &ub->verification_key,
+                                             &ub[1],
+                                             size - sizeof(struct UBlock),
+                                             pt)))
+    return;
+  /* parse; pt[0] is just '\0', so we skip over that */
+  eos = memchr (&pt[1], '\0', sizeof(pt) - 1);
+  if (NULL == eos)
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  if (NULL == (uri = GNUNET_FS_uri_parse (&pt[1], &emsg)))
+  {
+    if (GNUNET_FS_VERSION > 0x00090400)
+    {
+      /* we broke this in 0x00090300, so don't bitch
+         too loudly just one version up... */
+      GNUNET_break_op (0);        /* ublock malformed */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _ ("Failed to parse URI `%s': %s\n"),
+                  &pt[1],
+                  emsg);
+    }
+    GNUNET_free (emsg);
+    return;
+  }
+  j = eos - pt + 1;
+  if (sizeof(pt) == j)
+    meta = GNUNET_FS_meta_data_create ();
+  else
+    meta = GNUNET_FS_meta_data_deserialize (&pt[j], sizeof(pt) - j);
+  if (NULL == meta)
+  {
+    GNUNET_break_op (0);        /* ublock malformed */
+    GNUNET_FS_uri_destroy (uri);
+    return;
+  }
+  process_ksk_result (sc,
+                      &sc->requests[i],
+                      uri,
+                      meta);
+
+  /* clean up */
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_FS_uri_destroy (uri);
+}
+
+
+/**
+ * Process a namespace-search result.  The actual type of block is
+ * a UBlock; we know it is a namespace search result because that's
+ * what we were searching for.
+ *
+ * @param sc our search context
+ * @param ub the ublock with a namespace result
+ * @param size size of @a ub
+ */
+static void
+process_sblock (struct GNUNET_FS_SearchContext *sc,
+                const struct UBlock *ub,
+                size_t size)
+{
+  size_t len = size - sizeof(struct UBlock);
+  char pt[len];
+  struct GNUNET_FS_Uri *uri;
+  struct GNUNET_FS_MetaData *meta;
+  const char *id;
+  const char *uris;
+  size_t off;
+  char *emsg;
+
+  GNUNET_FS_ublock_decrypt_ (&ub[1], len,
+                             &sc->uri->data.sks.ns,
+                             sc->uri->data.sks.identifier,
+                             pt);
+  /* parse */
+  if (0 == (off = GNUNET_STRINGS_buffer_tokenize (pt, len, 2, &id, &uris)))
+  {
+    GNUNET_break_op (0);        /* ublock malformed */
+    return;
+  }
+  if (NULL == (meta = GNUNET_FS_meta_data_deserialize (&pt[off], len
+                                                       - off)))
+  {
+    GNUNET_break_op (0);        /* ublock malformed */
+    return;
+  }
+  if (NULL == (uri = GNUNET_FS_uri_parse (uris, &emsg)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to parse URI `%s': %s\n"),
+                uris, emsg);
+    GNUNET_break_op (0);        /* ublock malformed */
+    GNUNET_free (emsg);
+    GNUNET_FS_meta_data_destroy (meta);
+    return;
+  }
+  /* process */
+  process_sks_result (sc, id, uri, meta);
+  /* clean up */
+  GNUNET_FS_uri_destroy (uri);
+  GNUNET_FS_meta_data_destroy (meta);
+}
+
+
+/**
+ * Shutdown any existing connection to the FS
+ * service and try to establish a fresh one
+ * (and then re-transmit our search request).
+ *
+ * @param sc the search to reconnec
+ */
+static void
+try_reconnect (struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * We check a result message from the service.
+ *
+ * @param cls closure
+ * @param cm result message received
+ */
+static int
+check_result (void *cls,
+              const struct ClientPutMessage *cm)
+{
+  /* payload of any variable size is OK */
+  return GNUNET_OK;
+}
+
+
+/**
+ * We process a search result from the service.
+ *
+ * @param cls closure
+ * @param cm result message received
+ */
+static void
+handle_result (void *cls,
+               const struct ClientPutMessage *cm)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  uint16_t msize = ntohs (cm->header.size) - sizeof(*cm);
+  enum GNUNET_BLOCK_Type type = ntohl (cm->type);
+
+  if (GNUNET_TIME_absolute_get_duration (GNUNET_TIME_absolute_ntoh (
+                                           cm->expiration)).rel_value_us > 0)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Result received has already expired.\n");
+    return;                     /* result expired */
+  }
+  switch (type)
+  {
+  case GNUNET_BLOCK_TYPE_FS_UBLOCK:
+    if (GNUNET_FS_URI_SKS == sc->uri->type)
+      process_sblock (sc,
+                      (const struct UBlock *) &cm[1],
+                      msize);
+    else
+      process_kblock (sc,
+                      (const struct UBlock *) &cm[1],
+                      msize);
+    break;
+
+  case GNUNET_BLOCK_TYPE_ANY:
+    GNUNET_break (0);
+    break;
+
+  case GNUNET_BLOCK_TYPE_FS_DBLOCK:
+    GNUNET_break (0);
+    break;
+
+  case GNUNET_BLOCK_TYPE_FS_ONDEMAND:
+    GNUNET_break (0);
+    break;
+
+  case GNUNET_BLOCK_TYPE_FS_IBLOCK:
+    GNUNET_break (0);
+    break;
+
+  default:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Got result with unknown block type `%d', ignoring"),
+                type);
+    break;
+  }
+}
+
+
+/**
+ * Schedule the transmission of the (next) search request
+ * to the service.
+ *
+ * @param sc context for the search
+ */
+static void
+schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc);
+
+
+/**
+ * Closure for #build_result_set().
+ */
+struct MessageBuilderContext
+{
+  /**
+   * How many entries can we store to xoff.
+   */
+  unsigned int put_cnt;
+
+  /**
+   * How many entries should we skip.
+   */
+  unsigned int skip_cnt;
+
+  /**
+   * Where to store the keys.
+   */
+  struct GNUNET_HashCode *xoff;
+
+  /**
+   * Search context we are iterating for.
+   */
+  struct GNUNET_FS_SearchContext *sc;
+
+  /**
+   * Keyword offset the search result must match (0 for SKS)
+   */
+  unsigned int keyword_offset;
+};
+
+
+/**
+ * Iterating over the known results, pick those matching the given
+ * result range and store their keys at 'xoff'.
+ *
+ * @param cls the `struct MessageBuilderContext`
+ * @param key key for a result
+ * @param value the search result
+ * @return #GNUNET_OK to continue iterating
+ */
+static int
+build_result_set (void *cls,
+                  const struct GNUNET_HashCode *key,
+                  void *value)
+{
+  struct MessageBuilderContext *mbc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if ((NULL != sr->keyword_bitmap) &&
+      (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1
+                                                            << (mbc->
+                                                                keyword_offset
+                                                                % 8)))))
+    return GNUNET_OK; /* have no match for this keyword yet */
+  if (mbc->skip_cnt > 0)
+  {
+    mbc->skip_cnt--;
+    return GNUNET_OK;
+  }
+  if (0 == mbc->put_cnt)
+    return GNUNET_SYSERR;
+  mbc->xoff[--mbc->put_cnt] = *key;
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Iterating over the known results, count those matching the given
+ * result range and increment put count for each.
+ *
+ * @param cls the `struct MessageBuilderContext`
+ * @param key key for a result
+ * @param value the search result
+ * @return #GNUNET_OK to continue iterating
+ */
+static int
+find_result_set (void *cls,
+                 const struct GNUNET_HashCode *key,
+                 void *value)
+{
+  struct MessageBuilderContext *mbc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if ((NULL != sr->keyword_bitmap) &&
+      (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1
+                                                            << (mbc->
+                                                                keyword_offset
+                                                                % 8)))))
+    return GNUNET_OK; /* have no match for this keyword yet */
+  mbc->put_cnt++;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Schedule the transmission of the (next) search request
+ * to the service.
+ *
+ * @param sc context for the search
+ */
+static void
+schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
+{
+  struct MessageBuilderContext mbc;
+  struct GNUNET_MQ_Envelope *env;
+  struct SearchMessage *sm;
+  struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
+  unsigned int total_seen_results; /* total number of result hashes to send */
+  uint32_t options;
+  unsigned int left;
+  unsigned int todo;
+  unsigned int fit;
+  unsigned int search_request_map_offset;
+  unsigned int keyword_offset;
+  int first_call;
+
+  memset (&mbc, 0, sizeof(mbc));
+  mbc.sc = sc;
+  if (GNUNET_FS_uri_test_ksk (sc->uri))
+  {
+    /* This will calculate the result set size ONLY for
+       "keyword_offset == 0", so we will have to recalculate
+       it for the other keywords later! */
+    GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                           &find_result_set,
+                                           &mbc);
+    total_seen_results = mbc.put_cnt;
+  }
+  else
+  {
+    total_seen_results
+      = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map);
+  }
+  search_request_map_offset = 0;
+  keyword_offset = 0;
+  first_call = GNUNET_YES;
+  while ((0 != (left =
+                  (total_seen_results - search_request_map_offset))) ||
+         (GNUNET_YES == first_call))
+  {
+    first_call = GNUNET_NO;
+    options = SEARCH_MESSAGE_OPTION_NONE;
+    if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY))
+      options |= SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY;
+
+    fit = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof(*sm)) / sizeof(struct
+                                                               GNUNET_HashCode);
+    todo = GNUNET_MIN (fit,
+                       left);
+    env = GNUNET_MQ_msg_extra (sm,
+                               sizeof(struct GNUNET_HashCode) * todo,
+                               GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
+    mbc.skip_cnt = search_request_map_offset;
+    mbc.xoff = (struct GNUNET_HashCode *) &sm[1];
+    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
+    sm->anonymity_level = htonl (sc->anonymity);
+    memset (&sm->target,
+            0,
+            sizeof(struct GNUNET_PeerIdentity));
+
+    if (GNUNET_FS_uri_test_ksk (sc->uri))
+    {
+      mbc.keyword_offset = keyword_offset;
+      /* calculate how many results we can send in this message */
+      mbc.put_cnt = todo;
+      /* now build message */
+      sm->query = sc->requests[keyword_offset].uquery;
+      GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                             &build_result_set,
+                                             &mbc);
+      search_request_map_offset += todo;
+      GNUNET_assert (0 == mbc.put_cnt);
+      GNUNET_assert (total_seen_results >= search_request_map_offset);
+      if (total_seen_results != search_request_map_offset)
+      {
+        /* more requesting to be done... */
+        sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED);
+      }
+      else
+      {
+        sm->options = htonl (options);
+        keyword_offset++;
+        if (sc->uri->data.ksk.keywordCount != keyword_offset)
+        {
+          /* more keywords => more requesting to be done... */
+          first_call = GNUNET_YES;
+          search_request_map_offset = 0;
+          mbc.put_cnt = 0;
+          mbc.keyword_offset = keyword_offset;
+          GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                                 &find_result_set,
+                                                 &mbc);
+          total_seen_results = mbc.put_cnt;
+        }
+      }
+    }
+    else
+    {
+      GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri));
+
+      GNUNET_CRYPTO_ecdsa_public_key_derive (&sc->uri->data.sks.ns,
+                                             sc->uri->data.sks.identifier,
+                                             "fs-ublock",
+                                             &dpub);
+      GNUNET_CRYPTO_hash (&dpub,
+                          sizeof(dpub),
+                          &sm->query);
+      mbc.put_cnt = todo;
+      mbc.keyword_offset = 0;
+      GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                             &build_result_set,
+                                             &mbc);
+      GNUNET_assert (total_seen_results >= search_request_map_offset);
+      if (total_seen_results != search_request_map_offset)
+      {
+        /* more requesting to be done... */
+        sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED);
+      }
+      else
+      {
+        sm->options = htonl (options);
+      }
+    }
+    GNUNET_MQ_send (sc->mq,
+                    env);
+  }
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_SearchContext *`
+ * @param error error code
+ */
+static void
+search_mq_error_handler (void *cls,
+                         enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+
+  if (NULL != sc->mq)
+  {
+    GNUNET_MQ_destroy (sc->mq);
+    sc->mq = NULL;
+  }
+  try_reconnect (sc);
+}
+
+
+/**
+ * Reconnect to the FS service and transmit
+ * our queries NOW.
+ *
+ * @param cls our search context
+ */
+static void
+do_reconnect (void *cls)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (result,
+                           GNUNET_MESSAGE_TYPE_FS_PUT,
+                           struct ClientPutMessage,
+                           sc),
+    GNUNET_MQ_handler_end ()
+  };
+
+  sc->task = NULL;
+  sc->mq = GNUNET_CLIENT_connect (sc->h->cfg,
+                                  "fs",
+                                  handlers,
+                                  &search_mq_error_handler,
+                                  sc);
+  if (NULL == sc->mq)
+  {
+    try_reconnect (sc);
+    return;
+  }
+  schedule_transmit_search_request (sc);
+}
+
+
+/**
+ * Shutdown any existing connection to the FS
+ * service and try to establish a fresh one
+ * (and then re-transmit our search request).
+ *
+ * @param sc the search to reconnec
+ */
+static void
+try_reconnect (struct GNUNET_FS_SearchContext *sc)
+{
+  if (NULL != sc->mq)
+  {
+    GNUNET_MQ_destroy (sc->mq);
+    sc->mq = NULL;
+  }
+  sc->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (sc->reconnect_backoff);
+  sc->task =
+    GNUNET_SCHEDULER_add_delayed (sc->reconnect_backoff,
+                                  &do_reconnect,
+                                  sc);
+}
+
+
+/**
+ * Start search for content, internal API.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param uri specifies the search parameters; can be
+ *        a KSK URI or an SKS URI.
+ * @param anonymity desired level of anonymity
+ * @param options options for the search
+ * @param cctx initial value for the client context
+ * @param psearch parent search result (for namespace update searches)
+ * @return context that can be used to control the search
+ */
+static struct GNUNET_FS_SearchContext *
+search_start (struct GNUNET_FS_Handle *h,
+              const struct GNUNET_FS_Uri *uri,
+              uint32_t anonymity,
+              enum GNUNET_FS_SearchOptions options,
+              void *cctx,
+              struct GNUNET_FS_SearchResult *psearch)
+{
+  struct GNUNET_FS_SearchContext *sc;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  sc = GNUNET_new (struct GNUNET_FS_SearchContext);
+  sc->h = h;
+  sc->options = options;
+  sc->uri = GNUNET_FS_uri_dup (uri);
+  sc->anonymity = anonymity;
+  sc->start_time = GNUNET_TIME_absolute_get ();
+  if (NULL != psearch)
+  {
+    sc->psearch_result = psearch;
+    psearch->update_search = sc;
+  }
+  sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO);
+  sc->client_info = cctx;
+  if (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc))
+  {
+    GNUNET_FS_uri_destroy (sc->uri);
+    GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
+    GNUNET_free (sc);
+    return NULL;
+  }
+  GNUNET_FS_search_sync_ (sc);
+  pi.status = GNUNET_FS_STATUS_SEARCH_START;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  return sc;
+}
+
+
+/**
+ * Update the 'results' map for the individual keywords with the
+ * results from the 'global' result set.
+ *
+ * @param cls closure, the `struct GNUNET_FS_SearchContext *`
+ * @param key current key code
+ * @param value value in the hash map, the `struct GNUNET_FS_SearchResult *`
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+update_sre_result_maps (void *cls,
+                        const struct GNUNET_HashCode *key,
+                        void *value)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  for (unsigned int i = 0; i < sc->uri->data.ksk.keywordCount; i++)
+  {
+    if (0 != (sr->keyword_bitmap[i / 8] & (1 << (i % 8))))
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_CONTAINER_multihashmap_put (sc->requests[i].results,
+                                                       &sr->key,
+                                                       sr,
+                                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  }
+  return GNUNET_YES;
+}
+
+
+int
+GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc)
+{
+  unsigned int i;
+  const char *keyword;
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
+  struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+  struct SearchRequestEntry *sre;
+
+  GNUNET_assert (NULL == sc->mq);
+  if (GNUNET_FS_uri_test_ksk (sc->uri))
+  {
+    GNUNET_assert (0 != sc->uri->data.ksk.keywordCount);
+    anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
+    GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub);
+    sc->requests
+      = GNUNET_new_array (sc->uri->data.ksk.keywordCount,
+                          struct SearchRequestEntry);
+
+    for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
+    {
+      keyword = &sc->uri->data.ksk.keywords[i][1];
+      sre = &sc->requests[i];
+      sre->keyword = GNUNET_strdup (keyword);
+      GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub,
+                                             keyword,
+                                             "fs-ublock",
+                                             &sre->dpub);
+      GNUNET_CRYPTO_hash (&sre->dpub,
+                          sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey),
+                          &sre->uquery);
+      sre->mandatory = (sc->uri->data.ksk.keywords[i][0] == '+');
+      if (sre->mandatory)
+        sc->mandatory_count++;
+      sre->results = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_NO);
+    }
+    GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                           &update_sre_result_maps,
+                                           sc);
+  }
+  GNUNET_assert (NULL == sc->task);
+  do_reconnect (sc);
+  if (NULL == sc->mq)
+  {
+    GNUNET_SCHEDULER_cancel (sc->task);
+    sc->task = NULL;
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Freeze probes for the given search result.
+ *
+ * @param cls the global FS handle
+ * @param key the key for the search result (unused)
+ * @param value the search result to free
+ * @return #GNUNET_OK
+ */
+static int
+search_result_freeze_probes (void *cls,
+                             const struct GNUNET_HashCode *key,
+                             void *value)
+{
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (NULL != sr->probe_ctx)
+  {
+    GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
+    sr->probe_ctx = NULL;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
+  }
+  if (NULL != sr->probe_cancel_task)
+  {
+    GNUNET_SCHEDULER_cancel (sr->probe_cancel_task);
+    sr->probe_cancel_task = NULL;
+  }
+  if (NULL != sr->update_search)
+    GNUNET_FS_search_pause (sr->update_search);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Resume probes for the given search result.
+ *
+ * @param cls the global FS handle
+ * @param key the key for the search result (unused)
+ * @param value the search result to free
+ * @return #GNUNET_OK
+ */
+static int
+search_result_resume_probes (void *cls,
+                             const struct GNUNET_HashCode *key,
+                             void *value)
+{
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  GNUNET_FS_search_start_probe_ (sr);
+  if (NULL != sr->update_search)
+    GNUNET_FS_search_continue (sr->update_search);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Signal suspend and free the given search result.
+ *
+ * @param cls the global FS handle
+ * @param key the key for the search result (unused)
+ * @param value the search result to free
+ * @return #GNUNET_OK
+ */
+static int
+search_result_suspend (void *cls,
+                       const struct GNUNET_HashCode *key,
+                       void *value)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (NULL != sr->download)
+  {
+    GNUNET_FS_download_signal_suspend_ (sr->download);
+    sr->download = NULL;
+  }
+  if (NULL != sr->update_search)
+  {
+    GNUNET_FS_search_signal_suspend_ (sr->update_search);
+    sr->update_search = NULL;
+  }
+  GNUNET_FS_search_stop_probe_ (sr);
+  if (0 == sr->mandatory_missing)
+  {
+    /* client is aware of search result, notify about suspension event */
+    pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND;
+    pi.value.search.specifics.result_suspend.cctx = sr->client_info;
+    pi.value.search.specifics.result_suspend.meta = sr->meta;
+    pi.value.search.specifics.result_suspend.uri = sr->uri;
+    sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  }
+  GNUNET_break (NULL == sr->client_info);
+  GNUNET_free (sr->serialization);
+  GNUNET_FS_uri_destroy (sr->uri);
+  GNUNET_FS_meta_data_destroy (sr->meta);
+  GNUNET_free (sr->keyword_bitmap);
+  GNUNET_free (sr);
+  return GNUNET_OK;
+}
+
+
+void
+GNUNET_FS_search_signal_suspend_ (void *cls)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+  unsigned int i;
+
+  GNUNET_FS_end_top (sc->h, sc->top);
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &search_result_suspend, sc);
+  pi.status = GNUNET_FS_STATUS_SEARCH_SUSPEND;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  GNUNET_break (NULL == sc->client_info);
+  if (sc->task != NULL)
+  {
+    GNUNET_SCHEDULER_cancel (sc->task);
+    sc->task = NULL;
+  }
+  if (NULL != sc->mq)
+  {
+    GNUNET_MQ_destroy (sc->mq);
+    sc->mq = NULL;
+  }
+  GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
+  if (NULL != sc->requests)
+  {
+    GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri));
+    for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
+    {
+      GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results);
+      GNUNET_free (sc->requests[i].keyword);
+    }
+  }
+  GNUNET_free (sc->requests);
+  GNUNET_free (sc->emsg);
+  GNUNET_FS_uri_destroy (sc->uri);
+  GNUNET_free (sc->serialization);
+  GNUNET_free (sc);
+}
+
+
+/**
+ * Start search for content.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param uri specifies the search parameters; can be
+ *        a KSK URI or an SKS URI.
+ * @param anonymity desired level of anonymity
+ * @param options options for the search
+ * @param cctx initial value for the client context
+ * @return context that can be used to control the search
+ */
+struct GNUNET_FS_SearchContext *
+GNUNET_FS_search_start (struct GNUNET_FS_Handle *h,
+                        const struct GNUNET_FS_Uri *uri, uint32_t anonymity,
+                        enum GNUNET_FS_SearchOptions options, void *cctx)
+{
+  struct GNUNET_FS_SearchContext *ret;
+
+  ret = search_start (h, uri, anonymity, options, cctx, NULL);
+  if (NULL == ret)
+    return NULL;
+  ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, ret);
+  return ret;
+}
+
+
+/**
+ * Pause search.
+ *
+ * @param sc context for the search that should be paused
+ */
+void
+GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (NULL != sc->task)
+  {
+    GNUNET_SCHEDULER_cancel (sc->task);
+    sc->task = NULL;
+  }
+  if (NULL != sc->mq)
+  {
+    GNUNET_MQ_destroy (sc->mq);
+    sc->mq = NULL;
+  }
+  GNUNET_FS_search_sync_ (sc);
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &search_result_freeze_probes,
+                                         sc);
+  pi.status = GNUNET_FS_STATUS_SEARCH_PAUSED;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi,
+                                                   sc->h,
+                                                   sc);
+}
+
+
+/**
+ * Continue paused search.
+ *
+ * @param sc context for the search that should be resumed
+ */
+void
+GNUNET_FS_search_continue (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  GNUNET_assert (NULL == sc->mq);
+  GNUNET_assert (NULL == sc->task);
+  do_reconnect (sc);
+  GNUNET_FS_search_sync_ (sc);
+  pi.status = GNUNET_FS_STATUS_SEARCH_CONTINUED;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &search_result_resume_probes, sc);
+}
+
+
+/**
+ * Signal stop for the given search result.
+ *
+ * @param cls the global FS handle
+ * @param key the key for the search result (unused)
+ * @param value the search result to free
+ * @return #GNUNET_OK
+ */
+static int
+search_result_stop (void *cls,
+                    const struct GNUNET_HashCode *key,
+                    void *value)
+{
+  struct GNUNET_FS_SearchContext *sc = cls;
+  struct GNUNET_FS_SearchResult *sr = value;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  GNUNET_FS_search_stop_probe_ (sr);
+  if (NULL != sr->download)
+  {
+    sr->download->search = NULL;
+    sr->download->top
+      = GNUNET_FS_make_top (sr->download->h,
+                            &GNUNET_FS_download_signal_suspend_,
+                            sr->download);
+    if (NULL != sr->download->serialization)
+    {
+      GNUNET_FS_remove_sync_file_ (sc->h,
+                                   GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD,
+                                   sr->download->serialization);
+      GNUNET_free (sr->download->serialization);
+      sr->download->serialization = NULL;
+    }
+    pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT;
+    GNUNET_FS_download_make_status_ (&pi,
+                                     sr->download);
+    GNUNET_FS_download_sync_ (sr->download);
+    sr->download = NULL;
+  }
+  if (0 != sr->mandatory_missing)
+  {
+    /* client is unaware of search result as
+       it does not match required keywords */
+    GNUNET_break (NULL == sr->client_info);
+    return GNUNET_OK;
+  }
+  pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED;
+  pi.value.search.specifics.result_stopped.cctx = sr->client_info;
+  pi.value.search.specifics.result_stopped.meta = sr->meta;
+  pi.value.search.specifics.result_stopped.uri = sr->uri;
+  sr->client_info = GNUNET_FS_search_make_status_ (&pi, sr->h, sc);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Free the given search result.
+ *
+ * @param cls the global FS handle
+ * @param key the key for the search result (unused)
+ * @param value the search result to free
+ * @return #GNUNET_OK
+ */
+static int
+search_result_free (void *cls,
+                    const struct GNUNET_HashCode *key,
+                    void *value)
+{
+  struct GNUNET_FS_SearchResult *sr = value;
+
+  if (NULL != sr->update_search)
+  {
+    GNUNET_FS_search_stop (sr->update_search);
+    GNUNET_assert (NULL == sr->update_search);
+  }
+  GNUNET_break (NULL == sr->probe_ctx);
+  GNUNET_break (NULL == sr->probe_cancel_task);
+  GNUNET_break (NULL == sr->client_info);
+  GNUNET_free (sr->serialization);
+  GNUNET_FS_uri_destroy (sr->uri);
+  GNUNET_FS_meta_data_destroy (sr->meta);
+  GNUNET_free (sr->keyword_bitmap);
+  GNUNET_free (sr);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Stop search for content.
+ *
+ * @param sc context for the search that should be stopped
+ */
+void
+GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+  unsigned int i;
+
+  if (NULL != sc->top)
+    GNUNET_FS_end_top (sc->h, sc->top);
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &search_result_stop,
+                                         sc);
+  if (NULL != sc->psearch_result)
+    sc->psearch_result->update_search = NULL;
+  if (NULL != sc->serialization)
+  {
+    GNUNET_FS_remove_sync_file_ (sc->h,
+                                 (NULL != sc->psearch_result)
+                                 ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                 : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                 sc->serialization);
+    GNUNET_FS_remove_sync_dir_ (sc->h,
+                                (NULL != sc->psearch_result)
+                                ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                sc->serialization);
+    GNUNET_free (sc->serialization);
+  }
+  pi.status = GNUNET_FS_STATUS_SEARCH_STOPPED;
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi,
+                                                   sc->h,
+                                                   sc);
+  GNUNET_break (NULL == sc->client_info);
+  if (NULL != sc->task)
+  {
+    GNUNET_SCHEDULER_cancel (sc->task);
+    sc->task = NULL;
+  }
+  if (NULL != sc->mq)
+  {
+    GNUNET_MQ_destroy (sc->mq);
+    sc->mq = NULL;
+  }
+  GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                         &search_result_free, sc);
+  GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map);
+  if (NULL != sc->requests)
+  {
+    GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri));
+    for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
+    {
+      GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results);
+      GNUNET_free (sc->requests[i].keyword);
+    }
+  }
+  GNUNET_free (sc->requests);
+  GNUNET_free (sc->emsg);
+  GNUNET_FS_uri_destroy (sc->uri);
+  GNUNET_free (sc);
+}
+
+
+/* end of fs_search.c */
diff --git a/src/service/fs/fs_sharetree.c b/src/service/fs/fs_sharetree.c
new file mode 100644
index 000000000..6c246a3ad
--- /dev/null
+++ b/src/service/fs/fs_sharetree.c
@@ -0,0 +1,457 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2005-2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_sharetree.c
+ * @brief code to manipulate the 'struct GNUNET_FS_ShareTreeItem' tree
+ * @author LRN
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_scheduler_lib.h"
+#include <pthread.h>
+
+
+/**
+ * Entry for each unique keyword to track how often
+ * it occurred.  Contains the keyword and the counter.
+ */
+struct KeywordCounter
+{
+  /**
+   * This is a doubly-linked list
+   */
+  struct KeywordCounter *prev;
+
+  /**
+   * This is a doubly-linked list
+   */
+  struct KeywordCounter *next;
+
+  /**
+   * Keyword that was found.
+   */
+  const char *value;
+
+  /**
+   * How many files have this keyword?
+   */
+  unsigned int count;
+};
+
+
+/**
+ * Aggregate information we keep for meta data in each directory.
+ */
+struct MetaCounter
+{
+  /**
+   * This is a doubly-linked list
+   */
+  struct MetaCounter *prev;
+
+  /**
+   * This is a doubly-linked list
+   */
+  struct MetaCounter *next;
+
+  /**
+   * Name of the plugin that provided that piece of metadata
+   */
+  const char *plugin_name;
+
+  /**
+   * MIME-type of the metadata itself
+   */
+  const char *data_mime_type;
+
+  /**
+   * The actual meta data.
+   */
+  const char *data;
+
+  /**
+   * Number of bytes in 'data'.
+   */
+  size_t data_size;
+
+  /**
+   * Type of the data
+   */
+  enum EXTRACTOR_MetaType type;
+
+  /**
+   * Format of the data
+   */
+  enum EXTRACTOR_MetaFormat format;
+
+  /**
+   * How many files have meta entries matching this value?
+   * (type and format do not have to match).
+   */
+  unsigned int count;
+};
+
+
+/**
+ * A structure that forms a singly-linked list that serves as a stack
+ * for metadata-processing function.
+ */
+struct TrimContext
+{
+  /**
+   * Map from the hash over the keyword to an 'struct KeywordCounter *'
+   * counter that says how often this keyword was
+   * encountered in the current directory.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *keywordcounter;
+
+  /**
+   * Map from the hash over the metadata to an 'struct MetaCounter *'
+   * counter that says how often this metadata was
+   * encountered in the current directory.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *metacounter;
+
+  /**
+   * Position we are currently manipulating.
+   */
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  /**
+   * Number of times an item has to be found to be moved to the parent.
+   */
+  unsigned int move_threshold;
+};
+
+
+/**
+ * Add the given keyword to the keyword statistics tracker.
+ *
+ * @param cls the multihashmap we store the keyword counters in
+ * @param keyword the keyword to count
+ * @param is_mandatory ignored
+ * @return always GNUNET_OK
+ */
+static int
+add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory)
+{
+  struct GNUNET_CONTAINER_MultiHashMap *mcm = cls;
+  struct KeywordCounter *cnt;
+  struct GNUNET_HashCode hc;
+  size_t klen;
+
+  klen = strlen (keyword) + 1;
+  GNUNET_CRYPTO_hash (keyword, klen - 1, &hc);
+  cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc);
+  if (cnt == NULL)
+  {
+    cnt = GNUNET_malloc (sizeof(struct KeywordCounter) + klen);
+    cnt->value = (const char *) &cnt[1];
+    GNUNET_memcpy (&cnt[1], keyword, klen);
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_CONTAINER_multihashmap_put (mcm,
+                                                      &hc, cnt,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  }
+  cnt->count++;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called on each meta data item.  Increments the
+ * respective counter.
+ *
+ * @param cls the container multihashmap to update
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return 0 to continue extracting / iterating
+ */
+static int
+add_to_meta_counter (void *cls, const char *plugin_name,
+                     enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat
+                     format,
+                     const char *data_mime_type, const char *data, size_t
+                     data_len)
+{
+  struct GNUNET_CONTAINER_MultiHashMap *map = cls;
+  struct GNUNET_HashCode key;
+  struct MetaCounter *cnt;
+
+  GNUNET_CRYPTO_hash (data, data_len, &key);
+  cnt = GNUNET_CONTAINER_multihashmap_get (map, &key);
+  if (NULL == cnt)
+  {
+    cnt = GNUNET_new (struct MetaCounter);
+    cnt->data = data;
+    cnt->data_size = data_len;
+    cnt->plugin_name = plugin_name;
+    cnt->type = type;
+    cnt->format = format;
+    cnt->data_mime_type = data_mime_type;
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_CONTAINER_multihashmap_put (map,
+                                                      &key, cnt,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  }
+  cnt->count++;
+  return 0;
+}
+
+
+/**
+ * Remove keywords above the threshold.
+ *
+ * @param cls the 'struct TrimContext' with the pos to remove the keywords from
+ * @param keyword the keyword to check
+ * @param is_mandatory ignored
+ * @return always GNUNET_OK
+ */
+static int
+remove_high_frequency_keywords (void *cls, const char *keyword, int
+                                is_mandatory)
+{
+  struct TrimContext *tc = cls;
+  struct KeywordCounter *counter;
+  struct GNUNET_HashCode hc;
+  size_t klen;
+
+  klen = strlen (keyword) + 1;
+  GNUNET_CRYPTO_hash (keyword, klen - 1, &hc);
+  counter = GNUNET_CONTAINER_multihashmap_get (tc->keywordcounter, &hc);
+  GNUNET_assert (NULL != counter);
+  if (counter->count < tc->move_threshold)
+    return GNUNET_OK;
+  GNUNET_FS_uri_ksk_remove_keyword (tc->pos->ksk_uri,
+                                    counter->value);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Move "frequent" keywords over to the target ksk uri, free the
+ * counters.
+ *
+ * @param cls the 'struct TrimContext'
+ * @param key key of the entry
+ * @param value the 'struct KeywordCounter'
+ * @return GNUNET_YES (always)
+ */
+static int
+migrate_and_drop_keywords (void *cls, const struct GNUNET_HashCode *key,
+                           void *value)
+{
+  struct TrimContext *tc = cls;
+  struct KeywordCounter *counter = value;
+
+  if (counter->count >= tc->move_threshold)
+  {
+    if (NULL == tc->pos->ksk_uri)
+      tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1,
+                                                             &counter->value);
+    else
+      GNUNET_FS_uri_ksk_add_keyword (tc->pos->ksk_uri, counter->value,
+                                     GNUNET_NO);
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (tc->keywordcounter,
+                                                       key,
+                                                       counter));
+  GNUNET_free (counter);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Copy "frequent" metadata items over to the
+ * target metadata container, free the counters.
+ *
+ * @param cls the 'struct TrimContext'
+ * @param key key of the entry
+ * @param value the 'struct KeywordCounter'
+ * @return GNUNET_YES (always)
+ */
+static int
+migrate_and_drop_metadata (void *cls, const struct GNUNET_HashCode *key,
+                           void *value)
+{
+  struct TrimContext *tc = cls;
+  struct MetaCounter *counter = value;
+
+  if (counter->count >= tc->move_threshold)
+  {
+    if (NULL == tc->pos->meta)
+      tc->pos->meta = GNUNET_FS_meta_data_create ();
+    GNUNET_FS_meta_data_insert (tc->pos->meta,
+                                counter->plugin_name,
+                                counter->type,
+                                counter->format,
+                                counter->data_mime_type, counter->data,
+                                counter->data_size);
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (tc->metacounter,
+                                                       key,
+                                                       counter));
+  GNUNET_free (counter);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Process a share item tree, moving frequent keywords up and
+ * copying frequent metadata up.
+ *
+ * @param tc trim context with hash maps to use
+ * @param tree tree to trim
+ */
+static void
+share_tree_trim (struct TrimContext *tc,
+                 struct GNUNET_FS_ShareTreeItem *tree)
+{
+  struct GNUNET_FS_ShareTreeItem *pos;
+  unsigned int num_children;
+
+  /* first, trim all children */
+  num_children = 0;
+  for (pos = tree->children_head; NULL != pos; pos = pos->next)
+  {
+    share_tree_trim (tc, pos);
+    num_children++;
+  }
+
+  /* consider adding filename to directory meta data */
+  if (tree->is_directory == GNUNET_YES)
+  {
+    const char *user = getenv ("USER");
+    if ((user == NULL) ||
+        (0 != strncasecmp (user, tree->short_filename, strlen (user))))
+    {
+      /* only use filename if it doesn't match $USER */
+      if (NULL == tree->meta)
+        tree->meta = GNUNET_FS_meta_data_create ();
+      GNUNET_FS_meta_data_insert (tree->meta, "<libgnunetfs>",
+                                  EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", tree->short_filename,
+                                  strlen (tree->short_filename) + 1);
+    }
+  }
+
+  if (1 >= num_children)
+    return; /* nothing to trim */
+
+  /* now, count keywords and meta data in children */
+  for (pos = tree->children_head; NULL != pos; pos = pos->next)
+  {
+    if (NULL != pos->meta)
+      GNUNET_FS_meta_data_iterate (pos->meta, &add_to_meta_counter,
+                                   tc->metacounter);
+    if (NULL != pos->ksk_uri)
+      GNUNET_FS_uri_ksk_get_keywords (pos->ksk_uri, &add_to_keyword_counter,
+                                      tc->keywordcounter);
+  }
+
+  /* calculate threshold for moving keywords / meta data */
+  tc->move_threshold = 1 + (num_children / 2);
+
+  /* remove high-frequency keywords from children */
+  for (pos = tree->children_head; NULL != pos; pos = pos->next)
+  {
+    tc->pos = pos;
+    if (NULL != pos->ksk_uri)
+    {
+      struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup (pos->ksk_uri);
+      GNUNET_FS_uri_ksk_get_keywords (ksk_uri_copy,
+                                      &remove_high_frequency_keywords, tc);
+      GNUNET_FS_uri_destroy (ksk_uri_copy);
+    }
+  }
+
+  /* add high-frequency meta data and keywords to parent */
+  tc->pos = tree;
+  GNUNET_CONTAINER_multihashmap_iterate (tc->keywordcounter,
+                                         &migrate_and_drop_keywords,
+                                         tc);
+  GNUNET_CONTAINER_multihashmap_iterate (tc->metacounter,
+                                         &migrate_and_drop_metadata,
+                                         tc);
+}
+
+
+/**
+ * Process a share item tree, moving frequent keywords up and
+ * copying frequent metadata up.
+ *
+ * @param toplevel toplevel directory in the tree, returned by the scanner
+ */
+void
+GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel)
+{
+  struct TrimContext tc;
+
+  if (toplevel == NULL)
+    return;
+  tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
+  tc.metacounter = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
+  share_tree_trim (&tc, toplevel);
+  GNUNET_CONTAINER_multihashmap_destroy (tc.keywordcounter);
+  GNUNET_CONTAINER_multihashmap_destroy (tc.metacounter);
+}
+
+
+/**
+ * Release memory of a share item tree.
+ *
+ * @param toplevel toplevel of the tree to be freed
+ */
+void
+GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel)
+{
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  while (NULL != (pos = toplevel->children_head))
+    GNUNET_FS_share_tree_free (pos);
+  if (NULL != toplevel->parent)
+    GNUNET_CONTAINER_DLL_remove (toplevel->parent->children_head,
+                                 toplevel->parent->children_tail,
+                                 toplevel);
+  if (NULL != toplevel->meta)
+    GNUNET_FS_meta_data_destroy (toplevel->meta);
+  if (NULL != toplevel->ksk_uri)
+    GNUNET_FS_uri_destroy (toplevel->ksk_uri);
+  GNUNET_free (toplevel->filename);
+  GNUNET_free (toplevel->short_filename);
+  GNUNET_free (toplevel);
+}
+
+
+/* end fs_sharetree.c */
diff --git a/src/service/fs/fs_tree.c b/src/service/fs/fs_tree.c
new file mode 100644
index 000000000..65f589966
--- /dev/null
+++ b/src/service/fs/fs_tree.c
@@ -0,0 +1,452 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009-2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/fs_tree.c
+ * @brief Merkle-tree-ish-CHK file encoding for GNUnet
+ * @see http://gnunet.org/encoding.php3
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fs_tree.h"
+
+
+/**
+ * Context for an ECRS-based file encoder that computes
+ * the Merkle-ish-CHK tree.
+ */
+struct GNUNET_FS_TreeEncoder
+{
+  /**
+   * Global FS context.
+   */
+  struct GNUNET_FS_Handle *h;
+
+  /**
+   * Closure for all callbacks.
+   */
+  void *cls;
+
+  /**
+   * Function to call on encrypted blocks.
+   */
+  GNUNET_FS_TreeBlockProcessor proc;
+
+  /**
+   * Function to call with progress information.
+   */
+  GNUNET_FS_TreeProgressCallback progress;
+
+  /**
+   * Function to call to receive input data.
+   */
+  GNUNET_FS_DataReader reader;
+
+  /**
+   * Function to call once we're done with processing.
+   */
+  GNUNET_SCHEDULER_TaskCallback cont;
+
+  /**
+   * Set to an error message (if we had an error).
+   */
+  char *emsg;
+
+  /**
+   * Set to the URI (upon successful completion)
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Overall file size.
+   */
+  uint64_t size;
+
+  /**
+   * How far are we?
+   */
+  uint64_t publish_offset;
+
+  /**
+   * How deep are we?  Depth 0 is for the DBLOCKs.
+   */
+  unsigned int current_depth;
+
+  /**
+   * How deep is the tree? Always > 0.
+   */
+  unsigned int chk_tree_depth;
+
+  /**
+   * In-memory cache of the current CHK tree.
+   * This struct will contain the CHK values
+   * from the root to the currently processed
+   * node in the tree as identified by
+   * "current_depth" and "publish_offset".
+   * The "chktree" will be initially NULL,
+   * then allocated to a sufficient number of
+   * entries for the size of the file and
+   * finally freed once the upload is complete.
+   */
+  struct ContentHashKey *chk_tree;
+
+  /**
+   * Are we currently in 'GNUNET_FS_tree_encoder_next'?
+   * Flag used to prevent recursion.
+   */
+  int in_next;
+};
+
+
+/**
+ * Compute the depth of the CHK tree.
+ *
+ * @param flen file length for which to compute the depth
+ * @return depth of the tree, always > 0.  A depth of 1 means only a DBLOCK.
+ */
+unsigned int
+GNUNET_FS_compute_depth (uint64_t flen)
+{
+  unsigned int treeDepth;
+  uint64_t fl;
+
+  treeDepth = 1;
+  fl = DBLOCK_SIZE;
+  while (fl < flen)
+  {
+    treeDepth++;
+    if (fl * CHK_PER_INODE < fl)
+    {
+      /* integer overflow, this is a HUGE file... */
+      return treeDepth;
+    }
+    fl = fl * CHK_PER_INODE;
+  }
+  return treeDepth;
+}
+
+
+/**
+ * Calculate how many bytes of payload a block tree of the given
+ * depth MAY correspond to at most (this function ignores the fact that
+ * some blocks will only be present partially due to the total file
+ * size cutting some blocks off at the end).
+ *
+ * @param depth depth of the block.  depth==0 is a DBLOCK.
+ * @return number of bytes of payload a subtree of this depth may correspond to
+ */
+uint64_t
+GNUNET_FS_tree_compute_tree_size (unsigned int depth)
+{
+  uint64_t rsize;
+  unsigned int i;
+
+  rsize = DBLOCK_SIZE;
+  for (i = 0; i < depth; i++)
+    rsize *= CHK_PER_INODE;
+  return rsize;
+}
+
+
+/**
+ * Compute the size of the current IBLOCK.  The encoder is
+ * triggering the calculation of the size of an IBLOCK at the
+ * *end* (hence end_offset) of its construction.  The IBLOCK
+ * maybe a full or a partial IBLOCK, and this function is to
+ * calculate how long it should be.
+ *
+ * @param depth depth of the IBlock in the tree, 0 would be a DBLOCK,
+ *        must be > 0 (this function is for IBLOCKs only!)
+ * @param end_offset current offset in the payload (!) of the overall file,
+ *        must be > 0 (since this function is called at the
+ *        end of a block).
+ * @return size of the corresponding IBlock
+ */
+static uint16_t
+GNUNET_FS_tree_compute_iblock_size (unsigned int depth, uint64_t end_offset)
+{
+  unsigned int ret;
+  uint64_t mod;
+  uint64_t bds;
+
+  GNUNET_assert (depth > 0);
+  GNUNET_assert (end_offset > 0);
+  bds = GNUNET_FS_tree_compute_tree_size (depth);
+  mod = end_offset % bds;
+  if (0 == mod)
+  {
+    /* we were triggered at the end of a full block */
+    ret = CHK_PER_INODE;
+  }
+  else
+  {
+    /* we were triggered at the end of the file */
+    bds /= CHK_PER_INODE;
+    ret = mod / bds;
+    if (0 != mod % bds)
+      ret++;
+  }
+  return (uint16_t) (ret * sizeof(struct ContentHashKey));
+}
+
+
+size_t
+GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset,
+                                     unsigned int depth)
+{
+  size_t ret;
+  uint64_t rsize;
+  uint64_t epos;
+  unsigned int chks;
+
+  GNUNET_assert (fsize > 0);
+  GNUNET_assert (offset <= fsize);
+  if (depth == 0)
+  {
+    ret = DBLOCK_SIZE;
+    if ((offset + ret > fsize) || (offset + ret < offset))
+      ret = (size_t) (fsize - offset);
+    return ret;
+  }
+
+  rsize = GNUNET_FS_tree_compute_tree_size (depth - 1);
+  epos = offset + rsize * CHK_PER_INODE;
+  if ((epos < offset) || (epos > fsize))
+    epos = fsize;
+  /* round up when computing #CHKs in our IBlock */
+  chks = (epos - offset + rsize - 1) / rsize;
+  GNUNET_assert (chks <= CHK_PER_INODE);
+  return chks * sizeof(struct ContentHashKey);
+}
+
+
+/**
+ * Initialize a tree encoder.  This function will call @a proc and
+ * "progress" on each block in the tree.  Once all blocks have been
+ * processed, "cont" will be scheduled.  The @a reader will be called
+ * to obtain the (plaintext) blocks for the file.  Note that this
+ * function will not actually call @a proc.  The client must
+ * call #GNUNET_FS_tree_encoder_next to trigger encryption (and
+ * calling of @a proc) for the each block.
+ *
+ * @param h the global FS context
+ * @param size overall size of the file to encode
+ * @param cls closure for reader, proc, progress and cont
+ * @param reader function to call to read plaintext data
+ * @param proc function to call on each encrypted block
+ * @param progress function to call with progress information
+ * @param cont function to call when done
+ */
+struct GNUNET_FS_TreeEncoder *
+GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size,
+                               void *cls,
+                               GNUNET_FS_DataReader reader,
+                               GNUNET_FS_TreeBlockProcessor proc,
+                               GNUNET_FS_TreeProgressCallback progress,
+                               GNUNET_SCHEDULER_TaskCallback cont)
+{
+  struct GNUNET_FS_TreeEncoder *te;
+
+  te = GNUNET_new (struct GNUNET_FS_TreeEncoder);
+  te->h = h;
+  te->size = size;
+  te->cls = cls;
+  te->reader = reader;
+  te->proc = proc;
+  te->progress = progress;
+  te->cont = cont;
+  te->chk_tree_depth = GNUNET_FS_compute_depth (size);
+  te->chk_tree
+    = GNUNET_new_array (te->chk_tree_depth * CHK_PER_INODE,
+                        struct ContentHashKey);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Created tree encoder for file with %llu bytes and depth %u\n",
+              (unsigned long long) size,
+              te->chk_tree_depth);
+  return te;
+}
+
+
+/**
+ * Compute the offset of the CHK for the
+ * current block in the IBlock above.
+ *
+ * @param depth depth of the IBlock in the tree (aka overall
+ *               number of tree levels minus depth); 0 == DBlock
+ * @param end_offset current offset in the overall file,
+ *               at the *beginning* of the block for DBLOCKs (depth==0),
+ *               otherwise at the *end* of the block (exclusive)
+ * @return (array of CHKs') offset in the above IBlock
+ */
+static unsigned int
+compute_chk_offset (unsigned int depth, uint64_t end_offset)
+{
+  uint64_t bds;
+  unsigned int ret;
+
+  bds = GNUNET_FS_tree_compute_tree_size (depth);
+  if (depth > 0)
+    end_offset--;               /* round down since for depth > 0 offset is at the END of the block */
+  ret = end_offset / bds;
+  return ret % CHK_PER_INODE;
+}
+
+
+/**
+ * Encrypt the next block of the file (and call proc and progress
+ * accordingly; or of course "cont" if we have already completed
+ * encoding of the entire file).
+ *
+ * @param te tree encoder to use
+ */
+void
+GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te)
+{
+  struct ContentHashKey *mychk;
+  const void *pt_block;
+  uint16_t pt_size;
+  char iob[DBLOCK_SIZE];
+  char enc[DBLOCK_SIZE];
+  struct GNUNET_CRYPTO_SymmetricSessionKey sk;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  unsigned int off;
+
+  GNUNET_assert (GNUNET_NO == te->in_next);
+  te->in_next = GNUNET_YES;
+  if (te->chk_tree_depth == te->current_depth)
+  {
+    off = CHK_PER_INODE * (te->chk_tree_depth - 1);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TE done, reading CHK `%s' from %u\n",
+                GNUNET_h2s (&te->chk_tree[off].query), off);
+    te->uri = GNUNET_new (struct GNUNET_FS_Uri);
+    te->uri->type = GNUNET_FS_URI_CHK;
+    te->uri->data.chk.chk = te->chk_tree[off];
+    te->uri->data.chk.file_length = GNUNET_htonll (te->size);
+    te->in_next = GNUNET_NO;
+    te->cont (te->cls);
+    return;
+  }
+  if (0 == te->current_depth)
+  {
+    /* read DBLOCK */
+    pt_size = GNUNET_MIN (DBLOCK_SIZE, te->size - te->publish_offset);
+    if (pt_size !=
+        te->reader (te->cls, te->publish_offset, pt_size, iob, &te->emsg))
+    {
+      te->in_next = GNUNET_NO;
+      te->cont (te->cls);
+      return;
+    }
+    pt_block = iob;
+  }
+  else
+  {
+    pt_size =
+      GNUNET_FS_tree_compute_iblock_size (te->current_depth,
+                                          te->publish_offset);
+    pt_block = &te->chk_tree[(te->current_depth - 1) * CHK_PER_INODE];
+  }
+  off = compute_chk_offset (te->current_depth, te->publish_offset);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "TE is at offset %llu and depth %u with block size %u and target-CHK-offset %u\n",
+              (unsigned long long) te->publish_offset, te->current_depth,
+              (unsigned int) pt_size, (unsigned int) off);
+  mychk = &te->chk_tree[te->current_depth * CHK_PER_INODE + off];
+  GNUNET_CRYPTO_hash (pt_block, pt_size, &mychk->key);
+  GNUNET_CRYPTO_hash_to_aes_key (&mychk->key, &sk, &iv);
+  GNUNET_CRYPTO_symmetric_encrypt (pt_block, pt_size, &sk, &iv, enc);
+  GNUNET_CRYPTO_hash (enc, pt_size, &mychk->query);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "TE calculates query to be `%s', stored at %u\n",
+              GNUNET_h2s (&mychk->query),
+              te->current_depth * CHK_PER_INODE + off);
+  if (NULL != te->proc)
+    te->proc (te->cls, mychk, te->publish_offset, te->current_depth,
+              (0 ==
+               te->current_depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK :
+              GNUNET_BLOCK_TYPE_FS_IBLOCK, enc, pt_size);
+  if (NULL != te->progress)
+    te->progress (te->cls, te->publish_offset, pt_block, pt_size,
+                  te->current_depth);
+  if (0 == te->current_depth)
+  {
+    te->publish_offset += pt_size;
+    if ((te->publish_offset == te->size) ||
+        (0 == te->publish_offset % (CHK_PER_INODE * DBLOCK_SIZE)))
+      te->current_depth++;
+  }
+  else
+  {
+    if ((off == CHK_PER_INODE) || (te->publish_offset == te->size))
+      te->current_depth++;
+    else
+      te->current_depth = 0;
+  }
+  te->in_next = GNUNET_NO;
+}
+
+
+/**
+ * Get the resulting URI from the encoding.
+ *
+ * @param te the tree encoder to clean up
+ * @return uri set to the resulting URI (if encoding finished), NULL otherwise
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_tree_encoder_get_uri (struct GNUNET_FS_TreeEncoder *te)
+{
+  if (NULL != te->uri)
+    return GNUNET_FS_uri_dup (te->uri);
+  return NULL;
+}
+
+
+/**
+ * Clean up a tree encoder and return information
+ * about possible errors.
+ *
+ * @param te the tree encoder to clean up
+ * @param emsg set to an error message (if an error occurred
+ *        within the tree encoder; if this function is called
+ *        prior to completion and prior to an internal error,
+ *        both "*emsg" will be set to NULL).
+ */
+void
+GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te,
+                               char **emsg)
+{
+  if (NULL != te->reader)
+  {
+    (void) te->reader (te->cls, UINT64_MAX, 0, 0, NULL);
+    te->reader = NULL;
+  }
+  GNUNET_assert (GNUNET_NO == te->in_next);
+  if (NULL != te->uri)
+    GNUNET_FS_uri_destroy (te->uri);
+  if (emsg != NULL)
+    *emsg = te->emsg;
+  else
+    GNUNET_free (te->emsg);
+  GNUNET_free (te->chk_tree);
+  GNUNET_free (te);
+}
+
+
+/* end of fs_tree.c */
diff --git a/src/service/fs/fs_tree.h b/src/service/fs/fs_tree.h
new file mode 100644
index 000000000..1fb681d27
--- /dev/null
+++ b/src/service/fs/fs_tree.h
@@ -0,0 +1,217 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_tree.h
+ * @brief Merkle-tree-ish-CHK file encoding for GNUnet
+ * @see https://gnunet.org/encoding
+ * @author Krista Bennett
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - decide if this API should be made public (gnunet_fs_service.h)
+ *   or remain "internal" (but with exported symbols?)
+ */
+#ifndef GNUNET_FS_TREE_H
+#define GNUNET_FS_TREE_H
+
+#include "fs_api.h"
+
+/**
+ * Compute the depth of the CHK tree.
+ *
+ * @param flen file length for which to compute the depth
+ * @return depth of the tree, always > 0.  A depth of 1 means only a DBLOCK.
+ */
+unsigned int
+GNUNET_FS_compute_depth (uint64_t flen);
+
+
+/**
+ * Calculate how many bytes of payload a block tree of the given
+ * depth MAY correspond to at most (this function ignores the fact that
+ * some blocks will only be present partially due to the total file
+ * size cutting some blocks off at the end).
+ *
+ * @param depth depth of the block.  depth==0 is a DBLOCK.
+ * @return number of bytes of payload a subtree of this depth may correspond to
+ */
+uint64_t
+GNUNET_FS_tree_compute_tree_size (unsigned int depth);
+
+
+/**
+ * Compute how many bytes of data should be stored in
+ * the specified block.
+ *
+ * @param fsize overall file size, must be > 0.
+ * @param offset offset in the original data corresponding
+ *         to the beginning of the tree induced by the block;
+ *         must be < fsize
+ * @param depth depth of the node in the tree, 0 for DBLOCK
+ * @return number of bytes stored in this node
+ */
+size_t
+GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset,
+                                     unsigned int depth);
+
+
+/**
+ * Context for an ECRS-based file encoder that computes
+ * the Merkle-ish-CHK tree.
+ */
+struct GNUNET_FS_TreeEncoder;
+
+
+/**
+ * Function called asking for the current (encoded)
+ * block to be processed.  After processing the
+ * client should either call "GNUNET_FS_tree_encode_next"
+ * or (on error) "GNUNET_FS_tree_encode_finish".
+ *
+ * @param cls closure
+ * @param chk content hash key for the block
+ * @param offset offset of the block
+ * @param depth depth of the block, 0 for DBLOCKs
+ * @param type type of the block (IBLOCK or DBLOCK)
+ * @param block the (encrypted) block
+ * @param block_size size of block (in bytes)
+ */
+typedef void (*GNUNET_FS_TreeBlockProcessor) (void *cls,
+                                              const struct ContentHashKey *chk,
+                                              uint64_t offset,
+                                              unsigned int depth,
+                                              enum GNUNET_BLOCK_Type type,
+                                              const void *block,
+                                              uint16_t block_size);
+
+
+/**
+ * Function called with information about our
+ * progress in computing the tree encoding.
+ *
+ * @param cls closure
+ * @param offset where are we in the file
+ * @param pt_block plaintext of the currently processed block
+ * @param pt_size size of pt_block
+ * @param depth depth of the block in the tree, 0 for DBLOCKS
+ */
+typedef void (*GNUNET_FS_TreeProgressCallback) (void *cls, uint64_t offset,
+                                                const void *pt_block,
+                                                size_t pt_size,
+                                                unsigned int depth);
+
+
+/**
+ * Initialize a tree encoder.  This function will call "proc" and
+ * "progress" on each block in the tree.  Once all blocks have been
+ * processed, "cont" will be scheduled.  The "reader" will be called
+ * to obtain the (plaintext) blocks for the file.  Note that this
+ * function will actually never call "proc"; the "proc" function must
+ * be triggered by calling "GNUNET_FS_tree_encoder_next" to trigger
+ * encryption (and calling of "proc") for each block.
+ *
+ * @param h the global FS context
+ * @param size overall size of the file to encode
+ * @param cls closure for reader, proc, progress and cont
+ * @param reader function to call to read plaintext data
+ * @param proc function to call on each encrypted block
+ * @param progress function to call with progress information
+ * @param cont function to call when done
+ * @return tree encoder context
+ */
+struct GNUNET_FS_TreeEncoder *
+GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size,
+                               void *cls, GNUNET_FS_DataReader reader,
+                               GNUNET_FS_TreeBlockProcessor proc,
+                               GNUNET_FS_TreeProgressCallback progress,
+                               GNUNET_SCHEDULER_TaskCallback cont);
+
+
+/**
+ * Encrypt the next block of the file (and
+ * call proc and progress accordingly; or
+ * of course "cont" if we have already completed
+ * encoding of the entire file).
+ *
+ * @param te tree encoder to use
+ */
+void
+GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te);
+
+
+/**
+ * Get the resulting URI from the encoding.
+ *
+ * @param te the tree encoder to clean up
+ * @return uri set to the resulting URI (if encoding finished), NULL otherwise
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_tree_encoder_get_uri (struct GNUNET_FS_TreeEncoder *te);
+
+
+/**
+ * Clean up a tree encoder and return information
+ * about possible errors.
+ *
+ * @param te the tree encoder to clean up
+ * @param emsg set to an error message (if an error occurred
+ *        within the tree encoder; if this function is called
+ *        prior to completion and prior to an internal error,
+ *        both "*emsg" will be set to NULL).
+ */
+void
+GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te,
+                               char **emsg);
+
+
+#if 0
+/* the functions below will be needed for persistence
+   but are not yet implemented -- FIXME... */
+/**
+ * Get data that would be needed to resume
+ * the encoding later.
+ *
+ * @param te encoding to resume
+ * @param data set to the resume data
+ * @param size set to the size of the resume data
+ */
+void
+GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder *te,
+                                        void **data, size_t *size);
+
+
+/**
+ * Reset tree encoder to point previously
+ * obtained for resuming.
+ *
+ * @param te encoding to resume
+ * @param data the resume data
+ * @param size the size of the resume data
+ */
+void
+GNUNET_FS_tree_encoder_resume (struct GNUNET_FS_TreeEncoder *te,
+                               const void *data, size_t size);
+
+#endif
+
+#endif
+
+/* end of fs_tree.h */
diff --git a/src/service/fs/fs_unindex.c b/src/service/fs/fs_unindex.c
new file mode 100644
index 000000000..68ba667c4
--- /dev/null
+++ b/src/service/fs/fs_unindex.c
@@ -0,0 +1,902 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003--2013, 2016 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_unindex.c
+ * @author Krista Grothoff
+ * @author Christian Grothoff
+ * @brief Unindex file.
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_protocols.h"
+#include "fs_api.h"
+#include "fs_tree.h"
+#include "block_fs.h"
+#include "fs_publish_ublock.h"
+
+
+/**
+ * Function called by the tree encoder to obtain
+ * a block of plaintext data (for the lowest level
+ * of the tree).
+ *
+ * @param cls our publishing context
+ * @param offset identifies which block to get
+ * @param max (maximum) number of bytes to get; returning
+ *        fewer will also cause errors
+ * @param buf where to copy the plaintext buffer
+ * @param emsg location to store an error message (on error)
+ * @return number of bytes copied to buf, 0 on error
+ */
+static size_t
+unindex_reader (void *cls,
+                uint64_t offset,
+                size_t max,
+                void *buf,
+                char **emsg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  size_t pt_size;
+
+  pt_size = GNUNET_MIN (max, uc->file_size - offset);
+  if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET))
+  {
+    *emsg = GNUNET_strdup (_ ("Failed to find given position in file"));
+    return 0;
+  }
+  if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size))
+  {
+    *emsg = GNUNET_strdup (_ ("Failed to read file"));
+    return 0;
+  }
+  return pt_size;
+}
+
+
+/**
+ * Fill in all of the generic fields for
+ * an unindex event and call the callback.
+ *
+ * @param pi structure to fill in
+ * @param uc overall unindex context
+ * @param offset where we are in the file (for progress)
+ */
+void
+GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi,
+                                struct GNUNET_FS_UnindexContext *uc,
+                                uint64_t offset)
+{
+  pi->value.unindex.uc = uc;
+  pi->value.unindex.cctx = uc->client_info;
+  pi->value.unindex.filename = uc->filename;
+  pi->value.unindex.size = uc->file_size;
+  pi->value.unindex.eta =
+    GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size);
+  pi->value.unindex.duration =
+    GNUNET_TIME_absolute_get_duration (uc->start_time);
+  pi->value.unindex.completed = offset;
+  pi->fsh = uc->h;
+  uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi);
+}
+
+
+/**
+ * Function called with information about our
+ * progress in computing the tree encoding.
+ *
+ * @param cls closure
+ * @param offset where are we in the file
+ * @param pt_block plaintext of the currently processed block
+ * @param pt_size size of pt_block
+ * @param depth depth of the block in the tree, 0 for DBLOCK
+ */
+static void
+unindex_progress (void *cls,
+                  uint64_t offset,
+                  const void *pt_block,
+                  size_t pt_size,
+                  unsigned int depth)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS;
+  pi.value.unindex.specifics.progress.data = pt_block;
+  pi.value.unindex.specifics.progress.offset = offset;
+  pi.value.unindex.specifics.progress.data_len = pt_size;
+  pi.value.unindex.specifics.progress.depth = depth;
+  GNUNET_FS_unindex_make_status_ (&pi, uc, offset);
+}
+
+
+/**
+ * We've encountered an error during
+ * unindexing.  Signal the client.
+ *
+ * @param uc context for the failed unindexing operation
+ */
+static void
+signal_unindex_error (struct GNUNET_FS_UnindexContext *uc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+  pi.value.unindex.specifics.error.message = uc->emsg;
+  GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
+}
+
+
+/**
+ * Continuation called to notify client about result of the
+ * datastore removal operation.
+ *
+ * @param cls closure
+ * @param success #GNUNET_SYSERR on failure
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+process_cont (void *cls,
+              int success,
+              struct GNUNET_TIME_Absolute min_expiration,
+              const char *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  if (success == GNUNET_SYSERR)
+  {
+    uc->emsg = GNUNET_strdup (msg);
+    signal_unindex_error (uc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Datastore REMOVE operation succeeded\n");
+  GNUNET_FS_tree_encoder_next (uc->tc);
+}
+
+
+/**
+ * Function called asking for the current (encoded)
+ * block to be processed.  After processing the
+ * client should either call "GNUNET_FS_tree_encode_next"
+ * or (on error) "GNUNET_FS_tree_encode_finish".
+ *
+ * @param cls closure
+ * @param chk content hash key for the block (key for lookup in the datastore)
+ * @param offset offset of the block
+ * @param depth depth of the block, 0 for DBLOCK
+ * @param type type of the block (IBLOCK or DBLOCK)
+ * @param block the (encrypted) block
+ * @param block_size size of block (in bytes)
+ */
+static void
+unindex_process (void *cls,
+                 const struct ContentHashKey *chk,
+                 uint64_t offset,
+                 unsigned int depth,
+                 enum GNUNET_BLOCK_Type type,
+                 const void *block,
+                 uint16_t block_size)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  uint32_t size;
+  const void *data;
+  struct OnDemandBlock odb;
+
+  if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK)
+  {
+    size = block_size;
+    data = block;
+  }
+  else                          /* on-demand encoded DBLOCK */
+  {
+    size = sizeof(struct OnDemandBlock);
+    odb.offset = GNUNET_htonll (offset);
+    odb.file_id = uc->file_id;
+    data = &odb;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending REMOVE request to DATASTORE service\n");
+  GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1,
+                           &process_cont, uc);
+  uc->chk = *chk;
+}
+
+
+/**
+ * Function called with the response from the FS service to our
+ * unindexing request.
+ *
+ * @param cls closure, unindex context
+ * @param msg the response
+ */
+static void
+handle_unindex_response (void *cls,
+                         const struct GNUNET_MessageHeader *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (NULL != uc->mq)
+  {
+    GNUNET_MQ_destroy (uc->mq);
+    uc->mq = NULL;
+  }
+  uc->state = UNINDEX_STATE_COMPLETE;
+  pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_make_status_ (&pi,
+                                  uc,
+                                  uc->file_size);
+}
+
+
+/**
+ * Generic error handler, called with the appropriate error code and
+ * the same closure specified at the creation of the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with the `struct GNUNET_FS_UnindexContext *`
+ * @param error error code
+ */
+static void
+unindex_mq_error_handler (void *cls,
+                          enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  if (NULL != uc->mq)
+  {
+    GNUNET_MQ_destroy (uc->mq);
+    uc->mq = NULL;
+  }
+  uc->state = UNINDEX_STATE_ERROR;
+  uc->emsg = GNUNET_strdup (_ ("Error communicating with `fs' service."));
+  GNUNET_FS_unindex_sync_ (uc);
+  signal_unindex_error (uc);
+}
+
+
+/**
+ * Function called when we are done with removing UBlocks.
+ * Disconnect from datastore and notify FS service about
+ * the unindex event.
+ *
+ * @param uc our unindexing context
+ */
+static void
+unindex_finish (struct GNUNET_FS_UnindexContext *uc)
+{
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (unindex_response,
+                             GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK,
+                             struct GNUNET_MessageHeader,
+                             uc),
+    GNUNET_MQ_handler_end ()
+  };
+  char *emsg;
+  struct GNUNET_MQ_Envelope *env;
+  struct UnindexMessage *req;
+
+  /* generate final progress message */
+  unindex_progress (uc,
+                    uc->file_size,
+                    NULL,
+                    0,
+                    0);
+  GNUNET_FS_tree_encoder_finish (uc->tc,
+                                 &emsg);
+  uc->tc = NULL;
+  GNUNET_DISK_file_close (uc->fh);
+  uc->fh = NULL;
+  GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+  uc->dsh = NULL;
+  uc->state = UNINDEX_STATE_FS_NOTIFY;
+  GNUNET_FS_unindex_sync_ (uc);
+  uc->mq = GNUNET_CLIENT_connect (uc->h->cfg,
+                                  "fs",
+                                  handlers,
+                                  &unindex_mq_error_handler,
+                                  uc);
+  if (NULL == uc->mq)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg =
+      GNUNET_strdup (_ ("Failed to connect to FS service for unindexing."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending UNINDEX message to FS service\n");
+  env = GNUNET_MQ_msg (req,
+                       GNUNET_MESSAGE_TYPE_FS_UNINDEX);
+  req->reserved = 0;
+  req->file_id = uc->file_id;
+  GNUNET_MQ_send (uc->mq,
+                  env);
+}
+
+
+/**
+ * Function called by the directory scanner as we extract keywords
+ * that we will need to remove UBlocks.
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param filename which file we are making progress on
+ * @param is_directory #GNUNET_YES if this is a directory,
+ *                     #GNUNET_NO if this is a file
+ *                     #GNUNET_SYSERR if it is neither (or unknown)
+ * @param reason kind of progress we are making
+ */
+static void
+unindex_directory_scan_cb (void *cls,
+                           const char *filename,
+                           int is_directory,
+                           enum GNUNET_FS_DirScannerProgressUpdateReason reason)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  static struct GNUNET_FS_ShareTreeItem *directory_scan_result;
+
+  switch (reason)
+  {
+  case GNUNET_FS_DIRSCANNER_FINISHED:
+    directory_scan_result = GNUNET_FS_directory_scan_get_result (uc->dscan);
+    uc->dscan = NULL;
+    if (NULL != directory_scan_result->ksk_uri)
+    {
+      uc->ksk_uri = GNUNET_FS_uri_dup (directory_scan_result->ksk_uri);
+      uc->state = UNINDEX_STATE_DS_REMOVE_KBLOCKS;
+      GNUNET_FS_unindex_sync_ (uc);
+      GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+    }
+    else
+    {
+      uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan."));
+      GNUNET_FS_unindex_sync_ (uc);
+      unindex_finish (uc);
+    }
+    GNUNET_FS_share_tree_free (directory_scan_result);
+    break;
+
+  case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Internal error scanning `%s'.\n"),
+                uc->filename);
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+    uc->emsg = GNUNET_strdup (_ ("Failed to get KSKs from directory scan."));
+    GNUNET_FS_unindex_sync_ (uc);
+    unindex_finish (uc);
+    break;
+
+  default:
+    break;
+  }
+}
+
+
+/**
+ * If necessary, connect to the datastore and remove the UBlocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_extract_keywords_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  char *ex;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (uc->h->cfg, "FS", "EXTRACTORS",
+                                             &ex))
+    ex = NULL;
+  uc->dscan = GNUNET_FS_directory_scan_start (uc->filename,
+                                              GNUNET_NO, ex,
+                                              &unindex_directory_scan_cb,
+                                              uc);
+  GNUNET_free (ex);
+}
+
+
+/**
+ * Continuation called to notify client about result of the remove
+ * operation for the UBlock.
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
+ *                GNUNET_NO if content was already there
+ *                GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority content to be stored
+ *                by the datacache at this time, zero for unknown, forever if we have no
+ *                space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+continue_after_remove (void *cls,
+                       int32_t success,
+                       struct GNUNET_TIME_Absolute min_expiration,
+                       const char *msg)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->dqe = NULL;
+  if (success != GNUNET_YES)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to remove UBlock: %s\n"),
+                msg);
+  uc->ksk_offset++;
+  GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+}
+
+
+/**
+ * Function called from datastore with result from us looking for
+ * a UBlock.  There are four cases:
+ * 1) no result, means we move on to the next keyword
+ * 2) data hash is the same as an already seen data hash, means we move on to
+ *    next keyword
+ * 3) UBlock for a different CHK, means we keep looking for more
+ * 4) UBlock is for our CHK, means we remove the block and then move
+ *           on to the next keyword
+ *
+ * @param cls the 'struct GNUNET_FS_UnindexContext *'
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+process_kblock_for_unindex (void *cls,
+                            const struct GNUNET_HashCode *key,
+                            size_t size,
+                            const void *data,
+                            enum GNUNET_BLOCK_Type type,
+                            uint32_t priority,
+                            uint32_t anonymity,
+                            uint32_t replication,
+                            struct GNUNET_TIME_Absolute expiration,
+                            uint64_t uid)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  const struct UBlock *ub;
+  struct GNUNET_FS_Uri *chk_uri;
+  struct GNUNET_HashCode query;
+
+  uc->dqe = NULL;
+  if (NULL == data)
+  {
+    /* no result */
+    uc->ksk_offset++;
+    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
+    return;
+  }
+  GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
+  if (size < sizeof(struct UBlock))
+  {
+    GNUNET_break (0);
+    goto get_next;
+  }
+  ub = data;
+  GNUNET_CRYPTO_hash (&ub->verification_key,
+                      sizeof(ub->verification_key),
+                      &query);
+  if (0 != memcmp (&query,
+                   key,
+                   sizeof(struct GNUNET_HashCode)))
+  {
+    /* result does not match our keyword, skip */
+    goto get_next;
+  }
+  {
+    char pt[size - sizeof(struct UBlock)];
+    struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+    const char *keyword;
+
+    GNUNET_CRYPTO_ecdsa_key_get_public (
+      GNUNET_CRYPTO_ecdsa_key_get_anonymous (),
+      &anon_pub);
+    keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
+    GNUNET_FS_ublock_decrypt_ (&ub[1], size - sizeof(struct UBlock),
+                               &anon_pub,
+                               keyword,
+                               pt);
+    if (NULL == memchr (&pt[1], 0, sizeof(pt) - 1))
+    {
+      GNUNET_break_op (0);  /* malformed UBlock */
+      goto get_next;
+    }
+    chk_uri = GNUNET_FS_uri_parse (&pt[1], NULL);
+    if (NULL == chk_uri)
+    {
+      GNUNET_break_op (0);  /* malformed UBlock */
+      goto get_next;
+    }
+  }
+  if (0 != memcmp (&uc->chk,
+                   &chk_uri->data.chk.chk,
+                   sizeof(struct ContentHashKey)))
+  {
+    /* different CHK, ignore */
+    GNUNET_FS_uri_destroy (chk_uri);
+    goto get_next;
+  }
+  GNUNET_FS_uri_destroy (chk_uri);
+  /* matches! */
+  uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
+                                     key,
+                                     size,
+                                     data,
+                                     0 /* priority */,
+                                     1 /* queue size */,
+                                     &continue_after_remove,
+                                     uc);
+  return;
+get_next:
+  uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
+                                      uid + 1 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
+                                      1 /* queue size */,
+                                      &process_kblock_for_unindex,
+                                      uc);
+}
+
+
+/**
+ * If necessary, connect to the datastore and remove the KBlocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  const char *keyword;
+  const struct GNUNET_CRYPTO_EcdsaPrivateKey *anon;
+  struct GNUNET_CRYPTO_EcdsaPublicKey anon_pub;
+  struct GNUNET_CRYPTO_EcdsaPublicKey dpub;
+
+  if (NULL == uc->dsh)
+    uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
+  if (NULL == uc->dsh)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  if ((NULL == uc->ksk_uri) ||
+      (uc->ksk_offset >= uc->ksk_uri->data.ksk.keywordCount))
+  {
+    unindex_finish (uc);
+    return;
+  }
+  anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (anon,
+                                      &anon_pub);
+  keyword = &uc->ksk_uri->data.ksk.keywords[uc->ksk_offset][1];
+  GNUNET_CRYPTO_ecdsa_public_key_derive (&anon_pub,
+                                         keyword,
+                                         "fs-ublock",
+                                         &dpub);
+  GNUNET_CRYPTO_hash (&dpub,
+                      sizeof(dpub),
+                      &uc->uquery);
+  uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
+                                      0 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
+                                      1 /* queue size */,
+                                      &process_kblock_for_unindex,
+                                      uc);
+}
+
+
+/**
+ * Function called when the tree encoder has
+ * processed all blocks.  Clean up.
+ *
+ * @param cls our unindexing context
+ */
+static void
+unindex_extract_keywords (void *cls)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->state = UNINDEX_STATE_EXTRACT_KEYWORDS;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_do_extract_keywords_ (uc);
+}
+
+
+/**
+ * Connect to the datastore and remove the blocks.
+ *
+ * @param uc context for the unindex operation.
+ */
+void
+GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc)
+{
+  if (NULL == uc->dsh)
+    uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg);
+  if (NULL == uc->dsh)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_ ("Failed to connect to `datastore' service."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->fh =
+    GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ,
+                           GNUNET_DISK_PERM_NONE);
+  if (NULL == uc->fh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_ ("Failed to open file for unindexing."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->tc =
+    GNUNET_FS_tree_encoder_create (uc->h,
+                                   uc->file_size,
+                                   uc,
+                                   &unindex_reader,
+                                   &unindex_process,
+                                   &unindex_progress,
+                                   &unindex_extract_keywords);
+  GNUNET_FS_tree_encoder_next (uc->tc);
+}
+
+
+/**
+ * Function called once the hash of the file
+ * that is being unindexed has been computed.
+ *
+ * @param cls closure, unindex context
+ * @param file_id computed hash, NULL on error
+ */
+void
+GNUNET_FS_unindex_process_hash_ (void *cls,
+                                 const struct GNUNET_HashCode *file_id)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+
+  uc->fhc = NULL;
+  if (uc->state != UNINDEX_STATE_HASHING)
+  {
+    GNUNET_FS_unindex_stop (uc);
+    return;
+  }
+  if (file_id == NULL)
+  {
+    uc->state = UNINDEX_STATE_ERROR;
+    uc->emsg = GNUNET_strdup (_ ("Failed to compute hash of file."));
+    GNUNET_FS_unindex_sync_ (uc);
+    signal_unindex_error (uc);
+    return;
+  }
+  uc->file_id = *file_id;
+  uc->state = UNINDEX_STATE_DS_REMOVE;
+  GNUNET_FS_unindex_sync_ (uc);
+  GNUNET_FS_unindex_do_remove_ (uc);
+}
+
+
+/**
+ * Create SUSPEND event for the given unindex operation
+ * and then clean up our state (without stop signal).
+ *
+ * @param cls the `struct GNUNET_FS_UnindexContext` to signal for
+ */
+void
+GNUNET_FS_unindex_signal_suspend_ (void *cls)
+{
+  struct GNUNET_FS_UnindexContext *uc = cls;
+  struct GNUNET_FS_ProgressInfo pi;
+
+  /* FIXME: lots of duplication with unindex_stop here! */
+  if (uc->dscan != NULL)
+  {
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+  }
+  if (NULL != uc->dqe)
+  {
+    GNUNET_DATASTORE_cancel (uc->dqe);
+    uc->dqe = NULL;
+  }
+  if (uc->fhc != NULL)
+  {
+    GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
+    uc->fhc = NULL;
+  }
+  if (NULL != uc->ksk_uri)
+  {
+    GNUNET_FS_uri_destroy (uc->ksk_uri);
+    uc->ksk_uri = NULL;
+  }
+  if (NULL != uc->mq)
+  {
+    GNUNET_MQ_destroy (uc->mq);
+    uc->mq = NULL;
+  }
+  if (NULL != uc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+  }
+  if (NULL != uc->tc)
+  {
+    GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
+    uc->tc = NULL;
+  }
+  if (uc->fh != NULL)
+  {
+    GNUNET_DISK_file_close (uc->fh);
+    uc->fh = NULL;
+  }
+  GNUNET_FS_end_top (uc->h, uc->top);
+  pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND;
+  GNUNET_FS_unindex_make_status_ (&pi, uc,
+                                  (uc->state ==
+                                   UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
+  GNUNET_break (NULL == uc->client_info);
+  GNUNET_free (uc->filename);
+  GNUNET_free (uc->serialization);
+  GNUNET_free (uc->emsg);
+  GNUNET_free (uc);
+}
+
+
+/**
+ * Unindex a file.
+ *
+ * @param h handle to the file sharing subsystem
+ * @param filename file to unindex
+ * @param cctx initial value for the client context
+ * @return NULL on error, otherwise handle
+ */
+struct GNUNET_FS_UnindexContext *
+GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
+                         const char *filename,
+                         void *cctx)
+{
+  struct GNUNET_FS_UnindexContext *uc;
+  struct GNUNET_FS_ProgressInfo pi;
+  uint64_t size;
+
+  if (GNUNET_OK !=
+      GNUNET_DISK_file_size (filename,
+                             &size,
+                             GNUNET_YES,
+                             GNUNET_YES))
+    return NULL;
+  uc = GNUNET_new (struct GNUNET_FS_UnindexContext);
+  uc->h = h;
+  uc->filename = GNUNET_strdup (filename);
+  uc->start_time = GNUNET_TIME_absolute_get ();
+  uc->file_size = size;
+  uc->client_info = cctx;
+  GNUNET_FS_unindex_sync_ (uc);
+  pi.status = GNUNET_FS_STATUS_UNINDEX_START;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
+  GNUNET_FS_unindex_make_status_ (&pi, uc, 0);
+  uc->fhc =
+    GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
+                             filename,
+                             HASHING_BLOCKSIZE,
+                             &GNUNET_FS_unindex_process_hash_, uc);
+  uc->top = GNUNET_FS_make_top (h,
+                                &GNUNET_FS_unindex_signal_suspend_,
+                                uc);
+  return uc;
+}
+
+
+/**
+ * Clean up after completion of an unindex operation.
+ *
+ * @param uc handle
+ */
+void
+GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc)
+{
+  struct GNUNET_FS_ProgressInfo pi;
+
+  if (NULL != uc->dscan)
+  {
+    GNUNET_FS_directory_scan_abort (uc->dscan);
+    uc->dscan = NULL;
+  }
+  if (NULL != uc->dqe)
+  {
+    GNUNET_DATASTORE_cancel (uc->dqe);
+    uc->dqe = NULL;
+  }
+  if (NULL != uc->fhc)
+  {
+    GNUNET_CRYPTO_hash_file_cancel (uc->fhc);
+    uc->fhc = NULL;
+  }
+  if (NULL != uc->mq)
+  {
+    GNUNET_MQ_destroy (uc->mq);
+    uc->mq = NULL;
+  }
+  if (NULL != uc->dsh)
+  {
+    GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
+    uc->dsh = NULL;
+  }
+  if (NULL != uc->ksk_uri)
+  {
+    GNUNET_FS_uri_destroy (uc->ksk_uri);
+    uc->ksk_uri = NULL;
+  }
+  if (NULL != uc->tc)
+  {
+    GNUNET_FS_tree_encoder_finish (uc->tc, NULL);
+    uc->tc = NULL;
+  }
+  if (uc->fh != NULL)
+  {
+    GNUNET_DISK_file_close (uc->fh);
+    uc->fh = NULL;
+  }
+  GNUNET_FS_end_top (uc->h, uc->top);
+  if (uc->serialization != NULL)
+  {
+    GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX,
+                                 uc->serialization);
+    GNUNET_free (uc->serialization);
+    uc->serialization = NULL;
+  }
+  pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED;
+  pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO;
+  GNUNET_FS_unindex_make_status_ (&pi, uc,
+                                  (uc->state ==
+                                   UNINDEX_STATE_COMPLETE) ? uc->file_size : 0);
+  GNUNET_break (NULL == uc->client_info);
+  GNUNET_free (uc->emsg);
+  GNUNET_free (uc->filename);
+  GNUNET_free (uc);
+}
+
+
+/* end of fs_unindex.c */
diff --git a/src/service/fs/fs_uri.c b/src/service/fs/fs_uri.c
new file mode 100644
index 000000000..40ec76a03
--- /dev/null
+++ b/src/service/fs/fs_uri.c
@@ -0,0 +1,2062 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003--2014 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/fs_uri.c
+ * @brief Parses and produces uri strings.
+ * @author Igor Wronsky, Christian Grothoff
+ *
+ * GNUnet URIs are of the general form "gnunet://MODULE/IDENTIFIER".
+ * The specific structure of "IDENTIFIER" depends on the module and
+ * maybe differentiated into additional subcategories if applicable.
+ * This module only deals with fs identifiers (MODULE = "fs").
+ * <p>
+ *
+ * This module only parses URIs for the AFS module.  The FS URIs fall
+ * into four categories, "chk", "sks", "ksk" and "loc".  The first three
+ * categories were named in analogy (!) to Freenet, but they do NOT
+ * work in exactly the same way.  They are very similar from the user's
+ * point of view (unique file identifier, subspace, keyword), but the
+ * implementation is rather different in pretty much every detail.
+ * The concrete URI formats are:
+ *
+ * </p>
+ *
+ * <ul><li>
+ *
+ * First, there are URIs that identify a file.  They have the format
+ * "gnunet://fs/chk/HEX1.HEX2.SIZE".  These URIs can be used to
+ * download the file.  The description, filename, mime-type and other
+ * meta-data is NOT part of the file-URI since a URI uniquely
+ * identifies a resource (and the contents of the file would be the
+ * same even if it had a different description).
+ *
+ * </li><li>
+ *
+ * The second category identifies entries in a namespace.  The format
+ * is "gnunet://fs/sks/NAMESPACE/IDENTIFIER" where the namespace
+ * should be given in HEX.  Applications may allow using a nickname
+ * for the namespace if the nickname is not ambiguous.  The identifier
+ * can be either an ASCII sequence or a HEX-encoding.  If the
+ * identifier is in ASCII but the format is ambiguous and could denote
+ * a HEX-string a "/" is appended to indicate ASCII encoding.
+ *
+ * </li> <li>
+ *
+ * The third category identifies ordinary searches.  The format is
+ * "gnunet://fs/ksk/KEYWORD[+KEYWORD]*".  Using the "+" syntax
+ * it is possible to encode searches with the boolean "AND" operator.
+ * "+" is used since it indicates a commutative 'and' operation and
+ * is unlikely to be used in a keyword by itself.
+ *
+ * </li><li>
+ *
+ * The last category identifies a datum on a specific machine.  The
+ * format is "gnunet://fs/loc/HEX1.HEX2.SIZE.PEER.SIG.EXPTIME".  PEER is
+ * the BinName of the public key of the peer storing the datum.  The
+ * signature (SIG) certifies that this peer has this content.
+ * HEX1, HEX2 and SIZE correspond to a 'chk' URI.
+ *
+ * </li></ul>
+ *
+ * The encoding for hexadecimal values is defined in the hashing.c
+ * module in the gnunetutil library and discussed there.
+ *
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_signatures.h"
+#include "fs_api.h"
+#include <unitypes.h>
+#include <unicase.h>
+#include <uniconv.h>
+#include <unistr.h>
+#include <unistdio.h>
+
+
+int
+GNUNET_FS_uri_to_key (const struct GNUNET_FS_Uri *uri,
+                      struct GNUNET_HashCode *key)
+{
+  switch (uri->type)
+  {
+  case GNUNET_FS_URI_CHK:
+    *key = uri->data.chk.chk.query;
+    return GNUNET_OK;
+
+  case GNUNET_FS_URI_SKS:
+    GNUNET_CRYPTO_hash (uri->data.sks.identifier,
+                        strlen (uri->data.sks.identifier),
+                        key);
+    return GNUNET_OK;
+
+  case GNUNET_FS_URI_KSK:
+    if (uri->data.ksk.keywordCount > 0)
+    {
+      GNUNET_CRYPTO_hash (uri->data.ksk.keywords[0],
+                          strlen (uri->data.ksk.keywords[0]),
+                          key);
+      return GNUNET_OK;
+    }
+    else
+    {
+      memset (key, 0, sizeof(struct GNUNET_HashCode));
+      return GNUNET_SYSERR;
+    }
+    break;
+
+  case GNUNET_FS_URI_LOC:
+    GNUNET_CRYPTO_hash (&uri->data.loc.fi,
+                        sizeof(struct FileIdentifier)
+                        + sizeof(struct GNUNET_PeerIdentity),
+                        key);
+    return GNUNET_OK;
+
+  default:
+    memset (key, 0, sizeof(struct GNUNET_HashCode));
+    return GNUNET_SYSERR;
+  }
+}
+
+
+/**
+ * Convert keyword URI to a human readable format
+ * (i.e. the search query that was used in the first place)
+ *
+ * @param uri ksk uri to convert to a string
+ * @return string with the keywords
+ */
+char *
+GNUNET_FS_uri_ksk_to_string_fancy (const struct GNUNET_FS_Uri *uri)
+{
+  size_t n;
+  char *ret;
+  unsigned int i;
+  const char *keyword;
+  char **keywords;
+  unsigned int keywordCount;
+
+  if ((NULL == uri) || (GNUNET_FS_URI_KSK != uri->type))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  keywords = uri->data.ksk.keywords;
+  keywordCount = uri->data.ksk.keywordCount;
+  n = keywordCount + 1;
+  for (i = 0; i < keywordCount; i++)
+  {
+    keyword = keywords[i];
+    n += strlen (keyword) - 1;
+    if (NULL != strstr (&keyword[1], " "))
+      n += 2;
+    if (keyword[0] == '+')
+      n++;
+  }
+  ret = GNUNET_malloc (n);
+  strcpy (ret, "");
+  for (i = 0; i < keywordCount; i++)
+  {
+    keyword = keywords[i];
+    if (NULL != strstr (&keyword[1], " "))
+    {
+      strcat (ret, "\"");
+      if (keyword[0] == '+')
+        strcat (ret, keyword);
+      else
+        strcat (ret, &keyword[1]);
+      strcat (ret, "\"");
+    }
+    else
+    {
+      if (keyword[0] == '+')
+        strcat (ret, keyword);
+      else
+        strcat (ret, &keyword[1]);
+    }
+    strcat (ret, " ");
+  }
+  return ret;
+}
+
+
+/**
+ * Given a keyword with %-encoding (and possibly quotes to protect
+ * spaces), return a copy of the keyword without %-encoding and
+ * without double-quotes (%22).  Also, add a space at the beginning
+ * if there is not a '+'.
+ *
+ * @param in string with %-encoding
+ * @param emsg where to store the parser error message (if any)
+ * @return decoded string with leading space (or preserved plus)
+ */
+static char *
+percent_decode_keyword (const char *in, char **emsg)
+{
+  char *out;
+  char *ret;
+  unsigned int rpos;
+  unsigned int wpos;
+  unsigned int hx;
+
+  out = GNUNET_strdup (in);
+  rpos = 0;
+  wpos = 0;
+  while (out[rpos] != '\0')
+  {
+    if (out[rpos] == '%')
+    {
+      if (1 != sscanf (&out[rpos + 1], "%2X", &hx))
+      {
+        GNUNET_free (out);
+        *emsg = GNUNET_strdup (
+          _ (     /* xgettext:no-c-format */
+            "Malformed KSK URI (`%' must be followed by HEX number)"));
+        return NULL;
+      }
+      rpos += 3;
+      if (hx == '"')
+        continue;     /* skip double quote */
+      out[wpos++] = (char) hx;
+    }
+    else
+    {
+      out[wpos++] = out[rpos++];
+    }
+  }
+  out[wpos] = '\0';
+  if (out[0] == '+')
+  {
+    ret = GNUNET_strdup (out);
+  }
+  else
+  {
+    /* need to prefix with space */
+    ret = GNUNET_malloc (strlen (out) + 2);
+    strcpy (ret, " ");
+    strcat (ret, out);
+  }
+  GNUNET_free (out);
+  return ret;
+}
+
+
+#define GNUNET_FS_URI_KSK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_KSK_INFIX
+
+/**
+ * Parse a KSK URI.
+ *
+ * @param s an uri string
+ * @param emsg where to store the parser error message (if any)
+ * @return NULL on error, otherwise the KSK URI
+ */
+static struct GNUNET_FS_Uri *
+uri_ksk_parse (const char *s, char **emsg)
+{
+  struct GNUNET_FS_Uri *ret;
+  char **keywords;
+  unsigned int pos;
+  int max;
+  int iret;
+  int i;
+  size_t slen;
+  char *dup;
+  int saw_quote;
+
+  slen = strlen (s);
+  pos = strlen (GNUNET_FS_URI_KSK_PREFIX);
+  if ((slen <= pos) || (0 != strncmp (s, GNUNET_FS_URI_KSK_PREFIX, pos)))
+    return NULL; /* not KSK URI */
+  if ((s[slen - 1] == '+') || (s[pos] == '+'))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("Malformed KSK URI (must not begin or end with `+')"));
+    return NULL;
+  }
+  max = 1;
+  saw_quote = 0;
+  for (i = pos; i < slen; i++)
+  {
+    if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22")))
+    {
+      saw_quote = (saw_quote + 1) % 2;
+      i += 3;
+      continue;
+    }
+    if ((s[i] == '+') && (saw_quote == 0))
+    {
+      max++;
+      if (s[i - 1] == '+')
+      {
+        *emsg = GNUNET_strdup (_ ("Malformed KSK URI (`++' not allowed)"));
+        return NULL;
+      }
+    }
+  }
+  if (saw_quote == 1)
+  {
+    *emsg = GNUNET_strdup (_ ("Malformed KSK URI (quotes not balanced)"));
+    return NULL;
+  }
+  iret = max;
+  dup = GNUNET_strdup (s);
+  keywords = GNUNET_new_array (max, char *);
+  for (i = slen - 1; i >= (int) pos; i--)
+  {
+    if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22")))
+    {
+      saw_quote = (saw_quote + 1) % 2;
+      continue;
+    }
+    if ((dup[i] == '+') && (saw_quote == 0))
+    {
+      keywords[--max] = percent_decode_keyword (&dup[i + 1], emsg);
+      if (NULL == keywords[max])
+        goto CLEANUP;
+      dup[i] = '\0';
+    }
+  }
+  keywords[--max] = percent_decode_keyword (&dup[pos], emsg);
+  if (NULL == keywords[max])
+    goto CLEANUP;
+  GNUNET_assert (0 == max);
+  GNUNET_free (dup);
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_KSK;
+  ret->data.ksk.keywordCount = iret;
+  ret->data.ksk.keywords = keywords;
+  return ret;
+  CLEANUP:
+  for (i = 0; i < max; i++)
+    GNUNET_free (keywords[i]);
+  GNUNET_free (keywords);
+  GNUNET_free (dup);
+  return NULL;
+}
+
+
+#define GNUNET_FS_URI_SKS_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_SKS_INFIX
+
+/**
+ * Parse an SKS URI.
+ *
+ * @param s an uri string
+ * @param emsg where to store the parser error message (if any)
+ * @return NULL on error, SKS URI otherwise
+ */
+static struct GNUNET_FS_Uri *
+uri_sks_parse (const char *s, char **emsg)
+{
+  struct GNUNET_FS_Uri *ret;
+  struct GNUNET_CRYPTO_EcdsaPublicKey ns;
+  size_t pos;
+  char *end;
+
+  pos = strlen (GNUNET_FS_URI_SKS_PREFIX);
+  if ((strlen (s) <= pos) || (0 != strncmp (s, GNUNET_FS_URI_SKS_PREFIX, pos)))
+    return NULL; /* not an SKS URI */
+  end = strchr (&s[pos], '/');
+  if ((NULL == end) ||
+      (GNUNET_OK != GNUNET_STRINGS_string_to_data (&s[pos],
+                                                   end - &s[pos],
+                                                   &ns,
+                                                   sizeof(ns))))
+  {
+    *emsg = GNUNET_strdup (_ ("Malformed SKS URI (wrong syntax)"));
+    return NULL;   /* malformed */
+  }
+  end++; /* skip over '/' */
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_SKS;
+  ret->data.sks.ns = ns;
+  ret->data.sks.identifier = GNUNET_strdup (end);
+  return ret;
+}
+
+
+#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
+
+
+/**
+ * Parse a CHK URI.
+ *
+ * @param s an uri string
+ * @param emsg where to store the parser error message (if any)
+ * @return NULL on error, CHK URI otherwise
+ */
+static struct GNUNET_FS_Uri *
+uri_chk_parse (const char *s, char **emsg)
+{
+  struct GNUNET_FS_Uri *ret;
+  struct FileIdentifier fi;
+  unsigned int pos;
+  unsigned long long flen;
+  size_t slen;
+  char h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)];
+  char h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)];
+
+  slen = strlen (s);
+  pos = strlen (GNUNET_FS_URI_CHK_PREFIX);
+  if ((slen < pos + 2 * sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) ||
+      (0 != strncmp (s, GNUNET_FS_URI_CHK_PREFIX, pos)))
+    return NULL; /* not a CHK URI */
+  if ((s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') ||
+      (s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.'))
+  {
+    *emsg = GNUNET_strdup (_ ("Malformed CHK URI (wrong syntax)"));
+    return NULL;
+  }
+  GNUNET_memcpy (h1, &s[pos], sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded));
+  h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0';
+  GNUNET_memcpy (h2,
+                 &s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)],
+                 sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded));
+  h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0';
+
+  if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &fi.chk.key)) ||
+      (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &fi.chk.query)) ||
+      (1 !=
+       sscanf (&s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2],
+               "%llu",
+               &flen)))
+  {
+    *emsg = GNUNET_strdup (_ ("Malformed CHK URI (failed to decode CHK)"));
+    return NULL;
+  }
+  fi.file_length = GNUNET_htonll (flen);
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_CHK;
+  ret->data.chk = fi;
+  return ret;
+}
+
+
+GNUNET_NETWORK_STRUCT_BEGIN
+/**
+ * Structure that defines how the contents of a location URI must be
+ * assembled in memory to create or verify the signature of a location
+ * URI.
+ */
+struct LocUriAssembly
+{
+  /**
+   * What is being signed (rest of this struct).
+   */
+  struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
+
+  /**
+   * Expiration time of the offer.
+   */
+  struct GNUNET_TIME_AbsoluteNBO exptime;
+
+  /**
+   * File being offered.
+   */
+  struct FileIdentifier fi;
+
+  /**
+   * Peer offering the file.
+   */
+  struct GNUNET_PeerIdentity peer;
+};
+GNUNET_NETWORK_STRUCT_END
+
+
+#define GNUNET_FS_URI_LOC_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_LOC_INFIX
+
+#define SIGNATURE_ASCII_LENGTH 103
+
+/**
+ * Parse a LOC URI.
+ * Also verifies validity of the location URI.
+ *
+ * @param s an uri string
+ * @param emsg where to store the parser error message (if any)
+ * @return NULL on error, valid LOC URI otherwise
+ */
+static struct GNUNET_FS_Uri *
+uri_loc_parse (const char *s, char **emsg)
+{
+  struct GNUNET_FS_Uri *uri;
+  char h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)];
+  char h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)];
+  unsigned int pos;
+  unsigned int npos;
+  unsigned long long exptime;
+  unsigned long long flen;
+  struct GNUNET_TIME_Absolute et;
+  struct GNUNET_CRYPTO_EddsaSignature sig;
+  struct LocUriAssembly ass;
+  size_t slen;
+
+  slen = strlen (s);
+  pos = strlen (GNUNET_FS_URI_LOC_PREFIX);
+  if ((slen < pos + 2 * sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) ||
+      (0 != strncmp (s, GNUNET_FS_URI_LOC_PREFIX, pos)))
+    return NULL; /* not a LOC URI */
+  if ((s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') ||
+      (s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.'))
+  {
+    *emsg = GNUNET_strdup (_ ("LOC URI malformed (wrong syntax)"));
+    return NULL;
+  }
+  GNUNET_memcpy (h1, &s[pos], sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded));
+  h1[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0';
+  GNUNET_memcpy (h2,
+                 &s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded)],
+                 sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded));
+  h2[sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0';
+
+  if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &ass.fi.chk.key)) ||
+      (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &ass.fi.chk.query)) ||
+      (1 !=
+       sscanf (&s[pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2],
+               "%llu",
+               &flen)))
+  {
+    *emsg = GNUNET_strdup (_ ("LOC URI malformed (no CHK)"));
+    return NULL;
+  }
+  ass.fi.file_length = GNUNET_htonll (flen);
+
+  npos = pos + sizeof(struct GNUNET_CRYPTO_HashAsciiEncoded) * 2;
+  while ((s[npos] != '\0') && (s[npos] != '.'))
+    npos++;
+  if (s[npos] == '\0')
+  {
+    *emsg = GNUNET_strdup (_ ("LOC URI malformed (missing LOC)"));
+    goto ERR;
+  }
+  npos++;
+  if ((strlen (&s[npos]) <= GNUNET_CRYPTO_PKEY_ASCII_LENGTH + 1) ||
+      ('.' != s[npos + GNUNET_CRYPTO_PKEY_ASCII_LENGTH]))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (wrong syntax for public key)"));
+  }
+  if (
+    GNUNET_OK !=
+    GNUNET_CRYPTO_eddsa_public_key_from_string (&s[npos],
+                                                GNUNET_CRYPTO_PKEY_ASCII_LENGTH,
+                                                &ass.peer.public_key))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (could not decode public key)"));
+    goto ERR;
+  }
+  npos += GNUNET_CRYPTO_PKEY_ASCII_LENGTH;
+  if (s[npos++] != '.')
+  {
+    *emsg = GNUNET_strdup (_ ("LOC URI malformed (could not find signature)"));
+    goto ERR;
+  }
+  if ((strlen (&s[npos]) <= SIGNATURE_ASCII_LENGTH + 1) ||
+      ('.' != s[npos + SIGNATURE_ASCII_LENGTH]))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (wrong syntax for signature)"));
+    goto ERR;
+  }
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_string_to_data (&s[npos],
+                                     SIGNATURE_ASCII_LENGTH,
+                                     &sig,
+                                     sizeof(
+                                       struct GNUNET_CRYPTO_EddsaSignature)))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (could not decode signature)"));
+    goto ERR;
+  }
+  npos += SIGNATURE_ASCII_LENGTH;
+  if (s[npos++] != '.')
+  {
+    *emsg = GNUNET_strdup (
+      _ ("LOC URI malformed (wrong syntax for expiration time)"));
+    goto ERR;
+  }
+  if (1 != sscanf (&s[npos], "%llu", &exptime))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (could not parse expiration time)"));
+    goto ERR;
+  }
+  ass.purpose.size = htonl (sizeof(struct LocUriAssembly));
+  ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT);
+  et.abs_value_us = exptime * 1000LL * 1000LL;
+  ass.exptime = GNUNET_TIME_absolute_hton (et);
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_eddsa_verify (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT,
+                                  &ass,
+                                  &sig,
+                                  &ass.peer.public_key))
+  {
+    *emsg =
+      GNUNET_strdup (_ ("LOC URI malformed (signature failed validation)"));
+    goto ERR;
+  }
+  uri = GNUNET_new (struct GNUNET_FS_Uri);
+  uri->type = GNUNET_FS_URI_LOC;
+  uri->data.loc.fi = ass.fi;
+  uri->data.loc.peer = ass.peer;
+  uri->data.loc.expirationTime = et;
+  uri->data.loc.contentSignature = sig;
+
+  return uri;
+  ERR:
+  return NULL;
+}
+
+
+/**
+ * Convert a UTF-8 String to a URI.
+ *
+ * @param uri string to parse
+ * @param emsg where to store the parser error message (if any)
+ * @return NULL on error
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_parse (const char *uri, char **emsg)
+{
+  struct GNUNET_FS_Uri *ret;
+  char *msg;
+
+  if (NULL == uri)
+  {
+    GNUNET_break (0);
+    if (NULL != emsg)
+      *emsg = GNUNET_strdup (_ ("invalid argument"));
+    return NULL;
+  }
+  /**
+   * FIXME: Do we want to log this?
+   */
+  msg = NULL;
+  if (NULL != (ret = uri_chk_parse (uri, &msg)))
+    return ret;
+  GNUNET_free (msg);
+  if (NULL != (ret = uri_ksk_parse (uri, &msg)))
+    return ret;
+  GNUNET_free (msg);
+  if (NULL != (ret = uri_sks_parse (uri, &msg)))
+    return ret;
+  GNUNET_free (msg);
+  if (NULL != (ret = uri_loc_parse (uri, &msg)))
+    return ret;
+  GNUNET_free (msg);
+  if (NULL != emsg)
+    *emsg = GNUNET_strdup (_ ("Unrecognized URI type"));
+  return NULL;
+}
+
+
+/**
+ * Free URI.
+ *
+ * @param uri uri to free
+ */
+void
+GNUNET_FS_uri_destroy (struct GNUNET_FS_Uri *uri)
+{
+  unsigned int i;
+
+  switch (uri->type)
+  {
+  case GNUNET_FS_URI_KSK:
+    for (i = 0; i < uri->data.ksk.keywordCount; i++)
+      GNUNET_free (uri->data.ksk.keywords[i]);
+    GNUNET_array_grow (uri->data.ksk.keywords, uri->data.ksk.keywordCount, 0);
+    break;
+
+  case GNUNET_FS_URI_SKS:
+    GNUNET_free (uri->data.sks.identifier);
+    break;
+
+  case GNUNET_FS_URI_LOC:
+    break;
+
+  default:
+    /* do nothing */
+    break;
+  }
+  GNUNET_free (uri);
+}
+
+
+/**
+ * How many keywords are ANDed in this keyword URI?
+ *
+ * @param uri ksk uri to get the number of keywords from
+ * @return 0 if this is not a keyword URI
+ */
+unsigned int
+GNUNET_FS_uri_ksk_get_keyword_count (const struct GNUNET_FS_Uri *uri)
+{
+  if (uri->type != GNUNET_FS_URI_KSK)
+    return 0;
+  return uri->data.ksk.keywordCount;
+}
+
+
+int
+GNUNET_FS_uri_ksk_get_keywords (const struct GNUNET_FS_Uri *uri,
+                                GNUNET_FS_KeywordIterator iterator,
+                                void *iterator_cls)
+{
+  unsigned int i;
+  char *keyword;
+
+  if (uri->type != GNUNET_FS_URI_KSK)
+    return -1;
+  if (NULL == iterator)
+    return uri->data.ksk.keywordCount;
+  for (i = 0; i < uri->data.ksk.keywordCount; i++)
+  {
+    keyword = uri->data.ksk.keywords[i];
+    /* first character of keyword indicates
+     * if it is mandatory or not */
+    if (GNUNET_OK != iterator (iterator_cls, &keyword[1],(keyword[0] == '+') ))
+      return i;
+  }
+  return i;
+}
+
+
+/**
+ * Add the given keyword to the set of keywords represented by the URI.
+ * Does nothing if the keyword is already present.
+ *
+ * @param uri ksk uri to modify
+ * @param keyword keyword to add
+ * @param is_mandatory is this keyword mandatory?
+ */
+void
+GNUNET_FS_uri_ksk_add_keyword (struct GNUNET_FS_Uri *uri,
+                               const char *keyword,
+                               int is_mandatory)
+{
+  unsigned int i;
+  const char *old;
+  char *n;
+
+  GNUNET_assert (uri->type == GNUNET_FS_URI_KSK);
+  for (i = 0; i < uri->data.ksk.keywordCount; i++)
+  {
+    old = uri->data.ksk.keywords[i];
+    if (0 == strcmp (&old[1], keyword))
+      return;
+  }
+  GNUNET_asprintf (&n, is_mandatory ? "+%s" : " %s", keyword);
+  GNUNET_array_append (uri->data.ksk.keywords, uri->data.ksk.keywordCount, n);
+}
+
+
+/**
+ * Remove the given keyword from the set of keywords represented by the URI.
+ * Does nothing if the keyword is not present.
+ *
+ * @param uri ksk uri to modify
+ * @param keyword keyword to add
+ */
+void
+GNUNET_FS_uri_ksk_remove_keyword (struct GNUNET_FS_Uri *uri,
+                                  const char *keyword)
+{
+  unsigned int i;
+  char *old;
+
+  GNUNET_assert (uri->type == GNUNET_FS_URI_KSK);
+  for (i = 0; i < uri->data.ksk.keywordCount; i++)
+  {
+    old = uri->data.ksk.keywords[i];
+    if (0 == strcmp (&old[1], keyword))
+    {
+      uri->data.ksk.keywords[i] =
+        uri->data.ksk.keywords[uri->data.ksk.keywordCount - 1];
+      GNUNET_array_grow (uri->data.ksk.keywords,
+                         uri->data.ksk.keywordCount,
+                         uri->data.ksk.keywordCount - 1);
+      GNUNET_free (old);
+      return;
+    }
+  }
+}
+
+
+/**
+ * Obtain the identity of the peer offering the data
+ *
+ * @param uri the location URI to inspect
+ * @param peer where to store the identify of the peer (presumably) offering the content
+ * @return #GNUNET_SYSERR if this is not a location URI, otherwise #GNUNET_OK
+ */
+int
+GNUNET_FS_uri_loc_get_peer_identity (const struct GNUNET_FS_Uri *uri,
+                                     struct GNUNET_PeerIdentity *peer)
+{
+  if (uri->type != GNUNET_FS_URI_LOC)
+    return GNUNET_SYSERR;
+  *peer = uri->data.loc.peer;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Obtain the expiration of the LOC URI.
+ *
+ * @param uri location URI to get the expiration from
+ * @return expiration time of the URI
+ */
+struct GNUNET_TIME_Absolute
+GNUNET_FS_uri_loc_get_expiration (const struct GNUNET_FS_Uri *uri)
+{
+  GNUNET_assert (uri->type == GNUNET_FS_URI_LOC);
+  return uri->data.loc.expirationTime;
+}
+
+
+/**
+ * Obtain the URI of the content itself.
+ *
+ * @param uri location URI to get the content URI from
+ * @return NULL if argument is not a location URI
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_loc_get_uri (const struct GNUNET_FS_Uri *uri)
+{
+  struct GNUNET_FS_Uri *ret;
+
+  if (uri->type != GNUNET_FS_URI_LOC)
+    return NULL;
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_CHK;
+  ret->data.chk = uri->data.loc.fi;
+  return ret;
+}
+
+
+/**
+ * Construct a location URI (this peer will be used for the location).
+ * This function should only be called from within gnunet-service-fs,
+ * as it requires the peer's private key which is generally unavailable
+ * to processes directly under the user's control.  However, for
+ * testing and as it logically fits under URIs, it is in this API.
+ *
+ * @param base_uri content offered by the sender
+ * @param sign_key private key of the peer
+ * @param expiration_time how long will the content be offered?
+ * @return the location URI, NULL on error
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_loc_create (const struct GNUNET_FS_Uri *base_uri,
+                          const struct GNUNET_CRYPTO_EddsaPrivateKey *sign_key,
+                          struct GNUNET_TIME_Absolute expiration_time)
+{
+  struct GNUNET_FS_Uri *uri;
+  struct GNUNET_CRYPTO_EddsaPublicKey my_public_key;
+  struct LocUriAssembly ass;
+  struct GNUNET_TIME_Absolute et;
+
+  if (GNUNET_FS_URI_CHK != base_uri->type)
+    return NULL;
+  /* we round expiration time to full seconds for SKS URIs */
+  et.abs_value_us = (expiration_time.abs_value_us / 1000000LL) * 1000000LL;
+  GNUNET_CRYPTO_eddsa_key_get_public (sign_key, &my_public_key);
+  ass.purpose.size = htonl (sizeof(struct LocUriAssembly));
+  ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT);
+  ass.exptime = GNUNET_TIME_absolute_hton (et);
+  ass.fi = base_uri->data.chk;
+  ass.peer.public_key = my_public_key;
+  uri = GNUNET_new (struct GNUNET_FS_Uri);
+  uri->type = GNUNET_FS_URI_LOC;
+  uri->data.loc.fi = base_uri->data.chk;
+  uri->data.loc.expirationTime = et;
+  uri->data.loc.peer.public_key = my_public_key;
+  GNUNET_CRYPTO_eddsa_sign (sign_key,
+                            &ass,
+                            &uri->data.loc.contentSignature);
+  return uri;
+}
+
+
+/**
+ * Create an SKS URI from a namespace ID and an identifier.
+ *
+ * @param ns namespace ID
+ * @param id identifier
+ * @return an FS URI for the given namespace and identifier
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_sks_create (const struct GNUNET_CRYPTO_EcdsaPublicKey *ns,
+                          const char *id)
+{
+  struct GNUNET_FS_Uri *ns_uri;
+
+  ns_uri = GNUNET_new (struct GNUNET_FS_Uri);
+  ns_uri->type = GNUNET_FS_URI_SKS;
+  ns_uri->data.sks.ns = *ns;
+  ns_uri->data.sks.identifier = GNUNET_strdup (id);
+  return ns_uri;
+}
+
+
+/**
+ * Merge the sets of keywords from two KSK URIs.
+ * (useful for merging the canonicalized keywords with
+ * the original keywords for sharing).
+ *
+ * @param u1 first uri
+ * @param u2 second uri
+ * @return merged URI, NULL on error
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_ksk_merge (const struct GNUNET_FS_Uri *u1,
+                         const struct GNUNET_FS_Uri *u2)
+{
+  struct GNUNET_FS_Uri *ret;
+  unsigned int kc;
+  unsigned int i;
+  unsigned int j;
+  int found;
+  const char *kp;
+  char **kl;
+
+  if ((u1 == NULL) && (u2 == NULL))
+    return NULL;
+  if (u1 == NULL)
+    return GNUNET_FS_uri_dup (u2);
+  if (u2 == NULL)
+    return GNUNET_FS_uri_dup (u1);
+  if ((u1->type != GNUNET_FS_URI_KSK) || (u2->type != GNUNET_FS_URI_KSK))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  kc = u1->data.ksk.keywordCount;
+  kl = GNUNET_new_array (kc + u2->data.ksk.keywordCount, char *);
+  for (i = 0; i < u1->data.ksk.keywordCount; i++)
+    kl[i] = GNUNET_strdup (u1->data.ksk.keywords[i]);
+  for (i = 0; i < u2->data.ksk.keywordCount; i++)
+  {
+    kp = u2->data.ksk.keywords[i];
+    found = 0;
+    for (j = 0; j < u1->data.ksk.keywordCount; j++)
+      if (0 == strcmp (kp + 1, kl[j] + 1))
+      {
+        found = 1;
+        if (kp[0] == '+')
+          kl[j][0] = '+';
+        break;
+      }
+    if (0 == found)
+      kl[kc++] = GNUNET_strdup (kp);
+  }
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_KSK;
+  ret->data.ksk.keywordCount = kc;
+  ret->data.ksk.keywords = kl;
+  return ret;
+}
+
+
+/**
+ * Duplicate URI.
+ *
+ * @param uri the URI to duplicate
+ * @return copy of the URI
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_dup (const struct GNUNET_FS_Uri *uri)
+{
+  struct GNUNET_FS_Uri *ret;
+  unsigned int i;
+
+  if (uri == NULL)
+    return NULL;
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  GNUNET_memcpy (ret, uri, sizeof(struct GNUNET_FS_Uri));
+  switch (ret->type)
+  {
+  case GNUNET_FS_URI_KSK:
+    if (ret->data.ksk.keywordCount >=
+        GNUNET_MAX_MALLOC_CHECKED / sizeof(char *))
+    {
+      GNUNET_break (0);
+      GNUNET_free (ret);
+      return NULL;
+    }
+    if (ret->data.ksk.keywordCount > 0)
+    {
+      ret->data.ksk.keywords =
+        GNUNET_new_array (ret->data.ksk.keywordCount, char *);
+      for (i = 0; i < ret->data.ksk.keywordCount; i++)
+        ret->data.ksk.keywords[i] = GNUNET_strdup (uri->data.ksk.keywords[i]);
+    }
+    else
+      ret->data.ksk.keywords = NULL;   /* just to be sure */
+    break;
+
+  case GNUNET_FS_URI_SKS:
+    ret->data.sks.identifier = GNUNET_strdup (uri->data.sks.identifier);
+    break;
+
+  case GNUNET_FS_URI_LOC:
+    break;
+
+  default:
+    break;
+  }
+  return ret;
+}
+
+
+/**
+ * Create an FS URI from a single user-supplied string of keywords.
+ * The string is broken up at spaces into individual keywords.
+ * Keywords that start with "+" are mandatory.  Double-quotes can
+ * be used to prevent breaking up strings at spaces (and also
+ * to specify non-mandatory keywords starting with "+").
+ *
+ * Keywords must contain a balanced number of double quotes and
+ * double quotes can not be used in the actual keywords (for
+ * example, the string '""foo bar""' will be turned into two
+ * "OR"ed keywords 'foo' and 'bar', not into '"foo bar"'.
+ *
+ * @param keywords the keyword string
+ * @param emsg where to store an error message
+ * @return an FS URI for the given keywords, NULL
+ *  if keywords is not legal (i.e. empty).
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_ksk_create (const char *keywords, char **emsg)
+{
+  char **keywordarr;
+  unsigned int num_Words;
+  int inWord;
+  char *pos;
+  struct GNUNET_FS_Uri *uri;
+  char *searchString;
+  int saw_quote;
+
+  if (keywords == NULL)
+  {
+    *emsg = GNUNET_strdup (_ ("No keywords specified!\n"));
+    GNUNET_break (0);
+    return NULL;
+  }
+  searchString = GNUNET_strdup (keywords);
+  num_Words = 0;
+  inWord = 0;
+  saw_quote = 0;
+  pos = searchString;
+  while ('\0' != *pos)
+  {
+    if ((saw_quote == 0) && (isspace ((unsigned char) *pos)))
+    {
+      inWord = 0;
+    }
+    else if (0 == inWord)
+    {
+      inWord = 1;
+      ++num_Words;
+    }
+    if ('"' == *pos)
+      saw_quote = (saw_quote + 1) % 2;
+    pos++;
+  }
+  if (num_Words == 0)
+  {
+    GNUNET_free (searchString);
+    *emsg = GNUNET_strdup (_ ("No keywords specified!\n"));
+    return NULL;
+  }
+  if (saw_quote != 0)
+  {
+    GNUNET_free (searchString);
+    *emsg = GNUNET_strdup (_ ("Number of double-quotes not balanced!\n"));
+    return NULL;
+  }
+  keywordarr = GNUNET_new_array (num_Words, char *);
+  num_Words = 0;
+  inWord = 0;
+  pos = searchString;
+  while ('\0' != *pos)
+  {
+    if ((saw_quote == 0) && (isspace ((unsigned char) *pos)))
+    {
+      inWord = 0;
+      *pos = '\0';
+    }
+    else if (0 == inWord)
+    {
+      keywordarr[num_Words] = pos;
+      inWord = 1;
+      ++num_Words;
+    }
+    if ('"' == *pos)
+      saw_quote = (saw_quote + 1) % 2;
+    pos++;
+  }
+  uri =
+    GNUNET_FS_uri_ksk_create_from_args (num_Words, (const char **) keywordarr);
+  GNUNET_free (keywordarr);
+  GNUNET_free (searchString);
+  return uri;
+}
+
+
+/**
+ * Create an FS URI from a user-supplied command line of keywords.
+ * Arguments should start with "+" to indicate mandatory
+ * keywords.
+ *
+ * @param argc number of keywords
+ * @param argv keywords (double quotes are not required for
+ *             keywords containing spaces; however, double
+ *             quotes are required for keywords starting with
+ *             "+"); there is no mechanism for having double
+ *             quotes in the actual keywords (if the user
+ *             did specifically specify double quotes, the
+ *             caller should convert each double quote
+ *             into two single quotes).
+ * @return an FS URI for the given keywords, NULL
+ *  if keywords is not legal (i.e. empty).
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_ksk_create_from_args (unsigned int argc, const char **argv)
+{
+  unsigned int i;
+  struct GNUNET_FS_Uri *uri;
+  const char *keyword;
+  char *val;
+  const char *r;
+  char *w;
+  char *emsg;
+
+  if (argc == 0)
+    return NULL;
+  /* allow URI to be given as one and only keyword and
+   * handle accordingly */
+  emsg = NULL;
+  if ((argc == 1) && (strlen (argv[0]) > strlen (GNUNET_FS_URI_PREFIX)) &&
+      (0 == strncmp (argv[0],
+                     GNUNET_FS_URI_PREFIX,
+                     strlen (GNUNET_FS_URI_PREFIX))) &&
+      (NULL != (uri = GNUNET_FS_uri_parse (argv[0], &emsg))))
+    return uri;
+  GNUNET_free (emsg);
+  uri = GNUNET_new (struct GNUNET_FS_Uri);
+  uri->type = GNUNET_FS_URI_KSK;
+  uri->data.ksk.keywordCount = argc;
+  uri->data.ksk.keywords = GNUNET_new_array (argc, char *);
+  for (i = 0; i < argc; i++)
+  {
+    keyword = argv[i];
+    if (keyword[0] == '+')
+      val = GNUNET_strdup (keyword);
+    else
+      GNUNET_asprintf (&val, " %s", keyword);
+    r = val;
+    w = val;
+    while ('\0' != *r)
+    {
+      if ('"' == *r)
+        r++;
+      else
+        *(w++) = *(r++);
+    }
+    *w = '\0';
+    uri->data.ksk.keywords[i] = val;
+  }
+  return uri;
+}
+
+
+/**
+ * Test if two URIs are equal.
+ *
+ * @param u1 one of the URIs
+ * @param u2 the other URI
+ * @return #GNUNET_YES if the URIs are equal
+ */
+int
+GNUNET_FS_uri_test_equal (const struct GNUNET_FS_Uri *u1,
+                          const struct GNUNET_FS_Uri *u2)
+{
+  int ret;
+  unsigned int i;
+  unsigned int j;
+
+  GNUNET_assert (u1 != NULL);
+  GNUNET_assert (u2 != NULL);
+  if (u1->type != u2->type)
+    return GNUNET_NO;
+  switch (u1->type)
+  {
+  case GNUNET_FS_URI_CHK:
+    if (0 ==
+        memcmp (&u1->data.chk, &u2->data.chk, sizeof(struct FileIdentifier)))
+      return GNUNET_YES;
+    return GNUNET_NO;
+
+  case GNUNET_FS_URI_SKS:
+    if ((0 == memcmp (&u1->data.sks.ns,
+                      &u2->data.sks.ns,
+                      sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey))) &&
+        (0 == strcmp (u1->data.sks.identifier, u2->data.sks.identifier)))
+
+      return GNUNET_YES;
+    return GNUNET_NO;
+
+  case GNUNET_FS_URI_KSK:
+    if (u1->data.ksk.keywordCount != u2->data.ksk.keywordCount)
+      return GNUNET_NO;
+    for (i = 0; i < u1->data.ksk.keywordCount; i++)
+    {
+      ret = GNUNET_NO;
+      for (j = 0; j < u2->data.ksk.keywordCount; j++)
+      {
+        if (0 == strcmp (u1->data.ksk.keywords[i], u2->data.ksk.keywords[j]))
+        {
+          ret = GNUNET_YES;
+          break;
+        }
+      }
+      if (ret == GNUNET_NO)
+        return GNUNET_NO;
+    }
+    return GNUNET_YES;
+
+  case GNUNET_FS_URI_LOC:
+    if (memcmp (&u1->data.loc,
+                &u2->data.loc,
+                sizeof(struct FileIdentifier)
+                + sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey)
+                + sizeof(struct GNUNET_TIME_Absolute)
+                + sizeof(unsigned short) + sizeof(unsigned short)) != 0)
+      return GNUNET_NO;
+    return GNUNET_YES;
+
+  default:
+    return GNUNET_NO;
+  }
+}
+
+
+/**
+ * Is this a namespace URI?
+ *
+ * @param uri the uri to check
+ * @return #GNUNET_YES if this is an SKS uri
+ */
+int
+GNUNET_FS_uri_test_sks (const struct GNUNET_FS_Uri *uri)
+{
+  return uri->type == GNUNET_FS_URI_SKS;
+}
+
+
+/**
+ * Get the ID of a namespace from the given
+ * namespace URI.
+ *
+ * @param uri the uri to get the namespace ID from
+ * @param pseudonym where to store the ID of the namespace
+ * @return #GNUNET_OK on success
+ */
+int
+GNUNET_FS_uri_sks_get_namespace (const struct GNUNET_FS_Uri *uri,
+                                 struct GNUNET_CRYPTO_EcdsaPublicKey *pseudonym)
+{
+  if (! GNUNET_FS_uri_test_sks (uri))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  *pseudonym = uri->data.sks.ns;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Get the content identifier of an SKS URI.
+ *
+ * @param uri the sks uri
+ * @return NULL on error (not a valid SKS URI)
+ */
+char *
+GNUNET_FS_uri_sks_get_content_id (const struct GNUNET_FS_Uri *uri)
+{
+  if (! GNUNET_FS_uri_test_sks (uri))
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  return GNUNET_strdup (uri->data.sks.identifier);
+}
+
+
+/**
+ * Is this a keyword URI?
+ *
+ * @param uri the uri
+ * @return #GNUNET_YES if this is a KSK uri
+ */
+int
+GNUNET_FS_uri_test_ksk (const struct GNUNET_FS_Uri *uri)
+{
+#if EXTRA_CHECKS
+  unsigned int i;
+
+  if (uri->type == GNUNET_FS_URI_KSK)
+  {
+    for (i = 0; i < uri->data.ksk.keywordCount; i++)
+      GNUNET_assert (uri->data.ksk.keywords[i] != NULL);
+  }
+#endif
+  return uri->type == GNUNET_FS_URI_KSK;
+}
+
+
+/**
+ * Is this a file (or directory) URI?
+ *
+ * @param uri the uri to check
+ * @return #GNUNET_YES if this is a CHK uri
+ */
+int
+GNUNET_FS_uri_test_chk (const struct GNUNET_FS_Uri *uri)
+{
+  return uri->type == GNUNET_FS_URI_CHK;
+}
+
+
+/**
+ * What is the size of the file that this URI
+ * refers to?
+ *
+ * @param uri the CHK URI to inspect
+ * @return size of the file as specified in the CHK URI
+ */
+uint64_t
+GNUNET_FS_uri_chk_get_file_size (const struct GNUNET_FS_Uri *uri)
+{
+  switch (uri->type)
+  {
+  case GNUNET_FS_URI_CHK:
+    return GNUNET_ntohll (uri->data.chk.file_length);
+
+  case GNUNET_FS_URI_LOC:
+    return GNUNET_ntohll (uri->data.loc.fi.file_length);
+
+  default:
+    GNUNET_assert (0);
+  }
+  return 0; /* unreachable */
+}
+
+
+/**
+ * What is the hash of the original file's content
+ * that this URI refers to?
+ *
+ * @param uri the CHK URI to inspect
+ * @return hash of the file as specified in the CHK URI
+ */
+const struct GNUNET_HashCode*
+GNUNET_FS_uri_chk_get_file_hash (const struct GNUNET_FS_Uri *uri)
+{
+  if (GNUNET_FS_URI_CHK != uri->type)
+    GNUNET_assert (0);
+
+  return &(uri->data.chk.chk.key);
+}
+
+
+/**
+ * Is this a location URI?
+ *
+ * @param uri the uri to check
+ * @return #GNUNET_YES if this is a LOC uri
+ */
+int
+GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri)
+{
+  return uri->type == GNUNET_FS_URI_LOC;
+}
+
+
+/**
+ * Add a keyword as non-mandatory (with ' '-prefix) to the
+ * given keyword list at offset 'index'.  The array is
+ * guaranteed to be long enough.
+ *
+ * @param s keyword to add
+ * @param array array to add the keyword to
+ * @param index offset where to add the keyword
+ */
+static void
+insert_non_mandatory_keyword (const char *s, char **array, int index)
+{
+  char *nkword;
+
+  GNUNET_asprintf (&nkword,
+                   " %s", /* space to mark as 'non mandatory' */
+                   s);
+  array[index] = nkword;
+}
+
+
+/**
+ * Test if the given keyword @a s is already present in the
+ * given array, ignoring the '+'-mandatory prefix in the array.
+ *
+ * @param s keyword to test
+ * @param array keywords to test against, with ' ' or '+' prefix to ignore
+ * @param array_length length of the @a array
+ * @return #GNUNET_YES if the keyword exists, #GNUNET_NO if not
+ */
+static int
+find_duplicate (const char *s, const char **array, int array_length)
+{
+  int j;
+
+  for (j = array_length - 1; j >= 0; j--)
+    if (0 == strcmp (&array[j][1], s))
+      return GNUNET_YES;
+  return GNUNET_NO;
+}
+
+
+/**
+ * FIXME: comment
+ */
+static char *
+normalize_metadata (enum EXTRACTOR_MetaFormat format,
+                    const char *data,
+                    size_t data_len)
+{
+  uint8_t *free_str = NULL;
+  uint8_t *str_to_normalize = (uint8_t *) data;
+  uint8_t *normalized;
+  size_t r_len;
+
+  if (str_to_normalize == NULL)
+    return NULL;
+  /* Don't trust libextractor */
+  if (format == EXTRACTOR_METAFORMAT_UTF8)
+  {
+    free_str = (uint8_t *) u8_check ((const uint8_t *) data, data_len);
+    if (free_str == NULL)
+      free_str = NULL;
+    else
+      format = EXTRACTOR_METAFORMAT_C_STRING;
+  }
+  if (format == EXTRACTOR_METAFORMAT_C_STRING)
+  {
+    free_str = u8_strconv_from_encoding (data,
+                                         locale_charset (),
+                                         iconveh_escape_sequence);
+    if (free_str == NULL)
+      return NULL;
+  }
+
+  normalized = u8_tolower (str_to_normalize,
+                           strlen ((char *) str_to_normalize),
+                           NULL,
+                           UNINORM_NFD,
+                           NULL,
+                           &r_len);
+  /* free_str is allocated by libunistring internally, use free() */
+  if (free_str != NULL)
+    free (free_str);
+  if (normalized != NULL)
+  {
+    /* u8_tolower allocates a non-NULL-terminated string! */
+    free_str = GNUNET_malloc (r_len + 1);
+    GNUNET_memcpy (free_str, normalized, r_len);
+    free_str[r_len] = '\0';
+    free (normalized);
+    normalized = free_str;
+  }
+  return (char *) normalized;
+}
+
+
+/**
+ * Counts the number of UTF-8 characters (not bytes) in the string,
+ * returns that count.
+ */
+static size_t
+u8_strcount (const uint8_t *s)
+{
+  size_t count;
+  ucs4_t c;
+
+  GNUNET_assert (s != NULL);
+  if (s[0] == 0)
+    return 0;
+  for (count = 0; s != NULL; count++)
+    s = u8_next (&c, s);
+  return count - 1;
+}
+
+
+/**
+ * Break the filename up by matching [], () and {} pairs to make
+ * keywords. In case of nesting parentheses only the inner pair counts.
+ * You can't escape parentheses to scan something like "[blah\{foo]" to
+ * make a "blah{foo" keyword, this function is only a heuristic!
+ *
+ * @param s string to break down.
+ * @param array array to fill with enclosed tokens. If NULL, then tokens
+ *        are only counted.
+ * @param index index at which to start filling the array (entries prior
+ *        to it are used to check for duplicates). ignored if @a array == NULL.
+ * @return number of tokens counted (including duplicates), or number of
+ *         tokens extracted (excluding duplicates). 0 if there are no
+ *         matching parens in the string (when counting), or when all tokens
+ *         were duplicates (when extracting).
+ */
+static int
+get_keywords_from_parens (const char *s, char **array, int index)
+{
+  int count = 0;
+  char *open_paren;
+  char *close_paren;
+  char *ss;
+  char tmp;
+
+  if (NULL == s)
+    return 0;
+  ss = GNUNET_strdup (s);
+  open_paren = ss - 1;
+  while (NULL != (open_paren = strpbrk (open_paren + 1, "[{(")))
+  {
+    int match = 0;
+
+    close_paren = strpbrk (open_paren + 1, "]})");
+    if (NULL == close_paren)
+      continue;
+    switch (open_paren[0])
+    {
+    case '[':
+      if (']' == close_paren[0])
+        match = 1;
+      break;
+
+    case '{':
+      if ('}' == close_paren[0])
+        match = 1;
+      break;
+
+    case '(':
+      if (')' == close_paren[0])
+        match = 1;
+      break;
+
+    default:
+      break;
+    }
+    if (match && (close_paren - open_paren > 1))
+    {
+      tmp = close_paren[0];
+      close_paren[0] = '\0';
+      /* Keywords must be at least 3 characters long */
+      if (u8_strcount ((const uint8_t *) &open_paren[1]) <= 2)
+      {
+        close_paren[0] = tmp;
+        continue;
+      }
+      if (NULL != array)
+      {
+        char *normalized;
+        if (GNUNET_NO == find_duplicate ((const char *) &open_paren[1],
+                                         (const char **) array,
+                                         index + count))
+        {
+          insert_non_mandatory_keyword ((const char *) &open_paren[1],
+                                        array,
+                                        index + count);
+          count++;
+        }
+        normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8,
+                                         &open_paren[1],
+                                         close_paren - &open_paren[1]);
+        if (normalized != NULL)
+        {
+          if (GNUNET_NO == find_duplicate ((const char *) normalized,
+                                           (const char **) array,
+                                           index + count))
+          {
+            insert_non_mandatory_keyword ((const char *) normalized,
+                                          array,
+                                          index + count);
+            count++;
+          }
+          GNUNET_free (normalized);
+        }
+      }
+      else
+        count++;
+      close_paren[0] = tmp;
+    }
+  }
+  GNUNET_free (ss);
+  return count;
+}
+
+
+/**
+ * Where to break up keywords
+ */
+#define TOKENS "_. /-!?#&+@\"\'\\;:,()[]{}$<>|"
+
+/**
+ * Break the filename up by TOKENS to make
+ * keywords.
+ *
+ * @param s string to break down.
+ * @param array array to fill with tokens. If NULL, then tokens are only
+ *        counted.
+ * @param index index at which to start filling the array (entries prior
+ *        to it are used to check for duplicates). ignored if @a array == NULL.
+ * @return number of tokens (>1) counted (including duplicates), or number of
+ *         tokens extracted (excluding duplicates). 0 if there are no
+ *         separators in the string (when counting), or when all tokens were
+ *         duplicates (when extracting).
+ */
+static int
+get_keywords_from_tokens (const char *s, char **array, int index)
+{
+  char *p;
+  char *ss;
+  int seps = 0;
+
+  ss = GNUNET_strdup (s);
+  for (p = strtok (ss, TOKENS); p != NULL; p = strtok (NULL, TOKENS))
+  {
+    /* Keywords must be at least 3 characters long */
+    if (u8_strcount ((const uint8_t *) p) <= 2)
+      continue;
+    if (NULL != array)
+    {
+      char *normalized;
+      if (GNUNET_NO == find_duplicate (p, (const char **) array, index + seps))
+      {
+        insert_non_mandatory_keyword (p, array, index + seps);
+        seps++;
+      }
+      normalized =
+        normalize_metadata (EXTRACTOR_METAFORMAT_UTF8, p, strlen (p));
+      if (normalized != NULL)
+      {
+        if (GNUNET_NO == find_duplicate ((const char *) normalized,
+                                         (const char **) array,
+                                         index + seps))
+        {
+          insert_non_mandatory_keyword ((const char *) normalized,
+                                        array,
+                                        index + seps);
+          seps++;
+        }
+        GNUNET_free (normalized);
+      }
+    }
+    else
+      seps++;
+  }
+  GNUNET_free (ss);
+  return seps;
+}
+
+
+#undef TOKENS
+
+
+/**
+ * Function called on each value in the meta data.
+ * Adds it to the URI.
+ *
+ * @param cls URI to update
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in @a data
+ * @return 0 (always)
+ */
+static int
+gather_uri_data (void *cls,
+                 const char *plugin_name,
+                 enum EXTRACTOR_MetaType type,
+                 enum EXTRACTOR_MetaFormat format,
+                 const char *data_mime_type,
+                 const char *data,
+                 size_t data_len)
+{
+  struct GNUNET_FS_Uri *uri = cls;
+  char *normalized_data;
+  const char *sep;
+
+  if ((format != EXTRACTOR_METAFORMAT_UTF8) &&
+      (format != EXTRACTOR_METAFORMAT_C_STRING))
+    return 0;
+  /* Keywords must be at least 3 characters long
+   * If given non-utf8 string it will, most likely, find it to be invalid,
+   * and will return the length of its valid part, skipping the keyword.
+   * If it does - fix the extractor, not this check!
+   */if (u8_strcount ((const uint8_t *) data) <= 2)
+    return 0;
+  if ((EXTRACTOR_METATYPE_MIMETYPE == type) &&
+      (NULL != (sep = memchr (data, '/', data_len))) && (sep != data))
+  {
+    char *xtra;
+
+    GNUNET_asprintf (&xtra, "mimetype:%.*s", (int) (sep - data), data);
+    if (! find_duplicate (xtra,
+                          (const char **) uri->data.ksk.keywords,
+                          uri->data.ksk.keywordCount))
+    {
+      insert_non_mandatory_keyword (xtra,
+                                    uri->data.ksk.keywords,
+                                    uri->data.ksk.keywordCount);
+      uri->data.ksk.keywordCount++;
+    }
+    GNUNET_free (xtra);
+  }
+
+  normalized_data = normalize_metadata (format, data, data_len);
+  if (! find_duplicate (data,
+                        (const char **) uri->data.ksk.keywords,
+                        uri->data.ksk.keywordCount))
+  {
+    insert_non_mandatory_keyword (data,
+                                  uri->data.ksk.keywords,
+                                  uri->data.ksk.keywordCount);
+    uri->data.ksk.keywordCount++;
+  }
+  if (NULL != normalized_data)
+  {
+    if (! find_duplicate (normalized_data,
+                          (const char **) uri->data.ksk.keywords,
+                          uri->data.ksk.keywordCount))
+    {
+      insert_non_mandatory_keyword (normalized_data,
+                                    uri->data.ksk.keywords,
+                                    uri->data.ksk.keywordCount);
+      uri->data.ksk.keywordCount++;
+    }
+    GNUNET_free (normalized_data);
+  }
+  return 0;
+}
+
+
+/**
+ * Construct a keyword-URI from meta-data (take all entries
+ * in the meta-data and construct one large keyword URI
+ * that lists all keywords that can be found in the meta-data).
+ *
+ * @param md metadata to use
+ * @return NULL on error, otherwise a KSK URI
+ */
+struct GNUNET_FS_Uri *
+GNUNET_FS_uri_ksk_create_from_meta_data (
+  const struct GNUNET_FS_MetaData *md)
+{
+  struct GNUNET_FS_Uri *ret;
+  char *filename;
+  char *full_name = NULL;
+  char *ss;
+  int ent;
+  int tok_keywords = 0;
+  int paren_keywords = 0;
+
+  if (NULL == md)
+    return NULL;
+  ret = GNUNET_new (struct GNUNET_FS_Uri);
+  ret->type = GNUNET_FS_URI_KSK;
+  ent = GNUNET_FS_meta_data_iterate (md, NULL, NULL);
+  if (ent > 0)
+  {
+    full_name = GNUNET_FS_meta_data_get_first_by_types (
+      md,
+      EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
+      -1);
+    if (NULL != full_name)
+    {
+      filename = full_name;
+      while (NULL != (ss = strstr (filename, DIR_SEPARATOR_STR)))
+        filename = ss + 1;
+      tok_keywords = get_keywords_from_tokens (filename, NULL, 0);
+      paren_keywords = get_keywords_from_parens (filename, NULL, 0);
+    }
+    /* x3 because there might be a normalized variant of every keyword,
+       plus theoretically one more for mime... */
+    ret->data.ksk.keywords =
+      GNUNET_new_array ((ent + tok_keywords + paren_keywords) * 3, char *);
+    GNUNET_FS_meta_data_iterate (md, &gather_uri_data, ret);
+  }
+  if (tok_keywords > 0)
+    ret->data.ksk.keywordCount +=
+      get_keywords_from_tokens (filename,
+                                ret->data.ksk.keywords,
+                                ret->data.ksk.keywordCount);
+  if (paren_keywords > 0)
+    ret->data.ksk.keywordCount +=
+      get_keywords_from_parens (filename,
+                                ret->data.ksk.keywords,
+                                ret->data.ksk.keywordCount);
+  if (ent > 0)
+    GNUNET_free (full_name);
+  return ret;
+}
+
+
+/**
+ * In URI-encoding, does the given character
+ * need to be encoded using %-encoding?
+ */
+static int
+needs_percent (char c)
+{
+  return(! ((isalnum ((unsigned char) c)) || (c == '-') || (c == '_') ||
+            (c == '.') || (c == '~')));
+}
+
+
+/**
+ * Convert a KSK URI to a string.
+ *
+ * @param uri the URI to convert
+ * @return NULL on error (i.e. keywordCount == 0)
+ */
+static char *
+uri_ksk_to_string (const struct GNUNET_FS_Uri *uri)
+{
+  char **keywords;
+  unsigned int keywordCount;
+  size_t n;
+  char *ret;
+  unsigned int i;
+  unsigned int j;
+  unsigned int wpos;
+  size_t slen;
+  const char *keyword;
+
+  if (uri->type != GNUNET_FS_URI_KSK)
+    return NULL;
+  keywords = uri->data.ksk.keywords;
+  keywordCount = uri->data.ksk.keywordCount;
+  n = keywordCount + strlen (GNUNET_FS_URI_PREFIX)
+      + strlen (GNUNET_FS_URI_KSK_INFIX) + 1;
+  for (i = 0; i < keywordCount; i++)
+  {
+    keyword = keywords[i];
+    slen = strlen (keyword);
+    n += slen;
+    for (j = 0; j < slen; j++)
+    {
+      if ((j == 0) && (keyword[j] == ' '))
+      {
+        n--;
+        continue;       /* skip leading space */
+      }
+      if (needs_percent (keyword[j]))
+        n += 2;     /* will use %-encoding */
+    }
+  }
+  ret = GNUNET_malloc (n);
+  strcpy (ret, GNUNET_FS_URI_PREFIX);
+  strcat (ret, GNUNET_FS_URI_KSK_INFIX);
+  wpos = strlen (ret);
+  for (i = 0; i < keywordCount; i++)
+  {
+    keyword = keywords[i];
+    slen = strlen (keyword);
+    for (j = 0; j < slen; j++)
+    {
+      if ((j == 0) && (keyword[j] == ' '))
+        continue;     /* skip leading space */
+      if (needs_percent (keyword[j]))
+      {
+        sprintf (&ret[wpos], "%%%02X", (unsigned char) keyword[j]);
+        wpos += 3;
+      }
+      else
+      {
+        ret[wpos++] = keyword[j];
+      }
+    }
+    if (i != keywordCount - 1)
+      ret[wpos++] = '+';
+  }
+  return ret;
+}
+
+
+/**
+ * Convert SKS URI to a string.
+ *
+ * @param uri sks uri to convert
+ * @return NULL on error
+ */
+static char *
+uri_sks_to_string (const struct GNUNET_FS_Uri *uri)
+{
+  char *ret;
+  char buf[1024];
+
+  if (GNUNET_FS_URI_SKS != uri->type)
+    return NULL;
+  ret =
+    GNUNET_STRINGS_data_to_string (&uri->data.sks.ns,
+                                   sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey),
+                                   buf,
+                                   sizeof(buf));
+  GNUNET_assert (NULL != ret);
+  ret[0] = '\0';
+  GNUNET_asprintf (&ret,
+                   "%s%s%s/%s",
+                   GNUNET_FS_URI_PREFIX,
+                   GNUNET_FS_URI_SKS_INFIX,
+                   buf,
+                   uri->data.sks.identifier);
+  return ret;
+}
+
+
+/**
+ * Convert a CHK URI to a string.
+ *
+ * @param uri chk uri to convert
+ * @return NULL on error
+ */
+static char *
+uri_chk_to_string (const struct GNUNET_FS_Uri *uri)
+{
+  const struct FileIdentifier *fi;
+  char *ret;
+  struct GNUNET_CRYPTO_HashAsciiEncoded keyhash;
+  struct GNUNET_CRYPTO_HashAsciiEncoded queryhash;
+
+  if (uri->type != GNUNET_FS_URI_CHK)
+    return NULL;
+  fi = &uri->data.chk;
+  GNUNET_CRYPTO_hash_to_enc (&fi->chk.key, &keyhash);
+  GNUNET_CRYPTO_hash_to_enc (&fi->chk.query, &queryhash);
+
+  GNUNET_asprintf (&ret,
+                   "%s%s%s.%s.%llu",
+                   GNUNET_FS_URI_PREFIX,
+                   GNUNET_FS_URI_CHK_INFIX,
+                   (const char *) &keyhash,
+                   (const char *) &queryhash,
+                   (unsigned long long) GNUNET_ntohll (fi->file_length));
+  return ret;
+}
+
+
+/**
+ * Convert a LOC URI to a string.
+ *
+ * @param uri loc uri to convert
+ * @return NULL on error
+ */
+static char *
+uri_loc_to_string (const struct GNUNET_FS_Uri *uri)
+{
+  char *ret;
+  struct GNUNET_CRYPTO_HashAsciiEncoded keyhash;
+  struct GNUNET_CRYPTO_HashAsciiEncoded queryhash;
+  char *peer_id;
+  char peer_sig[SIGNATURE_ASCII_LENGTH + 1];
+
+  GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.key, &keyhash);
+  GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.query, &queryhash);
+  peer_id =
+    GNUNET_CRYPTO_eddsa_public_key_to_string (&uri->data.loc.peer.public_key);
+  GNUNET_assert (
+    NULL !=
+    GNUNET_STRINGS_data_to_string (&uri->data.loc.contentSignature,
+                                   sizeof(struct GNUNET_CRYPTO_EddsaSignature),
+                                   peer_sig,
+                                   sizeof(peer_sig)));
+  GNUNET_asprintf (&ret,
+                   "%s%s%s.%s.%llu.%s.%s.%llu",
+                   GNUNET_FS_URI_PREFIX,
+                   GNUNET_FS_URI_LOC_INFIX,
+                   (const char *) &keyhash,
+                   (const char *) &queryhash,
+                   (unsigned long long) GNUNET_ntohll (
+                     uri->data.loc.fi.file_length),
+                   peer_id,
+                   peer_sig,
+                   (unsigned long long)
+                   uri->data.loc.expirationTime.abs_value_us
+                   / 1000000LL);
+  GNUNET_free (peer_id);
+  return ret;
+}
+
+
+/**
+ * Convert a URI to a UTF-8 String.
+ *
+ * @param uri uri to convert to a string
+ * @return the UTF-8 string
+ */
+char *
+GNUNET_FS_uri_to_string (const struct GNUNET_FS_Uri *uri)
+{
+  if (uri == NULL)
+  {
+    GNUNET_break (0);
+    return NULL;
+  }
+  switch (uri->type)
+  {
+  case GNUNET_FS_URI_KSK:
+    return uri_ksk_to_string (uri);
+
+  case GNUNET_FS_URI_SKS:
+    return uri_sks_to_string (uri);
+
+  case GNUNET_FS_URI_CHK:
+    return uri_chk_to_string (uri);
+
+  case GNUNET_FS_URI_LOC:
+    return uri_loc_to_string (uri);
+
+  default:
+    GNUNET_break (0);
+    return NULL;
+  }
+}
+
+
+/* end of fs_uri.c */
diff --git a/src/service/fs/gnunet-daemon-fsprofiler.c b/src/service/fs/gnunet-daemon-fsprofiler.c
new file mode 100644
index 000000000..b99933cfa
--- /dev/null
+++ b/src/service/fs/gnunet-daemon-fsprofiler.c
@@ -0,0 +1,673 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012 Christian Grothoff
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-daemon-fsprofiler.c
+ * @brief daemon that publishes and downloads (random) files
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - how to signal driver that we're done?
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+#include "gnunet_statistics_service.h"
+
+/**
+ * We use 'patterns' of the form (x,y,t) to specify desired download/publish
+ * activities of a peer.  They are stored in a DLL.
+ */
+struct Pattern
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct Pattern *next;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct Pattern *prev;
+
+  /**
+   * Execution context for the pattern (FS-handle to the operation).
+   */
+  void *ctx;
+
+  /**
+   * Secondary execution context for the pattern (FS-handle to the operation).
+   */
+  void *sctx;
+
+  /**
+   * When did the operation start?
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * With how much delay should this operation be started?
+   */
+  struct GNUNET_TIME_Relative delay;
+
+  /**
+   * Task to run the operation.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Secondary task to run the operation.
+   */
+  struct GNUNET_SCHEDULER_Task *stask;
+
+  /**
+   * X-value.
+   */
+  unsigned long long x;
+
+  /**
+   * Y-value.
+   */
+  unsigned long long y;
+};
+
+
+/**
+ * Return value from 'main'.
+ */
+static int global_ret;
+
+/**
+ * Configuration we use.
+ */
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Handle to the statistics service.
+ */
+static struct GNUNET_STATISTICS_Handle *stats_handle;
+
+/**
+ * Peer's FS handle.
+ */
+static struct GNUNET_FS_Handle *fs_handle;
+
+/**
+ * Unique number for this peer in the testbed.
+ */
+static unsigned long long my_peerid;
+
+/**
+ * Desired anonymity level.
+ */
+static unsigned long long anonymity_level;
+
+/**
+ * Desired replication level.
+ */
+static unsigned long long replication_level;
+
+/**
+ * String describing which publishing operations this peer should
+ * perform.  The format is "(SIZE,SEED,TIME)*", for example:
+ * "(1,5,0)(7,3,13)" means to publish a file with 1 byte and
+ * seed/keyword 5 immediately and another file with 7 bytes and
+ * seed/keyword 3 after 13 ms.
+ */
+static char *publish_pattern;
+
+/**
+ * Head of the DLL of publish patterns.
+ */
+static struct Pattern *publish_head;
+
+/**
+ * Tail of the DLL of publish patterns.
+ */
+static struct Pattern *publish_tail;
+
+/**
+ * String describing which download operations this peer should
+ * perform. The format is "(KEYWORD,SIZE,DELAY)*"; for example,
+ * "(1,7,3)(3,8,8)" means to download one file of 7 bytes under
+ * keyword "1" starting the search after 3 ms; and another one of 8
+ * bytes under keyword '3' starting after 8 ms.  The file size is
+ * used to determine which search result(s) should be used or ignored.
+ */
+static char *download_pattern;
+
+/**
+ * Head of the DLL of publish patterns.
+ */
+static struct Pattern *download_head;
+
+/**
+ * Tail of the DLL of publish patterns.
+ */
+static struct Pattern *download_tail;
+
+
+/**
+ * Parse a pattern string and store the corresponding
+ * 'struct Pattern' in the given head/tail.
+ *
+ * @param head where to store the head
+ * @param tail where to store the tail
+ * @param pattern pattern to parse
+ * @return GNUNET_OK on success
+ */
+static int
+parse_pattern (struct Pattern **head,
+               struct Pattern **tail,
+               const char *pattern)
+{
+  struct Pattern *p;
+  unsigned long long x;
+  unsigned long long y;
+  unsigned long long t;
+
+  while (3 == sscanf (pattern,
+                      "(%llu,%llu,%llu)",
+                      &x, &y, &t))
+  {
+    p = GNUNET_new (struct Pattern);
+    p->x = x;
+    p->y = y;
+    p->delay.rel_value_us = (uint64_t) t;
+    GNUNET_CONTAINER_DLL_insert (*head, *tail, p);
+    pattern = strstr (pattern, ")");
+    GNUNET_assert (NULL != pattern);
+    pattern++;
+  }
+  return (0 == strlen (pattern)) ? GNUNET_OK : GNUNET_SYSERR;
+}
+
+
+/**
+ * Create a KSK URI from a number.
+ *
+ * @param kval the number
+ * @return corresponding KSK URI
+ */
+static struct GNUNET_FS_Uri *
+make_keywords (uint64_t kval)
+{
+  char kw[128];
+
+  GNUNET_snprintf (kw, sizeof(kw),
+                   "%llu", (unsigned long long) kval);
+  return GNUNET_FS_uri_ksk_create (kw, NULL);
+}
+
+
+/**
+ * Create a file of the given length with a deterministic amount
+ * of data to be published under keyword 'kval'.
+ *
+ * @param length number of bytes in the file
+ * @param kval keyword value and seed for the data of the file
+ * @param ctx context to pass to 'fi'
+ * @return file information handle for the file
+ */
+static struct GNUNET_FS_FileInformation *
+make_file (uint64_t length,
+           uint64_t kval,
+           void *ctx)
+{
+  struct GNUNET_FS_FileInformation *fi;
+  struct GNUNET_FS_BlockOptions bo;
+  char *data;
+  struct GNUNET_FS_Uri *keywords;
+  unsigned long long i;
+  uint64_t xor;
+
+  data = NULL; /* to make compilers happy */
+  if ((0 != length) &&
+      (NULL == (data = GNUNET_malloc_large ((size_t) length))))
+    return NULL;
+  /* initialize data with 'unique' data only depending on 'kval' and 'size',
+     making sure that blocks do not repeat */
+  for (i = 0; i < length; i += 8)
+  {
+    xor = length ^ kval ^ (uint64_t) (i / 32 / 1024);
+    GNUNET_memcpy (&data[i], &xor, GNUNET_MIN (length - i, sizeof(uint64_t)));
+  }
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
+  bo.anonymity_level = (uint32_t) anonymity_level;
+  bo.content_priority = 128;
+  bo.replication_level = (uint32_t) replication_level;
+  keywords = make_keywords (kval);
+  fi = GNUNET_FS_file_information_create_from_data (fs_handle,
+                                                    ctx,
+                                                    length,
+                                                    data, keywords,
+                                                    NULL, GNUNET_NO, &bo);
+  GNUNET_FS_uri_destroy (keywords);
+  return fi;
+}
+
+
+/**
+ * Task run during shutdown.
+ *
+ * @param cls unused
+ */
+static void
+shutdown_task (void *cls)
+{
+  struct Pattern *p;
+
+  while (NULL != (p = publish_head))
+  {
+    if (NULL != p->task)
+      GNUNET_SCHEDULER_cancel (p->task);
+    if (NULL != p->ctx)
+      GNUNET_FS_publish_stop (p->ctx);
+    GNUNET_CONTAINER_DLL_remove (publish_head, publish_tail, p);
+    GNUNET_free (p);
+  }
+  while (NULL != (p = download_head))
+  {
+    if (NULL != p->task)
+      GNUNET_SCHEDULER_cancel (p->task);
+    if (NULL != p->stask)
+      GNUNET_SCHEDULER_cancel (p->stask);
+    if (NULL != p->ctx)
+      GNUNET_FS_download_stop (p->ctx, GNUNET_YES);
+    if (NULL != p->sctx)
+      GNUNET_FS_search_stop (p->sctx);
+    GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p);
+    GNUNET_free (p);
+  }
+  if (NULL != fs_handle)
+  {
+    GNUNET_FS_stop (fs_handle);
+    fs_handle = NULL;
+  }
+  if (NULL != stats_handle)
+  {
+    GNUNET_STATISTICS_destroy (stats_handle, GNUNET_YES);
+    stats_handle = NULL;
+  }
+}
+
+
+/**
+ * Task run when a publish operation should be stopped.
+ *
+ * @param cls the 'struct Pattern' of the publish operation to stop
+ */
+static void
+publish_stop_task (void *cls)
+{
+  struct Pattern *p = cls;
+
+  p->task = NULL;
+  GNUNET_FS_publish_stop (p->ctx);
+}
+
+
+/**
+ * Task run when a download operation should be stopped.
+ *
+ * @param cls the 'struct Pattern' of the download operation to stop
+ */
+static void
+download_stop_task (void *cls)
+{
+  struct Pattern *p = cls;
+
+  p->task = NULL;
+  GNUNET_FS_download_stop (p->ctx, GNUNET_YES);
+}
+
+
+/**
+ * Task run when a download operation should be stopped.
+ *
+ * @param cls the 'struct Pattern' of the download operation to stop
+ */
+static void
+search_stop_task (void *cls)
+{
+  struct Pattern *p = cls;
+
+  p->stask = NULL;
+  GNUNET_FS_search_stop (p->sctx);
+}
+
+
+/**
+ * Notification of FS to a client about the progress of an
+ * operation.  Callbacks of this type will be used for uploads,
+ * downloads and searches.  Some of the arguments depend a bit
+ * in their meaning on the context in which the callback is used.
+ *
+ * @param cls closure
+ * @param info details about the event, specifying the event type
+ *        and various bits about the event
+ * @return client-context (for the next progress call
+ *         for this operation; should be set to NULL for
+ *         SUSPEND and STOPPED events).  The value returned
+ *         will be passed to future callbacks in the respective
+ *         field in the GNUNET_FS_ProgressInfo struct.
+ */
+static void *
+progress_cb (void *cls,
+             const struct GNUNET_FS_ProgressInfo *info)
+{
+  struct Pattern *p;
+  const struct GNUNET_FS_Uri *uri;
+
+  switch (info->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_START:
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    p = info->value.publish.cctx;
+    return p;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    p = info->value.publish.cctx;
+    return p;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Publishing failed\n");
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# failed publish operations", 1, GNUNET_NO);
+    p = info->value.publish.cctx;
+    p->task = GNUNET_SCHEDULER_add_now (&publish_stop_task, p);
+    return p;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    p = info->value.publish.cctx;
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# publishing time (ms)",
+                              (long long) GNUNET_TIME_absolute_get_duration (
+                                p->start_time).rel_value_us / 1000LL,
+                              GNUNET_NO);
+    p->task = GNUNET_SCHEDULER_add_now (&publish_stop_task, p);
+    return p;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    p = info->value.publish.cctx;
+    p->ctx = NULL;
+    GNUNET_CONTAINER_DLL_remove (publish_head, publish_tail, p);
+    GNUNET_free (p);
+    return NULL;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_START:
+  case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS:
+  case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE:
+  case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE:
+    p = info->value.download.cctx;
+    return p;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Download failed\n");
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# failed downloads", 1, GNUNET_NO);
+    p = info->value.download.cctx;
+    p->task = GNUNET_SCHEDULER_add_now (&download_stop_task, p);
+    return p;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED:
+    p = info->value.download.cctx;
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# download time (ms)",
+                              (long long) GNUNET_TIME_absolute_get_duration (
+                                p->start_time).rel_value_us / 1000LL,
+                              GNUNET_NO);
+    p->task = GNUNET_SCHEDULER_add_now (&download_stop_task, p);
+    return p;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_STOPPED:
+    p = info->value.download.cctx;
+    p->ctx = NULL;
+    if (NULL == p->sctx)
+    {
+      GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p);
+      GNUNET_free (p);
+    }
+    return NULL;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+  case GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE:
+    p = info->value.search.cctx;
+    return p;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    p = info->value.search.cctx;
+    uri = info->value.search.specifics.result.uri;
+    if (GNUNET_YES != GNUNET_FS_uri_test_chk (uri))
+      return NULL;   /* not what we want */
+    if (p->y != GNUNET_FS_uri_chk_get_file_size (uri))
+      return NULL;   /* not what we want */
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# search time (ms)",
+                              (long long) GNUNET_TIME_absolute_get_duration (
+                                p->start_time).rel_value_us / 1000LL,
+                              GNUNET_NO);
+    p->start_time = GNUNET_TIME_absolute_get ();
+    p->ctx = GNUNET_FS_download_start (fs_handle, uri,
+                                       NULL, NULL, NULL,
+                                       0, GNUNET_FS_uri_chk_get_file_size (uri),
+                                       anonymity_level,
+                                       GNUNET_FS_DOWNLOAD_NO_TEMPORARIES,
+                                       p,
+                                       NULL);
+    p->stask = GNUNET_SCHEDULER_add_now (&search_stop_task, p);
+    return NULL;
+
+  case GNUNET_FS_STATUS_SEARCH_UPDATE:
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    return NULL;   /* don't care */
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Search failed\n");
+    GNUNET_STATISTICS_update (stats_handle,
+                              "# failed searches", 1, GNUNET_NO);
+    p = info->value.search.cctx;
+    p->stask = GNUNET_SCHEDULER_add_now (&search_stop_task, p);
+    return p;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    p = info->value.search.cctx;
+    p->sctx = NULL;
+    if (NULL == p->ctx)
+    {
+      GNUNET_CONTAINER_DLL_remove (download_head, download_tail, p);
+      GNUNET_free (p);
+    }
+    return NULL;
+
+  default:
+    /* unexpected event during profiling */
+    GNUNET_break (0);
+    return NULL;
+  }
+}
+
+
+/**
+ * Start publish operation.
+ *
+ * @param cls the 'struct Pattern' specifying the operation to perform
+ */
+static void
+start_publish (void *cls)
+{
+  struct Pattern *p = cls;
+  struct GNUNET_FS_FileInformation *fi;
+
+  p->task = NULL;
+  fi = make_file (p->x, p->y, p);
+  p->start_time = GNUNET_TIME_absolute_get ();
+  p->ctx = GNUNET_FS_publish_start (fs_handle,
+                                    fi,
+                                    NULL, NULL, NULL,
+                                    GNUNET_FS_PUBLISH_OPTION_NONE);
+}
+
+
+/**
+ * Start download operation.
+ *
+ * @param cls the 'struct Pattern' specifying the operation to perform
+ */
+static void
+start_download (void *cls)
+{
+  struct Pattern *p = cls;
+  struct GNUNET_FS_Uri *keywords;
+
+  p->task = NULL;
+  keywords = make_keywords (p->x);
+  p->start_time = GNUNET_TIME_absolute_get ();
+  p->sctx = GNUNET_FS_search_start (fs_handle, keywords,
+                                    anonymity_level,
+                                    GNUNET_FS_SEARCH_OPTION_NONE,
+                                    p);
+}
+
+
+/**
+ * @brief Main function that will be run by the scheduler.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used (for saving, can be NULL!)
+ * @param cfg_ configuration
+ */
+static void
+run (void *cls, char *const *args GNUNET_UNUSED,
+     const char *cfgfile GNUNET_UNUSED,
+     const struct GNUNET_CONFIGURATION_Handle *cfg_)
+{
+  char myoptname[128];
+  struct Pattern *p;
+
+  cfg = cfg_;
+  /* Scheduled the task to clean up when shutdown is called */
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             "TESTBED", "PEERID",
+                                             &my_peerid))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "TESTBED", "PEERID");
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             "FSPROFILER", "ANONYMITY_LEVEL",
+                                             &anonymity_level))
+    anonymity_level = 1;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             "FSPROFILER", "REPLICATION_LEVEL",
+                                             &replication_level))
+    replication_level = 1;
+  GNUNET_snprintf (myoptname, sizeof(myoptname),
+                   "DOWNLOAD-PATTERN-%llu", my_peerid);
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "FSPROFILER", myoptname,
+                                             &download_pattern))
+    download_pattern = GNUNET_strdup ("");
+  GNUNET_snprintf (myoptname, sizeof(myoptname),
+                   "PUBLISH-PATTERN-%llu", my_peerid);
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "FSPROFILER", myoptname,
+                                             &publish_pattern))
+    publish_pattern = GNUNET_strdup ("");
+  if ((GNUNET_OK !=
+       parse_pattern (&download_head,
+                      &download_tail,
+                      download_pattern)) ||
+      (GNUNET_OK !=
+       parse_pattern (&publish_head,
+                      &publish_tail,
+                      publish_pattern)))
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+
+  stats_handle = GNUNET_STATISTICS_create ("fsprofiler", cfg);
+  fs_handle =
+    GNUNET_FS_start (cfg,
+                     "fsprofiler",
+                     &progress_cb, NULL,
+                     GNUNET_FS_FLAGS_NONE,
+                     GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM, 1,
+                     GNUNET_FS_OPTIONS_REQUEST_PARALLELISM, 1,
+                     GNUNET_FS_OPTIONS_END);
+  if (NULL == fs_handle)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Could not acquire FS handle. Exiting.\n");
+    global_ret = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  for (p = publish_head; NULL != p; p = p->next)
+    p->task = GNUNET_SCHEDULER_add_delayed (p->delay,
+                                            &start_publish, p);
+  for (p = download_head; NULL != p; p = p->next)
+    p->task = GNUNET_SCHEDULER_add_delayed (p->delay,
+                                            &start_download, p);
+}
+
+
+/**
+ * Program that performs various "random" FS activities.
+ *
+ * @param argc number of arguments from the command line
+ * @param argv command line arguments
+ * @return 0 ok, 1 on error
+ */
+int
+main (int argc, char *const *argv)
+{
+  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_END
+  };
+
+  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+    return 2;
+  return (GNUNET_OK ==
+          GNUNET_PROGRAM_run (argc, argv, "gnunet-daemon-fsprofiler",
+                              gettext_noop
+                              (
+                                "Daemon to use file-sharing to measure its performance."),
+                              options, &run, NULL)) ? global_ret : 1;
+}
+
+
+/* end of gnunet-daemon-fsprofiler.c */
diff --git a/src/service/fs/gnunet-fs-profiler.c b/src/service/fs/gnunet-fs-profiler.c
new file mode 100644
index 000000000..62da46834
--- /dev/null
+++ b/src/service/fs/gnunet-fs-profiler.c
@@ -0,0 +1,245 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-fs-profiler.c
+ * @brief tool to benchmark/profile file-sharing
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testbed_service.h"
+
+/**
+ * Final status code.
+ */
+static int ret;
+
+/**
+ * Data file with the hosts for the testbed.
+ */
+static char *host_filename;
+
+/**
+ * Number of peers to run in the experiment.
+ */
+static unsigned int num_peers;
+
+/**
+ * After how long do we abort the test?
+ */
+static struct GNUNET_TIME_Relative timeout;
+
+/**
+ * Handle to the task run during termination.
+ */
+static struct GNUNET_SCHEDULER_Task *terminate_taskid;
+
+
+/**
+ * Function called after we've collected the statistics.
+ *
+ * @param cls NULL
+ * @param op the operation that has been finished
+ * @param emsg error message in case the operation has failed; will be NULL if
+ *          operation has executed successfully.
+ */
+static void
+shutdown_task (void *cls,
+               struct GNUNET_TESTBED_Operation *op,
+               const char *emsg)
+{
+  if (NULL != emsg)
+    fprintf (stderr,
+             "Error collecting statistics: %s\n",
+             emsg);
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Callback function to process statistic values from all peers.
+ * Prints them out.
+ *
+ * @param cls closure
+ * @param peer the peer the statistic belong to
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+process_stats (void *cls,
+               const struct GNUNET_TESTBED_Peer *peer,
+               const char *subsystem,
+               const char *name,
+               uint64_t value,
+               int is_persistent)
+{
+  fprintf (stdout,
+           "%p-%s: %s = %llu\n",
+           peer,
+           subsystem,
+           name,
+           (unsigned long long) value);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Task run on shutdown to terminate.  Triggers printing out
+ * all statistics.
+ *
+ * @param cls NULL
+ */
+static void
+terminate_task (void *cls)
+{
+  if (NULL != terminate_taskid)
+  {
+    GNUNET_SCHEDULER_cancel (terminate_taskid);
+    terminate_taskid = NULL;
+  }
+  GNUNET_TESTBED_get_statistics (0, NULL,
+                                 NULL, NULL,
+                                 &process_stats,
+                                 &shutdown_task,
+                                 NULL);
+}
+
+
+/**
+ * Task run on timeout to terminate.  Triggers printing out
+ * all statistics.
+ *
+ * @param cls NULL
+ */
+static void
+timeout_task (void *cls)
+{
+  terminate_taskid = NULL;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Signature of a main function for a testcase.
+ *
+ * @param cls closure
+ * @param h the run handle
+ * @param num_peers number of peers in 'peers'
+ * @param peers handle to peers run in the testbed
+ * @param links_succeeded the number of overlay link connection attempts that
+ *          succeeded
+ * @param links_failed the number of overlay link connection attempts that
+ *          failed
+ */
+static void
+test_master (void *cls,
+             struct GNUNET_TESTBED_RunHandle *h,
+             unsigned int num_peers,
+             struct GNUNET_TESTBED_Peer **peers,
+             unsigned int links_succeeded,
+             unsigned int links_failed)
+{
+  // const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
+  // FIXME: enable clients to signal 'completion' before timeout;
+  // in that case, run the 'terminate_task' "immediately"
+
+  if (0 != timeout.rel_value_us)
+    terminate_taskid = GNUNET_SCHEDULER_add_delayed (timeout,
+                                                     &timeout_task,
+                                                     NULL);
+  GNUNET_SCHEDULER_add_shutdown (&terminate_task,
+                                 NULL);
+}
+
+
+/**
+ * Main function that will be run by the scheduler.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used (for saving, can be NULL!)
+ * @param cfg configuration
+ */
+static void
+run (void *cls, char *const *args, const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  GNUNET_TESTBED_run (host_filename,
+                      cfg,
+                      num_peers,
+                      0, NULL, NULL,
+                      &test_master, (void *) cfg);
+}
+
+
+/**
+ * Program to run a file-sharing testbed.
+ *
+ * @param argc number of arguments from the command line
+ * @param argv command line arguments
+ * @return 0 ok, 1 on error
+ */
+int
+main (int argc, char *const *argv)
+{
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_option_uint ('n',
+                               "num-peers",
+                               "COUNT",
+                               gettext_noop (
+                                 "run the experiment with COUNT peers"),
+                               &num_peers),
+
+    GNUNET_GETOPT_option_string ('H',
+                                 "hosts",
+                                 "HOSTFILE",
+                                 gettext_noop (
+                                   "specifies name of a file with the HOSTS the testbed should use"),
+                                 &host_filename),
+
+    GNUNET_GETOPT_option_relative_time ('t',
+                                        "timeout",
+                                        "DELAY",
+                                        gettext_noop (
+                                          "automatically terminate experiment after DELAY"),
+                                        &timeout),
+
+    GNUNET_GETOPT_OPTION_END
+  };
+
+  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+    return 2;
+
+  ret = (GNUNET_OK ==
+         GNUNET_PROGRAM_run (argc, argv, "gnunet-fs-profiler",
+                             gettext_noop (
+                               "run a testbed to measure file-sharing performance"),
+                             options, &run,
+                             NULL)) ? ret : 1;
+  GNUNET_free_nz ((void *) argv);
+  return ret;
+}
+
+
+/* end of gnunet-fs-profiler.c */
diff --git a/src/service/fs/gnunet-helper-fs-publish.c b/src/service/fs/gnunet-helper-fs-publish.c
new file mode 100644
index 000000000..0e07b79dc
--- /dev/null
+++ b/src/service/fs/gnunet-helper-fs-publish.c
@@ -0,0 +1,579 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file src/fs/gnunet-helper-fs-publish.c
+ * @brief Tool to help extract meta data asynchronously
+ * @author Christian Grothoff
+ *
+ * This program will scan a directory for files with meta data
+ * and report the results to stdout.
+ */
+#include "platform.h"
+
+#include "gnunet_fs_service.h"
+
+
+/**
+ * A node of a directory tree.
+ */
+struct ScanTreeNode
+{
+  /**
+   * This is a doubly-linked list
+   */
+  struct ScanTreeNode *next;
+
+  /**
+   * This is a doubly-linked list
+   */
+  struct ScanTreeNode *prev;
+
+  /**
+   * Parent of this node, NULL for top-level entries.
+   */
+  struct ScanTreeNode *parent;
+
+  /**
+   * This is a doubly-linked tree
+   * NULL for files and empty directories
+   */
+  struct ScanTreeNode *children_head;
+
+  /**
+   * This is a doubly-linked tree
+   * NULL for files and empty directories
+   */
+  struct ScanTreeNode *children_tail;
+
+  /**
+   * Name of the file/directory
+   */
+  char *filename;
+
+  /**
+   * Size of the file (if it is a file), in bytes.
+   * At the moment it is set to 0 for directories.
+   */
+  uint64_t file_size;
+
+  /**
+   * #GNUNET_YES if this is a directory
+   */
+  int is_directory;
+};
+
+
+#if HAVE_LIBEXTRACTOR
+/**
+ * List of libextractor plugins to use for extracting.
+ */
+static struct EXTRACTOR_PluginList *plugins;
+#endif
+
+/**
+ * File descriptor we use for IPC with the parent.
+ */
+static int output_stream;
+
+
+#if HAVE_LIBEXTRACTOR
+/**
+ * Add meta data that libextractor finds to our meta data
+ * container.
+ *
+ * @param cls closure, our meta data container
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in @a data
+ * @return always 0 to continue extracting
+ */
+static int
+add_to_md (void *cls,
+           const char *plugin_name,
+           enum EXTRACTOR_MetaType type,
+           enum EXTRACTOR_MetaFormat format,
+           const char *data_mime_type,
+           const char *data,
+           size_t data_len)
+{
+  struct GNUNET_FS_MetaData *md = cls;
+
+  if (((EXTRACTOR_METAFORMAT_UTF8 == format) ||
+       (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
+      ('\0' != data[data_len - 1]))
+  {
+    char zdata[data_len + 1];
+    GNUNET_memcpy (zdata, data, data_len);
+    zdata[data_len] = '\0';
+    (void) GNUNET_FS_meta_data_insert (md,
+                                       plugin_name,
+                                       type,
+                                       format,
+                                       data_mime_type,
+                                       zdata,
+                                       data_len + 1);
+  }
+  else
+  {
+    (void) GNUNET_FS_meta_data_insert (md,
+                                       plugin_name,
+                                       type,
+                                       format,
+                                       data_mime_type,
+                                       data,
+                                       data_len);
+  }
+  return 0;
+}
+
+
+#endif
+
+
+/**
+ * Free memory of the @a tree structure
+ *
+ * @param tree tree to free
+ */
+static void
+free_tree (struct ScanTreeNode *tree)
+{
+  struct ScanTreeNode *pos;
+
+  while (NULL != (pos = tree->children_head))
+    free_tree (pos);
+  if (NULL != tree->parent)
+    GNUNET_CONTAINER_DLL_remove (tree->parent->children_head,
+                                 tree->parent->children_tail,
+                                 tree);
+  GNUNET_free (tree->filename);
+  GNUNET_free (tree);
+}
+
+
+/**
+ * Write @a size bytes from @a buf into the #output_stream.
+ *
+ * @param buf buffer with data to write
+ * @param size number of bytes to write
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+write_all (const void *buf, size_t size)
+{
+  const char *cbuf = buf;
+  size_t total;
+  ssize_t wr;
+
+  total = 0;
+  do
+  {
+    wr = write (output_stream, &cbuf[total], size - total);
+    if (wr > 0)
+      total += wr;
+  }
+  while ((wr > 0) && (total < size));
+  if (wr <= 0)
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Failed to write to stdout: %s\n",
+                strerror (errno));
+  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
+}
+
+
+/**
+ * Write message to the master process.
+ *
+ * @param message_type message type to use
+ * @param data data to append, NULL for none
+ * @param data_length number of bytes in @a data
+ * @return #GNUNET_SYSERR to stop scanning (the pipe was broken somehow)
+ */
+static int
+write_message (uint16_t message_type, const char *data, size_t data_length)
+{
+  struct GNUNET_MessageHeader hdr;
+
+#if 0
+  fprintf (stderr,
+           "Helper sends %u-byte message of type %u\n",
+           (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
+           (unsigned int) message_type);
+#endif
+  hdr.type = htons (message_type);
+  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
+  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
+      (GNUNET_OK != write_all (data, data_length)))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called to (recursively) add all of the files in the
+ * directory to the tree.  Called by the directory scanner to initiate
+ * the scan.  Does NOT yet add any metadata.
+ *
+ * @param filename file or directory to scan
+ * @param dst where to store the resulting share tree item;
+ *         NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned)
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+preprocess_file (const char *filename, struct ScanTreeNode **dst);
+
+
+/**
+ * Closure for the 'scan_callback'
+ */
+struct RecursionContext
+{
+  /**
+   * Parent to add the files to.
+   */
+  struct ScanTreeNode *parent;
+
+  /**
+   * Flag to set to GNUNET_YES on serious errors.
+   */
+  int stop;
+};
+
+
+/**
+ * Function called by the directory iterator to (recursively) add all
+ * of the files in the directory to the tree.  Called by the directory
+ * scanner to initiate the scan.  Does NOT yet add any metadata.
+ *
+ * @param cls the `struct RecursionContext`
+ * @param filename file or directory to scan
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+scan_callback (void *cls, const char *filename)
+{
+  struct RecursionContext *rc = cls;
+  struct ScanTreeNode *chld;
+
+  if (GNUNET_OK != preprocess_file (filename, &chld))
+  {
+    rc->stop = GNUNET_YES;
+    return GNUNET_SYSERR;
+  }
+  if (NULL == chld)
+    return GNUNET_OK;
+  chld->parent = rc->parent;
+  GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
+                               rc->parent->children_tail,
+                               chld);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function called to (recursively) add all of the files in the
+ * directory to the tree.  Called by the directory scanner to initiate
+ * the scan.  Does NOT yet add any metadata.
+ *
+ * @param filename file or directory to scan
+ * @param dst where to store the resulting share tree item;
+ *         NULL is stored in @a dst upon recoverable errors (#GNUNET_OK is returned)
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+preprocess_file (const char *filename, struct ScanTreeNode **dst)
+{
+  struct ScanTreeNode *item;
+  struct stat sbuf;
+  uint64_t fsize = 0;
+
+  if ((0 != stat (filename, &sbuf)) ||
+      ((! S_ISDIR (sbuf.st_mode)) &&
+       (GNUNET_OK !=
+        GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES))))
+  {
+    /* If the file doesn't exist (or is not stat-able for any other reason)
+       skip it (but report it), but do continue. */
+    if (GNUNET_OK !=
+        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE,
+                       filename,
+                       strlen (filename) + 1))
+      return GNUNET_SYSERR;
+    /* recoverable error, store 'NULL' in *dst */
+    *dst = NULL;
+    return GNUNET_OK;
+  }
+
+  /* Report the progress */
+  if (
+    GNUNET_OK !=
+    write_message (S_ISDIR (sbuf.st_mode)
+                   ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
+                   : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE,
+                   filename,
+                   strlen (filename) + 1))
+    return GNUNET_SYSERR;
+  item = GNUNET_new (struct ScanTreeNode);
+  item->filename = GNUNET_strdup (filename);
+  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
+  item->file_size = fsize;
+  if (GNUNET_YES == item->is_directory)
+  {
+    struct RecursionContext rc;
+
+    rc.parent = item;
+    rc.stop = GNUNET_NO;
+    GNUNET_DISK_directory_scan (filename, &scan_callback, &rc);
+    if (
+      (GNUNET_YES == rc.stop) ||
+      (GNUNET_OK !=
+       write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
+                      "..",
+                      3)))
+    {
+      free_tree (item);
+      return GNUNET_SYSERR;
+    }
+  }
+  *dst = item;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Extract metadata from files.
+ *
+ * @param item entry we are processing
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on fatal errors
+ */
+static int
+extract_files (struct ScanTreeNode *item)
+{
+  struct GNUNET_FS_MetaData *meta;
+  ssize_t size;
+  size_t slen;
+
+  if (GNUNET_YES == item->is_directory)
+  {
+    /* for directories, we simply only descent, no extraction, no
+       progress reporting */
+    struct ScanTreeNode *pos;
+
+    for (pos = item->children_head; NULL != pos; pos = pos->next)
+      if (GNUNET_OK != extract_files (pos))
+        return GNUNET_SYSERR;
+    return GNUNET_OK;
+  }
+
+  /* this is the expensive operation, *afterwards* we'll check for aborts */
+  meta = GNUNET_FS_meta_data_create ();
+#if HAVE_LIBEXTRACTOR
+  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
+#endif
+  slen = strlen (item->filename) + 1;
+  size = GNUNET_FS_meta_data_get_serialized_size (meta);
+  if (-1 == size)
+  {
+    /* no meta data */
+    GNUNET_FS_meta_data_destroy (meta);
+    if (GNUNET_OK !=
+        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
+                       item->filename,
+                       slen))
+      return GNUNET_SYSERR;
+    return GNUNET_OK;
+  }
+  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
+  {
+    /* We can't transfer more than 64k bytes in one message. */
+    size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
+  }
+  {
+    char buf[size + slen];
+    char *dst = &buf[slen];
+
+    GNUNET_memcpy (buf, item->filename, slen);
+    size = GNUNET_FS_meta_data_serialize (
+      meta,
+      &dst,
+      size,
+      GNUNET_FS_META_DATA_SERIALIZE_PART);
+    if (size < 0)
+    {
+      GNUNET_break (0);
+      size = 0;
+    }
+    GNUNET_FS_meta_data_destroy (meta);
+    if (GNUNET_OK !=
+        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
+                       buf,
+                       slen + size))
+      return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Install a signal handler to ignore SIGPIPE.
+ */
+static void
+ignore_sigpipe ()
+{
+  struct sigaction oldsig;
+  struct sigaction sig;
+
+  memset (&sig, 0, sizeof(struct sigaction));
+  sig.sa_handler = SIG_IGN;
+  sigemptyset (&sig.sa_mask);
+#ifdef SA_INTERRUPT
+  sig.sa_flags = SA_INTERRUPT; /* SunOS */
+#else
+  sig.sa_flags = SA_RESTART;
+#endif
+  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
+    fprintf (stderr,
+             "Failed to install SIGPIPE handler: %s\n",
+             strerror (errno));
+}
+
+
+/**
+ * Turn the given file descriptor in to '/dev/null'.
+ *
+ * @param fd fd to bind to /dev/null
+ * @param flags flags to use (O_RDONLY or O_WRONLY)
+ */
+static void
+make_dev_zero (int fd, int flags)
+{
+  int z;
+
+  GNUNET_assert (0 == close (fd));
+  z = open ("/dev/null", flags);
+  GNUNET_assert (-1 != z);
+  if (z == fd)
+    return;
+  GNUNET_break (fd == dup2 (z, fd));
+  GNUNET_assert (0 == close (z));
+}
+
+
+/**
+ * Main function of the helper process to extract meta data.
+ *
+ * @param argc should be 3
+ * @param argv [0] our binary name
+ *             [1] name of the file or directory to process
+ *             [2] "-" to disable extraction, NULL for defaults,
+ *                 otherwise custom plugins to load from LE
+ * @return 0 on success
+ */
+int
+main (int argc, char *const *argv)
+{
+  const char *filename_expanded;
+  const char *ex;
+  struct ScanTreeNode *root;
+
+  ignore_sigpipe ();
+  /* move stdout to some other FD for IPC, bind
+     stdout/stderr to /dev/null */
+  output_stream = dup (1);
+  make_dev_zero (1, O_WRONLY);
+  make_dev_zero (2, O_WRONLY);
+
+  /* parse command line */
+  if ((3 != argc) && (2 != argc))
+  {
+    fprintf (stderr,
+             "%s",
+             "gnunet-helper-fs-publish needs exactly one or two arguments\n");
+    return 1;
+  }
+  filename_expanded = argv[1];
+  ex = argv[2];
+  if ((NULL == ex) || (0 != strcmp (ex, "-")))
+  {
+#if HAVE_LIBEXTRACTOR
+    plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
+    if (NULL != ex)
+      plugins = EXTRACTOR_plugin_add_config (plugins,
+                                             ex,
+                                             EXTRACTOR_OPTION_DEFAULT_POLICY);
+#endif
+  }
+
+  /* scan tree to find out how much work there is to be done */
+  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
+  {
+    (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0);
+#if HAVE_LIBEXTRACTOR
+    EXTRACTOR_plugin_remove_all (plugins);
+#endif
+    return 2;
+  }
+  /* signal that we're done counting files, so that a percentage of
+     progress can now be calculated */
+  if (GNUNET_OK !=
+      write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE,
+                     NULL,
+                     0))
+  {
+#if HAVE_LIBEXTRACTOR
+    EXTRACTOR_plugin_remove_all (plugins);
+#endif
+    return 3;
+  }
+  if (NULL != root)
+  {
+    if (GNUNET_OK != extract_files (root))
+    {
+      (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR,
+                            NULL,
+                            0);
+      free_tree (root);
+#if HAVE_LIBEXTRACTOR
+      EXTRACTOR_plugin_remove_all (plugins);
+#endif
+      return 4;
+    }
+    free_tree (root);
+  }
+  /* enable "clean" shutdown by telling parent that we are done */
+  (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED,
+                        NULL,
+                        0);
+#if HAVE_LIBEXTRACTOR
+  EXTRACTOR_plugin_remove_all (plugins);
+#endif
+  return 0;
+}
+
+
+/* end of gnunet-helper-fs-publish.c */
diff --git a/src/service/fs/gnunet-service-fs.c b/src/service/fs/gnunet-service-fs.c
new file mode 100644
index 000000000..1ab6ac2b8
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs.c
@@ -0,0 +1,1378 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009-2014, 2016 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs.c
+ * @brief gnunet anonymity protocol implementation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include <float.h>
+#include "gnunet_constants.h"
+#include "gnunet_core_service.h"
+#include "gnunet_dht_service.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_load_lib.h"
+#include "gnunet_protocols.h"
+#include "gnunet_signatures.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_util_lib.h"
+#include "gnunet-service-fs_cp.h"
+#include "gnunet-service-fs_indexing.h"
+#include "gnunet-service-fs_pe.h"
+#include "gnunet-service-fs_pr.h"
+#include "gnunet-service-fs_push.h"
+#include "gnunet-service-fs_put.h"
+#include "gnunet-service-fs_cadet.h"
+#include "fs.h"
+#include "fs_api.h"
+
+/**
+ * Size for the hash map for DHT requests from the FS
+ * service.  Should be about the number of concurrent
+ * DHT requests we plan to make.
+ */
+#define FS_DHT_HT_SIZE 1024
+
+
+/**
+ * How quickly do we age cover traffic?  At the given
+ * time interval, remaining cover traffic counters are
+ * decremented by 1/16th.
+ */
+#define COVER_AGE_FREQUENCY GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_SECONDS, 5)
+
+/**
+ * Collect an instance number of statistics?  May cause excessive IPC.
+ */
+#define INSANE_STATISTICS GNUNET_NO
+
+
+/**
+ * Doubly-linked list of requests we are performing
+ * on behalf of the same client.
+ */
+struct ClientRequest
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct ClientRequest *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct ClientRequest *prev;
+
+  /**
+   * Request this entry represents.
+   */
+  struct GSF_PendingRequest *pr;
+
+  /**
+   * Client list this request belongs to.
+   */
+  struct GSF_LocalClient *lc;
+
+  /**
+   * Task scheduled to destroy the request.
+   */
+  struct GNUNET_SCHEDULER_Task *kill_task;
+};
+
+
+/**
+ * Replies to be transmitted to the client.  The actual
+ * response message is allocated after this struct.
+ */
+struct ClientResponse
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct ClientResponse *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct ClientResponse *prev;
+
+  /**
+   * Client list entry this response belongs to.
+   */
+  struct GSF_LocalClient *lc;
+
+  /**
+   * Number of bytes in the response.
+   */
+  size_t msize;
+};
+
+
+/**
+ * Information we track while handling an index
+ * start request from a client.
+ */
+struct IndexStartContext
+{
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexStartContext *next;
+
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexStartContext *prev;
+
+  /**
+   * Name of the indexed file.
+   */
+  char *filename;
+
+  /**
+   * Context for transmitting confirmation to client.
+   */
+  struct GSF_LocalClient *lc;
+
+  /**
+   * Context for hashing of the file.
+   */
+  struct GNUNET_CRYPTO_FileHashContext *fhc;
+
+  /**
+   * Hash of the contents of the file.
+   */
+  struct GNUNET_HashCode file_id;
+};
+
+
+/**
+ * A local client.
+ */
+struct GSF_LocalClient
+{
+  /**
+   * ID of the client.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Queue for sending replies.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Head of list of requests performed on behalf
+   * of this client right now.
+   */
+  struct ClientRequest *cr_head;
+
+  /**
+   * Tail of list of requests performed on behalf
+   * of this client right now.
+   */
+  struct ClientRequest *cr_tail;
+
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexStartContext *isc_head;
+
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexStartContext *isc_tail;
+
+  /**
+   * Head of linked list of responses.
+   */
+  struct ClientResponse *res_head;
+
+  /**
+   * Tail of linked list of responses.
+   */
+  struct ClientResponse *res_tail;
+};
+
+
+/* ****************************** globals ****************************** */
+
+/**
+ * Our connection to the datastore.
+ */
+struct GNUNET_DATASTORE_Handle *GSF_dsh;
+
+/**
+ * Our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *GSF_cfg;
+
+/**
+ * Handle for reporting statistics.
+ */
+struct GNUNET_STATISTICS_Handle *GSF_stats;
+
+/**
+ * Handle for DHT operations.
+ */
+struct GNUNET_DHT_Handle *GSF_dht;
+
+/**
+ * How long do requests typically stay in the routing table?
+ */
+struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime;
+
+/**
+ * Running average of the observed latency to other peers (round trip).
+ * Initialized to 5s as the initial default.
+ */
+struct GNUNET_TIME_Relative GSF_avg_latency = { 500 };
+
+
+/**
+ * Typical priorities we're seeing from other peers right now.  Since
+ * most priorities will be zero, this value is the weighted average of
+ * non-zero priorities seen "recently".  In order to ensure that new
+ * values do not dramatically change the ratio, values are first
+ * "capped" to a reasonable range (+N of the current value) and then
+ * averaged into the existing value by a ratio of 1:N.  Hence
+ * receiving the largest possible priority can still only raise our
+ * "current_priorities" by at most 1.
+ */
+double GSF_current_priorities;
+
+/**
+ * Size of the datastore queue we assume for common requests.
+ */
+unsigned int GSF_datastore_queue_size;
+
+/**
+ * How many query messages have we received 'recently' that
+ * have not yet been claimed as cover traffic?
+ */
+unsigned int GSF_cover_query_count;
+
+/**
+ * How many content messages have we received 'recently' that
+ * have not yet been claimed as cover traffic?
+ */
+unsigned int GSF_cover_content_count;
+
+/**
+ * Our block context.
+ */
+struct GNUNET_BLOCK_Context *GSF_block_ctx;
+
+/**
+ * Pointer to handle to the core service (points to NULL until we've
+ * connected to it).
+ */
+struct GNUNET_CORE_Handle *GSF_core;
+
+/**
+ * Are we introducing randomized delays for better anonymity?
+ */
+int GSF_enable_randomized_delays;
+
+/**
+ * Identity of this peer.
+ */
+struct GNUNET_PeerIdentity GSF_my_id;
+
+/* ***************************** locals ******************************* */
+
+/**
+ * Configuration for block library.
+ */
+static struct GNUNET_CONFIGURATION_Handle *block_cfg;
+
+/**
+ * Private key of this peer.  Used to sign LOC URI requests.
+ */
+static struct GNUNET_CRYPTO_EddsaPrivateKey pk;
+
+/**
+ * ID of our task that we use to age the cover counters.
+ */
+static struct GNUNET_SCHEDULER_Task *cover_age_task;
+
+/**
+ * Datastore 'GET' load tracking.
+ */
+static struct GNUNET_LOAD_Value *datastore_get_load;
+
+
+/**
+ * Creates a fresh local client handle.
+ *
+ * @param cls NULL
+ * @param client handle of the client
+ * @param mq message queue for @a client
+ * @return handle to local client entry
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *client,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct GSF_LocalClient *pos;
+
+  pos = GNUNET_new (struct GSF_LocalClient);
+  pos->client = client;
+  pos->mq = mq;
+  return pos;
+}
+
+
+/**
+ * Free the given client request.
+ *
+ * @param cls the client request to free
+ */
+static void
+client_request_destroy (void *cls)
+{
+  struct ClientRequest *cr = cls;
+  struct GSF_LocalClient *lc = cr->lc;
+
+  cr->kill_task = NULL;
+  GNUNET_CONTAINER_DLL_remove (lc->cr_head,
+                               lc->cr_tail,
+                               cr);
+  GSF_pending_request_cancel_ (cr->pr,
+                               GNUNET_YES);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# client searches active"),
+                            -1,
+                            GNUNET_NO);
+  GNUNET_free (cr);
+}
+
+
+/**
+ * Handle a reply to a pending request.  Also called if a request
+ * expires (then with data == NULL).  The handler may be called
+ * many times (depending on the request type), but will not be
+ * called during or after a call to #GSF_pending_request_cancel()
+ * and will also not be called anymore after a call signalling
+ * expiration.
+ *
+ * @param cls user-specified closure
+ * @param eval evaluation of the result
+ * @param pr handle to the original pending request
+ * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown"
+ * @param expiration when does @a data expire?
+ * @param last_transmission when was the last time we've tried to download this block? (FOREVER if unknown)
+ * @param type type of the block
+ * @param data response data, NULL on request expiration
+ * @param data_len number of bytes in @a data
+ */
+static void
+client_response_handler (void *cls,
+                         enum GNUNET_BLOCK_ReplyEvaluationResult eval,
+                         struct GSF_PendingRequest *pr,
+                         uint32_t reply_anonymity_level,
+                         struct GNUNET_TIME_Absolute expiration,
+                         struct GNUNET_TIME_Absolute last_transmission,
+                         enum GNUNET_BLOCK_Type type,
+                         const void *data,
+                         size_t data_len)
+{
+  struct ClientRequest *cr = cls;
+  struct GSF_LocalClient *lc;
+  struct GNUNET_MQ_Envelope *env;
+  struct ClientPutMessage *pm;
+  const struct GSF_PendingRequestData *prd;
+
+  if (NULL == data)
+  {
+    /* local-only request, with no result, clean up. */
+    if (NULL == cr->kill_task)
+      cr->kill_task = GNUNET_SCHEDULER_add_now (&client_request_destroy,
+                                                cr);
+    return;
+  }
+  prd = GSF_pending_request_get_data_ (pr);
+  GNUNET_break (type != GNUNET_BLOCK_TYPE_ANY);
+  if ((prd->type != type) && (prd->type != GNUNET_BLOCK_TYPE_ANY))
+  {
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop
+                              ("# replies received for local clients"), 1,
+                            GNUNET_NO);
+  GNUNET_assert (pr == cr->pr);
+  lc = cr->lc;
+  env = GNUNET_MQ_msg_extra (pm,
+                             data_len,
+                             GNUNET_MESSAGE_TYPE_FS_PUT);
+  pm->type = htonl (type);
+  pm->expiration = GNUNET_TIME_absolute_hton (expiration);
+  pm->last_transmission = GNUNET_TIME_absolute_hton (last_transmission);
+  pm->num_transmissions = htonl (prd->num_transmissions);
+  pm->respect_offered = htonl (prd->respect_offered);
+  GNUNET_memcpy (&pm[1],
+                 data,
+                 data_len);
+  GNUNET_MQ_send (lc->mq,
+                  env);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Queued reply to query `%s' for local client\n",
+              GNUNET_h2s (&prd->query));
+  if (GNUNET_BLOCK_REPLY_OK_LAST != eval)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Evaluation %d - keeping query alive\n",
+                (int) eval);
+    return;
+  }
+  if (NULL == cr->kill_task)
+    cr->kill_task = GNUNET_SCHEDULER_add_now (&client_request_destroy,
+                                              cr);
+}
+
+
+/**
+ * A client disconnected from us.  Tear down the local client
+ * record.
+ *
+ * @param cls unused
+ * @param client handle of the client
+ * @param app_ctx the `struct GSF_LocalClient`
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *app_ctx)
+{
+  struct GSF_LocalClient *lc = app_ctx;
+  struct IndexStartContext *isc;
+  struct ClientRequest *cr;
+  struct ClientResponse *res;
+
+  while (NULL != (cr = lc->cr_head))
+  {
+    if (NULL != cr->kill_task)
+      GNUNET_SCHEDULER_cancel (cr->kill_task);
+    client_request_destroy (cr);
+  }
+  while (NULL != (res = lc->res_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (lc->res_head,
+                                 lc->res_tail,
+                                 res);
+    GNUNET_free (res);
+  }
+  while (NULL != (isc = lc->isc_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (lc->isc_head,
+                                 lc->isc_tail,
+                                 isc);
+    GNUNET_CRYPTO_hash_file_cancel (isc->fhc);
+    GNUNET_free (isc);
+  }
+  GNUNET_free (lc);
+}
+
+
+/**
+ * Task that periodically ages our cover traffic statistics.
+ *
+ * @param cls unused closure
+ */
+static void
+age_cover_counters (void *cls)
+{
+  GSF_cover_content_count = (GSF_cover_content_count * 15) / 16;
+  GSF_cover_query_count = (GSF_cover_query_count * 15) / 16;
+  cover_age_task =
+    GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY,
+                                  &age_cover_counters,
+                                  NULL);
+}
+
+
+/**
+ * We've just now completed a datastore request.  Update our
+ * datastore load calculations.
+ *
+ * @param start time when the datastore request was issued
+ */
+void
+GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start)
+{
+  struct GNUNET_TIME_Relative delay;
+
+  delay = GNUNET_TIME_absolute_get_duration (start);
+  GNUNET_LOAD_update (datastore_get_load, delay.rel_value_us);
+}
+
+
+/**
+ * Test if the DATABASE (GET) load on this peer is too high
+ * to even consider processing the query at
+ * all.
+ *
+ * @param priority priority of the request (used as a reference point to compare with the load)
+ * @return #GNUNET_YES if the load is too high to do anything (load high)
+ *         #GNUNET_NO to process normally (load normal)
+ *         #GNUNET_SYSERR to process for free (load low)
+ */
+int
+GSF_test_get_load_too_high_ (uint32_t priority)
+{
+  double ld;
+
+  ld = GNUNET_LOAD_get_load (datastore_get_load);
+  if (ld < 1)
+    return GNUNET_SYSERR;
+  if (ld <= priority)
+    return GNUNET_NO;
+  return GNUNET_YES;
+}
+
+
+/**
+ * Check P2P "PUT" message.
+ *
+ * @param cls closure with the `struct GSF_ConnectedPeer`
+ * @param put the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_p2p_put (void *cls,
+               const struct PutMessage *put)
+{
+  enum GNUNET_BLOCK_Type type;
+
+  type = ntohl (put->type);
+  if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We have a new request, consider forwarding it to the given
+ * peer.
+ *
+ * @param cls the `struct GSF_PendingRequest`
+ * @param peer identity of the peer
+ * @param cp handle to the connected peer record
+ * @param ppd peer performance data
+ */
+static void
+consider_request_for_forwarding (void *cls,
+                                 const struct GNUNET_PeerIdentity *peer,
+                                 struct GSF_ConnectedPeer *cp,
+                                 const struct GSF_PeerPerformanceData *ppd)
+{
+  struct GSF_PendingRequest *pr = cls;
+
+  if (GNUNET_YES !=
+      GSF_pending_request_test_target_ (pr, peer))
+  {
+#if INSANE_STATISTICS
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# Loopback routes suppressed"), 1,
+                              GNUNET_NO);
+#endif
+    return;
+  }
+  GSF_plan_add_ (cp,
+                 pr);
+}
+
+
+/**
+ * Function to be called after we're done processing
+ * replies from the local lookup.  If the result status
+ * code indicates that there may be more replies, plan
+ * forwarding the request.
+ *
+ * @param cls closure (NULL)
+ * @param pr the pending request we were processing
+ * @param result final datastore lookup result
+ */
+void
+GSF_consider_forwarding (void *cls,
+                         struct GSF_PendingRequest *pr,
+                         enum GNUNET_BLOCK_ReplyEvaluationResult result)
+{
+  if (GNUNET_BLOCK_REPLY_OK_LAST == result)
+    return;                     /* we're done... */
+  if (GNUNET_YES !=
+      GSF_pending_request_test_active_ (pr))
+    return; /* request is not actually active, skip! */
+  GSF_iterate_connected_peers_ (&consider_request_for_forwarding,
+                                pr);
+}
+
+
+/**
+ * Check P2P "GET" request.
+ *
+ * @param cls closure
+ * @param gm the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+check_p2p_get (void *cls,
+               const struct GetMessage *gm)
+{
+  size_t msize;
+  unsigned int bm;
+  unsigned int bits;
+  size_t bfsize;
+
+  msize = ntohs (gm->header.size);
+  bm = ntohl (gm->hash_bitmap);
+  bits = 0;
+  while (bm > 0)
+  {
+    if (1 == (bm & 1))
+      bits++;
+    bm >>= 1;
+  }
+  if (msize < sizeof(struct GetMessage) + bits * sizeof(struct
+                                                        GNUNET_PeerIdentity))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  bfsize = msize - sizeof(struct GetMessage) - bits * sizeof(struct
+                                                             GNUNET_PeerIdentity);
+  /* bfsize must be power of 2, check! */
+  if (0 != ((bfsize - 1) & bfsize))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We're done with the local lookup, now consider
+ * P2P processing (depending on request options and
+ * result status).  Also signal that we can now
+ * receive more request information from the client.
+ *
+ * @param cls the client doing the request (`struct GSF_LocalClient`)
+ * @param pr the pending request we were processing
+ * @param result final datastore lookup result
+ */
+static void
+start_p2p_processing (void *cls,
+                      struct GSF_PendingRequest *pr,
+                      enum GNUNET_BLOCK_ReplyEvaluationResult result)
+{
+  struct GSF_LocalClient *lc = cls;
+  struct GSF_PendingRequestData *prd;
+
+  GNUNET_SERVICE_client_continue (lc->client);
+  if (GNUNET_BLOCK_REPLY_OK_LAST == result)
+    return;                     /* we're done, 'pr' was already destroyed... */
+  prd = GSF_pending_request_get_data_ (pr);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished database lookup for local request `%s' with result %d\n",
+              GNUNET_h2s (&prd->query),
+              result);
+  if (0 == prd->anonymity_level)
+  {
+    switch (prd->type)
+    {
+    case GNUNET_BLOCK_TYPE_FS_DBLOCK:
+    case GNUNET_BLOCK_TYPE_FS_IBLOCK:
+      /* the above block types MAY be available via 'cadet' */
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Considering cadet-based download for block\n");
+      GSF_cadet_lookup_ (pr);
+      break;
+
+    case GNUNET_BLOCK_TYPE_FS_UBLOCK:
+      /* the above block types are in the DHT */
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Considering DHT-based search for block\n");
+      GSF_dht_lookup_ (pr);
+      break;
+
+    default:
+      GNUNET_break (0);
+      break;
+    }
+  }
+  GSF_consider_forwarding (NULL,
+                           pr,
+                           result);
+}
+
+
+/**
+ * Check #GNUNET_MESSAGE_TYPE_FS_START_SEARCH-message (search request
+ * from client).
+ *
+ * @param cls identification of the client
+ * @param sm the actual message
+ * @return #GNUNET_OK if @a sm is well-formed
+ */
+static int
+check_client_start_search (void *cls,
+                           const struct SearchMessage *sm)
+{
+  uint16_t msize;
+
+  msize = ntohs (sm->header.size) - sizeof(struct SearchMessage);
+  if (0 != msize % sizeof(struct GNUNET_HashCode))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle #GNUNET_MESSAGE_TYPE_FS_START_SEARCH-message (search request
+ * from client).
+ *
+ * Responsible for creating the request entry itself and setting
+ * up reply callback and cancellation on client disconnect.
+ *
+ * @param cls identification of the client
+ * @param sm the actual message
+ */
+static void
+handle_client_start_search (void *cls,
+                            const struct SearchMessage *sm)
+{
+  static struct GNUNET_PeerIdentity all_zeros;
+  struct GSF_LocalClient *lc = cls;
+  struct ClientRequest *cr;
+  struct GSF_PendingRequestData *prd;
+  uint16_t msize;
+  unsigned int sc;
+  enum GNUNET_BLOCK_Type type;
+  enum GSF_PendingRequestOptions options;
+
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# client searches received"),
+                            1,
+                            GNUNET_NO);
+  msize = ntohs (sm->header.size) - sizeof(struct SearchMessage);
+  sc = msize / sizeof(struct GNUNET_HashCode);
+  type = ntohl (sm->type);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received request for `%s' of type %u from local client\n",
+              GNUNET_h2s (&sm->query),
+              (unsigned int) type);
+  cr = NULL;
+  /* detect duplicate UBLOCK requests */
+  if ((type == GNUNET_BLOCK_TYPE_FS_UBLOCK) ||
+      (type == GNUNET_BLOCK_TYPE_ANY))
+  {
+    cr = lc->cr_head;
+    while (NULL != cr)
+    {
+      prd = GSF_pending_request_get_data_ (cr->pr);
+      /* only unify with queries that hae not yet started local processing
+         (SEARCH_MESSAGE_OPTION_CONTINUED was always set) and that have a
+         matching query and type */
+      if ((GNUNET_YES != prd->has_started) &&
+          (0 != memcmp (&prd->query,
+                        &sm->query,
+                        sizeof(struct GNUNET_HashCode))) &&
+          (prd->type == type))
+        break;
+      cr = cr->next;
+    }
+  }
+  if (NULL != cr)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Have existing request, merging content-seen lists.\n");
+    GSF_pending_request_update_ (cr->pr,
+                                 (const struct GNUNET_HashCode *) &sm[1],
+                                 sc);
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# client searches updated (merged content seen list)"),
+                              1,
+                              GNUNET_NO);
+  }
+  else
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# client searches active"),
+                              1,
+                              GNUNET_NO);
+    cr = GNUNET_new (struct ClientRequest);
+    cr->lc = lc;
+    GNUNET_CONTAINER_DLL_insert (lc->cr_head,
+                                 lc->cr_tail,
+                                 cr);
+    options = GSF_PRO_LOCAL_REQUEST;
+    if (0 != (SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY & ntohl (sm->options)))
+      options |= GSF_PRO_LOCAL_ONLY;
+    cr->pr = GSF_pending_request_create_ (options, type,
+                                          &sm->query,
+                                          (0 !=
+                                           memcmp (&sm->target,
+                                                   &all_zeros,
+                                                   sizeof(struct
+                                                          GNUNET_PeerIdentity)))
+                                          ? &sm->target : NULL, NULL,
+                                          0 /* bf */,
+                                          ntohl (sm->anonymity_level),
+                                          0 /* priority */,
+                                          0 /* ttl */,
+                                          0 /* sender PID */,
+                                          0 /* origin PID */,
+                                          (const struct
+                                           GNUNET_HashCode *) &sm[1], sc,
+                                          &client_response_handler,
+                                          cr);
+  }
+  if (0 != (SEARCH_MESSAGE_OPTION_CONTINUED & ntohl (sm->options)))
+  {
+    GNUNET_SERVICE_client_continue (lc->client);
+    return;
+  }
+  GSF_pending_request_get_data_ (cr->pr)->has_started = GNUNET_YES;
+  GSF_local_lookup_ (cr->pr,
+                     &start_p2p_processing,
+                     lc);
+}
+
+
+/**
+ * Handle request to sign a LOC URI (from client).
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_client_loc_sign (void *cls,
+                        const struct RequestLocSignatureMessage *msg)
+{
+  struct GSF_LocalClient *lc = cls;
+  struct GNUNET_FS_Uri base;
+  struct GNUNET_FS_Uri *loc;
+  struct GNUNET_MQ_Envelope *env;
+  struct ResponseLocSignatureMessage *resp;
+
+  GNUNET_break (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT ==
+                ntohl (msg->purpose));
+  base.type = GNUNET_FS_URI_CHK;
+  base.data.chk.chk = msg->chk;
+  base.data.chk.file_length = GNUNET_ntohll (msg->file_length);
+  loc = GNUNET_FS_uri_loc_create (&base,
+                                  &pk,
+                                  GNUNET_TIME_absolute_ntoh (
+                                    msg->expiration_time));
+  env = GNUNET_MQ_msg (resp,
+                       GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE);
+  resp->purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT);
+  resp->expiration_time = GNUNET_TIME_absolute_hton (
+    loc->data.loc.expirationTime);
+  resp->signature = loc->data.loc.contentSignature;
+  resp->peer = loc->data.loc.peer;
+  GNUNET_FS_uri_destroy (loc);
+  GNUNET_MQ_send (lc->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (lc->client);
+}
+
+
+/**
+ * Check INDEX_START-message.
+ *
+ * @param cls identification of the client
+ * @param ism the actual message
+ * @return #GNUNET_OK if @a ism is well-formed
+ */
+static int
+check_client_index_start (void *cls,
+                          const struct IndexStartMessage *ism)
+{
+  char *fn;
+
+  GNUNET_MQ_check_zero_termination (ism);
+  if (0 != ism->reserved)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]);
+  if (NULL == fn)
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (fn);
+  return GNUNET_OK;
+}
+
+
+/**
+ * We've validated the hash of the file we're about to index.  Signal
+ * success to the client and update our internal data structures.
+ *
+ * @param isc the data about the index info entry for the request
+ */
+static void
+signal_index_ok (struct IndexStartContext *isc)
+{
+  struct GSF_LocalClient *lc = isc->lc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+
+  GNUNET_FS_add_to_index (isc->filename,
+                          &isc->file_id);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK);
+  GNUNET_MQ_send (lc->mq,
+                  env);
+  GNUNET_free (isc->filename);
+  GNUNET_free (isc);
+  GNUNET_SERVICE_client_continue (lc->client);
+}
+
+
+/**
+ * Function called once the hash computation over an
+ * indexed file has completed.
+ *
+ * @param cls closure, our publishing context
+ * @param res resulting hash, NULL on error
+ */
+static void
+hash_for_index_val (void *cls,
+                    const struct GNUNET_HashCode *res)
+{
+  struct IndexStartContext *isc = cls;
+  struct GSF_LocalClient *lc = isc->lc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+
+  GNUNET_CONTAINER_DLL_remove (lc->isc_head,
+                               lc->isc_tail,
+                               isc);
+  isc->fhc = NULL;
+  if ((NULL == res) ||
+      (0 != memcmp (res,
+                    &isc->file_id,
+                    sizeof(struct GNUNET_HashCode))))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Hash mismatch trying to index file `%s' which does not have hash `%s'\n"),
+                isc->filename,
+                GNUNET_h2s (&isc->file_id));
+
+    const char *emsg = "hash mismatch";
+    const size_t msize = strlen (emsg) + 1;
+
+    env = GNUNET_MQ_msg_extra (msg,
+                               msize,
+                               GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED);
+    memcpy ((char*) &msg[1], emsg, msize);
+    GNUNET_MQ_send (lc->mq,
+                    env);
+    GNUNET_SERVICE_client_continue (lc->client);
+    GNUNET_free (isc);
+    return;
+  }
+  signal_index_ok (isc);
+}
+
+
+/**
+ * Handle INDEX_START-message.
+ *
+ * @param cls identification of the client
+ * @param ism the actual message
+ */
+static void
+handle_client_index_start (void *cls,
+                           const struct IndexStartMessage *ism)
+{
+  struct GSF_LocalClient *lc = cls;
+  struct IndexStartContext *isc;
+  char *fn;
+  uint64_t dev;
+  uint64_t ino;
+  uint64_t mydev;
+  uint64_t myino;
+
+  fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]);
+  GNUNET_assert (NULL != fn);
+  dev = GNUNET_ntohll (ism->device);
+  ino = GNUNET_ntohll (ism->inode);
+  isc = GNUNET_new (struct IndexStartContext);
+  isc->filename = fn;
+  isc->file_id = ism->file_id;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received START_INDEX message for file `%s'\n",
+              isc->filename);
+  isc->lc = lc;
+  mydev = 0;
+  myino = 0;
+  if (((dev != 0) ||
+       (ino != 0)) &&
+      (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn,
+                                                      &mydev,
+                                                      &myino)) &&
+      (dev == mydev) &&
+      (ino == myino))
+  {
+    /* fast validation OK! */
+    signal_index_ok (isc);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n",
+              (unsigned long long) ino,
+              (unsigned long long) myino,
+              (unsigned int) dev,
+              (unsigned int) mydev);
+  /* slow validation, need to hash full file (again) */
+  GNUNET_CONTAINER_DLL_insert (lc->isc_head,
+                               lc->isc_tail,
+                               isc);
+  isc->fhc = GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                      isc->filename,
+                                      HASHING_BLOCKSIZE,
+                                      &hash_for_index_val,
+                                      isc);
+  if (NULL == isc->fhc)
+    hash_for_index_val (isc,
+                        NULL);
+}
+
+
+/**
+ * Handle INDEX_LIST_GET-message.
+ *
+ * @param cls closure
+ * @param message the actual message
+ */
+static void
+handle_client_index_list_get (void *cls,
+                              const struct GNUNET_MessageHeader *message)
+{
+  struct GSF_LocalClient *lc = cls;
+
+  GNUNET_FS_indexing_send_list (lc->mq);
+  GNUNET_SERVICE_client_continue (lc->client);
+}
+
+
+/**
+ * Handle UNINDEX-message.
+ *
+ * @param cls identification of the client
+ * @param um the actual message
+ */
+static void
+handle_client_unindex (void *cls,
+                       const struct UnindexMessage *um)
+{
+  struct GSF_LocalClient *lc = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+  int found;
+
+  GNUNET_break (0 == um->reserved);
+  found = GNUNET_FS_indexing_do_unindex (&um->file_id);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client requested unindexing of file `%s': %s\n",
+              GNUNET_h2s (&um->file_id),
+              found ? "found" : "not found");
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK);
+  GNUNET_MQ_send (lc->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (lc->client);
+}
+
+
+/**
+ * Task run during shutdown.
+ *
+ * @param cls unused
+ */
+static void
+shutdown_task (void *cls)
+{
+  GSF_cadet_stop_server ();
+  if (NULL != GSF_core)
+  {
+    GNUNET_CORE_disconnect (GSF_core);
+    GSF_core = NULL;
+  }
+  GSF_put_done_ ();
+  GSF_push_done_ ();
+  GSF_pending_request_done_ ();
+  GSF_plan_done ();
+  GSF_connected_peer_done_ ();
+  GNUNET_DATASTORE_disconnect (GSF_dsh,
+                               GNUNET_NO);
+  GSF_dsh = NULL;
+  GNUNET_DHT_disconnect (GSF_dht);
+  GSF_dht = NULL;
+  GNUNET_BLOCK_context_destroy (GSF_block_ctx);
+  GSF_block_ctx = NULL;
+  GNUNET_CONFIGURATION_destroy (block_cfg);
+  block_cfg = NULL;
+  GNUNET_STATISTICS_destroy (GSF_stats, GNUNET_NO);
+  GSF_stats = NULL;
+  if (NULL != cover_age_task)
+  {
+    GNUNET_SCHEDULER_cancel (cover_age_task);
+    cover_age_task = NULL;
+  }
+  GNUNET_FS_indexing_done ();
+  GNUNET_LOAD_value_free (datastore_get_load);
+  datastore_get_load = NULL;
+  GNUNET_LOAD_value_free (GSF_rt_entry_lifetime);
+  GSF_rt_entry_lifetime = NULL;
+}
+
+
+/**
+ * Function called after GNUNET_CORE_connect has succeeded
+ * (or failed for good).  Note that the private key of the
+ * peer is intentionally not exposed here; if you need it,
+ * your process should try to read the private key file
+ * directly (which should work if you are authorized...).
+ *
+ * @param cls closure
+ * @param my_identity ID of this peer, NULL if we failed
+ */
+static void
+peer_init_handler (void *cls,
+                   const struct GNUNET_PeerIdentity *my_identity)
+{
+  if (0 != GNUNET_memcmp (&GSF_my_id,
+                          my_identity))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Peer identity mismatch, refusing to start!\n");
+    GNUNET_SCHEDULER_shutdown ();
+  }
+}
+
+
+/**
+ * Process fs requests.
+ *
+ * @param c configuration to use
+ */
+static int
+main_init (const struct GNUNET_CONFIGURATION_Handle *c)
+{
+  struct GNUNET_MQ_MessageHandler no_p2p_handlers[] = {
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_MQ_MessageHandler p2p_handlers[] = {
+    GNUNET_MQ_hd_var_size (p2p_get,
+                           GNUNET_MESSAGE_TYPE_FS_GET,
+                           struct GetMessage,
+                           NULL),
+    GNUNET_MQ_hd_var_size (p2p_put,
+                           GNUNET_MESSAGE_TYPE_FS_PUT,
+                           struct PutMessage,
+                           NULL),
+    GNUNET_MQ_hd_fixed_size (p2p_migration_stop,
+                             GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP,
+                             struct MigrationStopMessage,
+                             NULL),
+    GNUNET_MQ_handler_end ()
+  };
+  int anon_p2p_off;
+  char *keyfile;
+
+  /* this option is really only for testcases that need to disable
+     _anonymous_ file-sharing for some reason */
+  anon_p2p_off = (GNUNET_YES ==
+                  GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg,
+                                                        "fs",
+                                                        "DISABLE_ANON_TRANSFER"));
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (GSF_cfg,
+                                               "PEER",
+                                               "PRIVATE_KEY",
+                                               &keyfile))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ (
+                  "FS service is lacking HOSTKEY configuration setting.  Exiting.\n"));
+    GNUNET_SCHEDULER_shutdown ();
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_SYSERR ==
+      GNUNET_CRYPTO_eddsa_key_from_file (keyfile,
+                                         GNUNET_YES,
+                                         &pk))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to setup peer's private key\n");
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_free (keyfile);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (keyfile);
+  GNUNET_CRYPTO_eddsa_key_get_public (&pk,
+                                      &GSF_my_id.public_key);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "I am peer %s\n",
+              GNUNET_i2s (&GSF_my_id));
+  GSF_core
+    = GNUNET_CORE_connect (GSF_cfg,
+                           NULL,
+                           &peer_init_handler,
+                           &GSF_peer_connect_handler,
+                           &GSF_peer_disconnect_handler,
+                           (GNUNET_YES == anon_p2p_off)
+                           ? no_p2p_handlers
+                           : p2p_handlers);
+  if (NULL == GSF_core)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to connect to `%s' service.\n"),
+                "core");
+    return GNUNET_SYSERR;
+  }
+  cover_age_task =
+    GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY,
+                                  &age_cover_counters,
+                                  NULL);
+  datastore_get_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE);
+  GSF_cadet_start_server ();
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Process fs requests.
+ *
+ * @param cls closure
+ * @param cfg configuration to use
+ * @param service the initialized service
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_SERVICE_Handle *service)
+{
+  unsigned long long dqs;
+
+  GSF_cfg = cfg;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_size (GSF_cfg,
+                                           "fs",
+                                           "DATASTORE_QUEUE_SIZE",
+                                           &dqs))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO,
+                               "fs",
+                               "DATASTORE_QUEUE_SIZE");
+    dqs = 32;
+  }
+  GSF_datastore_queue_size = (unsigned int) dqs;
+  GSF_enable_randomized_delays =
+    GNUNET_CONFIGURATION_get_value_yesno (cfg, "fs", "DELAY");
+  GSF_dsh = GNUNET_DATASTORE_connect (cfg);
+  if (NULL == GSF_dsh)
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GSF_rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL);
+  GSF_stats = GNUNET_STATISTICS_create ("fs", cfg);
+  block_cfg = GNUNET_CONFIGURATION_create ();
+  GSF_block_ctx = GNUNET_BLOCK_context_create (block_cfg);
+  GNUNET_assert (NULL != GSF_block_ctx);
+  GSF_dht = GNUNET_DHT_connect (cfg, FS_DHT_HT_SIZE);
+  GSF_plan_init ();
+  GSF_pending_request_init_ ();
+  GSF_connected_peer_init_ ();
+
+  GSF_push_init_ ();
+  GSF_put_init_ ();
+  if ((GNUNET_OK != GNUNET_FS_indexing_init (cfg,
+                                             GSF_dsh)) ||
+      (GNUNET_OK != main_init (cfg)))
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    shutdown_task (NULL);
+    return;
+  }
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+  ("fs",
+  GNUNET_SERVICE_OPTION_NONE,
+  &run,
+  &client_connect_cb,
+  &client_disconnect_cb,
+  NULL,
+  GNUNET_MQ_hd_var_size (client_index_start,
+                         GNUNET_MESSAGE_TYPE_FS_INDEX_START,
+                         struct IndexStartMessage,
+                         NULL),
+  GNUNET_MQ_hd_fixed_size (client_index_list_get,
+                           GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+  GNUNET_MQ_hd_fixed_size (client_unindex,
+                           GNUNET_MESSAGE_TYPE_FS_UNINDEX,
+                           struct UnindexMessage,
+                           NULL),
+  GNUNET_MQ_hd_var_size (client_start_search,
+                         GNUNET_MESSAGE_TYPE_FS_START_SEARCH,
+                         struct SearchMessage,
+                         NULL),
+  GNUNET_MQ_hd_fixed_size (client_loc_sign,
+                           GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN,
+                           struct RequestLocSignatureMessage,
+                           NULL),
+  GNUNET_MQ_handler_end ());
+
+
+/* end of gnunet-service-fs.c */
diff --git a/src/service/fs/gnunet-service-fs.h b/src/service/fs/gnunet-service-fs.h
new file mode 100644
index 000000000..e102a1fba
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs.h
@@ -0,0 +1,304 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs.h
+ * @brief shared data structures of gnunet-service-fs.c
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_H
+#define GNUNET_SERVICE_FS_H
+
+#include "gnunet_util_lib.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_core_service.h"
+#include "gnunet_block_lib.h"
+#include "fs.h"
+
+
+/**
+ * By which amount do we decrement the TTL for simple forwarding /
+ * indirection of the query; in milli-seconds.  Set somewhat in
+ * accordance to your network latency (above the time it'll take you
+ * to send a packet and get a reply).
+ */
+#define TTL_DECREMENT 5000
+
+/**
+ * At what frequency should our datastore load decrease
+ * automatically (since if we don't use it, clearly the
+ * load must be going down).
+ */
+#define DATASTORE_LOAD_AUTODECLINE GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MILLISECONDS, 250)
+
+/**
+ * Only the (mandatory) query is included.
+ */
+#define GET_MESSAGE_BIT_QUERY_ONLY 0
+
+/**
+ * The peer identity of a peer waiting for the
+ * reply is included (used if the response
+ * should be transmitted to someone other than
+ * the sender of the GET).
+ */
+#define GET_MESSAGE_BIT_RETURN_TO 1
+
+/**
+ * The peer identity of a peer that had claimed to have the content
+ * previously is included (can be used if responder-anonymity is not
+ * desired; note that the precursor presumably lacked a direct
+ * connection to the specified peer; still, the receiver is in no way
+ * required to limit forwarding only to the specified peer, it should
+ * only prefer it somewhat if possible).
+ */
+#define GET_MESSAGE_BIT_TRANSMIT_TO 4
+
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
+ * Message sent between peers asking for FS-content.
+ */
+struct GetMessage
+{
+  /**
+   * Message type will be #GNUNET_MESSAGE_TYPE_FS_GET.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Type of the query (block type).
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * How important is this request (network byte order)
+   */
+  uint32_t priority GNUNET_PACKED;
+
+  /**
+   * Relative time to live in MILLISECONDS (network byte order)
+   */
+  int32_t ttl GNUNET_PACKED;
+
+  /**
+   * These days not used.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * Which of the optional hash codes are present at the end of the
+   * message?  See GET_MESSAGE_BIT_xx constants.  For each bit that is
+   * set, an additional `struct GNUNET_HashCode` with the respective content
+   * (in order of the bits) will be appended to the end of the GET
+   * message.
+   */
+  uint32_t hash_bitmap GNUNET_PACKED;
+
+  /**
+   * Hashcodes of the file(s) we're looking for.
+   * Details depend on the query type.
+   */
+  struct GNUNET_HashCode query;
+
+  /* this is followed by PeerIdentities as specified in the "hash_bitmap";
+   * after that, an optional bloomfilter (with bits set for replies
+   * that should be suppressed) can be present */
+};
+
+
+/**
+ * Message send by a peer that wants to be excluded
+ * from migration for a while.
+ */
+struct MigrationStopMessage
+{
+  /**
+   * Message type will be
+   * GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t reserved GNUNET_PACKED;
+
+  /**
+   * How long should the block last?
+   */
+  struct GNUNET_TIME_RelativeNBO duration;
+};
+GNUNET_NETWORK_STRUCT_END
+
+/**
+ * A connected peer.
+ */
+struct GSF_ConnectedPeer;
+
+/**
+ * An active request.
+ */
+struct GSF_PendingRequest;
+
+/**
+ * A local client.
+ */
+struct GSF_LocalClient;
+
+/**
+ * Information kept per plan per request ('pe' module).
+ */
+struct GSF_RequestPlan;
+
+/**
+ * Bijection between request plans and pending requests.
+ */
+struct GSF_PendingRequestPlanBijection;
+
+/**
+ * Our connection to the datastore.
+ */
+extern struct GNUNET_DATASTORE_Handle *GSF_dsh;
+
+/**
+ * Our configuration.
+ */
+extern const struct GNUNET_CONFIGURATION_Handle *GSF_cfg;
+
+/**
+ * Handle for reporting statistics.
+ */
+extern struct GNUNET_STATISTICS_Handle *GSF_stats;
+
+/**
+ * Pointer to handle to the core service (points to NULL until we've
+ * connected to it).
+ */
+extern struct GNUNET_CORE_Handle *GSF_core;
+
+/**
+ * Handle for DHT operations.
+ */
+extern struct GNUNET_DHT_Handle *GSF_dht;
+
+/**
+ * How long do requests typically stay in the routing table?
+ */
+extern struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime;
+
+/**
+ * Running average of the observed latency to other peers (round trip).
+ */
+extern struct GNUNET_TIME_Relative GSF_avg_latency;
+
+/**
+ * Handle to ATS service.
+ */
+extern struct GNUNET_ATS_PerformanceHandle *GSF_ats;
+
+/**
+ * Identity of this peer.
+ */
+extern struct GNUNET_PeerIdentity GSF_my_id;
+
+/**
+ * Typical priorities we're seeing from other peers right now.  Since
+ * most priorities will be zero, this value is the weighted average of
+ * non-zero priorities seen "recently".  In order to ensure that new
+ * values do not dramatically change the ratio, values are first
+ * "capped" to a reasonable range (+N of the current value) and then
+ * averaged into the existing value by a ratio of 1:N.  Hence
+ * receiving the largest possible priority can still only raise our
+ * "current_priorities" by at most 1.
+ */
+extern double GSF_current_priorities;
+
+/**
+ * How many query messages have we received 'recently' that
+ * have not yet been claimed as cover traffic?
+ */
+extern unsigned int GSF_cover_query_count;
+
+/**
+ * How many content messages have we received 'recently' that
+ * have not yet been claimed as cover traffic?
+ */
+extern unsigned int GSF_cover_content_count;
+
+/**
+ * Our block context.
+ */
+extern struct GNUNET_BLOCK_Context *GSF_block_ctx;
+
+/**
+ * Are we introducing randomized delays for better anonymity?
+ */
+extern int GSF_enable_randomized_delays;
+
+/**
+ * Size of the datastore queue we assume for common requests.
+ */
+extern unsigned int GSF_datastore_queue_size;
+
+
+/**
+ * Function to be called after we're done processing
+ * replies from the local lookup.  If the result status
+ * code indicates that there may be more replies, plan
+ * forwarding the request.
+ *
+ * @param cls closure (NULL)
+ * @param pr the pending request we were processing
+ * @param result final datastore lookup result
+ */
+void
+GSF_consider_forwarding (void *cls,
+                         struct GSF_PendingRequest *pr,
+                         enum GNUNET_BLOCK_ReplyEvaluationResult result);
+
+
+/**
+ * Test if the DATABASE (GET) load on this peer is too high
+ * to even consider processing the query at
+ * all.
+ *
+ * @return #GNUNET_YES if the load is too high to do anything (load high)
+ *         #GNUNET_NO to process normally (load normal)
+ *         #GNUNET_SYSERR to process for free (load low)
+ */
+int
+GSF_test_get_load_too_high_ (uint32_t priority);
+
+
+/**
+ * We've just now completed a datastore request.  Update our
+ * datastore load calculations.
+ *
+ * @param start time when the datastore request was issued
+ */
+void
+GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start);
+
+
+#endif
+/* end of gnunet-service-fs.h */
diff --git a/src/service/fs/gnunet-service-fs_cadet.h b/src/service/fs/gnunet-service-fs_cadet.h
new file mode 100644
index 000000000..c02021a0d
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_cadet.h
@@ -0,0 +1,168 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_cadet.h
+ * @brief non-anonymous file-transfer
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_CADET_H
+#define GNUNET_SERVICE_FS_CADET_H
+
+/**
+ * Handle for a request that is going out via cadet API.
+ */
+struct GSF_CadetRequest;
+
+
+/**
+ * Function called with a reply from the cadet.
+ *
+ * @param cls closure
+ * @param type type of the block, ANY on error
+ * @param expiration expiration time for the block
+ * @param data_size number of bytes in @a data, 0 on error
+ * @param data reply block data, NULL on error
+ */
+typedef void
+(*GSF_CadetReplyProcessor)(void *cls,
+                           enum GNUNET_BLOCK_Type type,
+                           struct GNUNET_TIME_Absolute expiration,
+                           size_t data_size,
+                           const void *data);
+
+
+/**
+ * Look for a block by directly contacting a particular peer.
+ *
+ * @param target peer that should have the block
+ * @param query hash to query for the block
+ * @param type desired type for the block
+ * @param proc function to call with result
+ * @param proc_cls closure for @a proc
+ * @return handle to cancel the operation
+ */
+struct GSF_CadetRequest *
+GSF_cadet_query (const struct GNUNET_PeerIdentity *target,
+                 const struct GNUNET_HashCode *query,
+                 enum GNUNET_BLOCK_Type type,
+                 GSF_CadetReplyProcessor proc,
+                 void *proc_cls);
+
+/**
+ * Function called on each active cadets to shut them down.
+ *
+ * @param cls NULL
+ * @param key target peer, unused
+ * @param value the `struct CadetHandle` to destroy
+ * @return #GNUNET_YES (continue to iterate)
+ */
+int
+GSF_cadet_release_clients (void *cls,
+                           const struct GNUNET_PeerIdentity *key,
+                           void *value);
+
+
+/**
+ * Cancel an active request; must not be called after 'proc'
+ * was called.
+ *
+ * @param sr request to cancel
+ */
+void
+GSF_cadet_query_cancel (struct GSF_CadetRequest *sr);
+
+
+/**
+ * Initialize subsystem for non-anonymous file-sharing.
+ */
+void
+GSF_cadet_start_server (void);
+
+
+/**
+ * Shutdown subsystem for non-anonymous file-sharing.
+ */
+void
+GSF_cadet_stop_server (void);
+
+/**
+ * Cadet channel for creating outbound channels.
+ */
+extern struct GNUNET_CADET_Handle *cadet_handle;
+
+/**
+ * Map from peer identities to 'struct CadetHandles' with cadet
+ * channels to those peers.
+ */
+extern struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
+
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
+ * Query from one peer, asking the other for CHK-data.
+ */
+struct CadetQueryMessage
+{
+  /**
+   * Type is GNUNET_MESSAGE_TYPE_FS_CADET_QUERY.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Block type must be DBLOCK or IBLOCK.
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * Query hash from CHK (hash of encrypted block).
+   */
+  struct GNUNET_HashCode query;
+};
+
+
+/**
+ * Reply to a CadetQueryMessage.
+ */
+struct CadetReplyMessage
+{
+  /**
+   * Type is GNUNET_MESSAGE_TYPE_FS_CADET_REPLY.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Block type must be DBLOCK or IBLOCK.
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * Expiration time for the block.
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration;
+
+  /* followed by the encrypted block */
+};
+
+GNUNET_NETWORK_STRUCT_END
+
+
+#endif
diff --git a/src/service/fs/gnunet-service-fs_cadet_client.c b/src/service/fs/gnunet-service-fs_cadet_client.c
new file mode 100644
index 000000000..398fcd604
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_cadet_client.c
@@ -0,0 +1,728 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012, 2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_cadet_client.c
+ * @brief non-anonymous file-transfer
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - PORT is set to old application type, unsure if we should keep
+ *   it that way (fine for now)
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_cadet_service.h"
+#include "gnunet_protocols.h"
+#include "gnunet_applications.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_indexing.h"
+#include "gnunet-service-fs_cadet.h"
+
+
+/**
+ * After how long do we reset connections without replies?
+ */
+#define CLIENT_RETRY_TIMEOUT \
+  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
+
+
+/**
+ * Handle for a cadet to another peer.
+ */
+struct CadetHandle;
+
+
+/**
+ * Handle for a request that is going out via cadet API.
+ */
+struct GSF_CadetRequest
+{
+  /**
+   * DLL.
+   */
+  struct GSF_CadetRequest *next;
+
+  /**
+   * DLL.
+   */
+  struct GSF_CadetRequest *prev;
+
+  /**
+   * Which cadet is this request associated with?
+   */
+  struct CadetHandle *mh;
+
+  /**
+   * Function to call with the result.
+   */
+  GSF_CadetReplyProcessor proc;
+
+  /**
+   * Closure for @e proc
+   */
+  void *proc_cls;
+
+  /**
+   * Query to transmit to the other peer.
+   */
+  struct GNUNET_HashCode query;
+
+  /**
+   * Desired type for the reply.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Did we transmit this request already? #GNUNET_YES if we are
+   * in the 'waiting_map', #GNUNET_NO if we are in the 'pending' DLL.
+   */
+  int was_transmitted;
+};
+
+
+/**
+ * Handle for a cadet to another peer.
+ */
+struct CadetHandle
+{
+  /**
+   * Head of DLL of pending requests on this cadet.
+   */
+  struct GSF_CadetRequest *pending_head;
+
+  /**
+   * Tail of DLL of pending requests on this cadet.
+   */
+  struct GSF_CadetRequest *pending_tail;
+
+  /**
+   * Map from query to `struct GSF_CadetRequest`s waiting for
+   * a reply.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *waiting_map;
+
+  /**
+   * Channel to the other peer.
+   */
+  struct GNUNET_CADET_Channel *channel;
+
+  /**
+   * Which peer does this cadet go to?
+   */
+  struct GNUNET_PeerIdentity target;
+
+  /**
+   * Task to kill inactive cadets (we keep them around for
+   * a few seconds to give the application a chance to give
+   * us another query).
+   */
+  struct GNUNET_SCHEDULER_Task *timeout_task;
+
+  /**
+   * Task to reset cadets that had errors (asynchronously,
+   * as we may not be able to do it immediately during a
+   * callback from the cadet API).
+   */
+  struct GNUNET_SCHEDULER_Task *reset_task;
+};
+
+
+/**
+ * Cadet channel for creating outbound channels.
+ */
+struct GNUNET_CADET_Handle *cadet_handle;
+
+/**
+ * Map from peer identities to 'struct CadetHandles' with cadet
+ * channels to those peers.
+ */
+struct GNUNET_CONTAINER_MultiPeerMap *cadet_map;
+
+
+/* ********************* client-side code ************************* */
+
+
+/**
+ * Transmit pending requests via the cadet.
+ *
+ * @param cls `struct CadetHandle` to process
+ */
+static void
+transmit_pending (void *cls);
+
+
+/**
+ * Iterator called on each entry in a waiting map to
+ * move it back to the pending list.
+ *
+ * @param cls the `struct CadetHandle`
+ * @param key the key of the entry in the map (the query)
+ * @param value the `struct GSF_CadetRequest` to move to pending
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+move_to_pending (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct CadetHandle *mh = cls;
+  struct GSF_CadetRequest *sr = value;
+
+  GNUNET_assert (
+    GNUNET_YES ==
+    GNUNET_CONTAINER_multihashmap_remove (mh->waiting_map, key, value));
+  GNUNET_CONTAINER_DLL_insert (mh->pending_head, mh->pending_tail, sr);
+  sr->was_transmitted = GNUNET_NO;
+  return GNUNET_YES;
+}
+
+
+/**
+ * Functions with this signature are called whenever a complete reply
+ * is received.
+ *
+ * @param cls closure with the `struct CadetHandle`
+ * @param srm the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ */
+static int
+check_reply (void *cls, const struct CadetReplyMessage *srm)
+{
+  /* We check later... */
+  return GNUNET_OK;
+}
+
+
+/**
+ * Task called when it is time to reset an cadet.
+ *
+ * @param cls the `struct CadetHandle` to tear down
+ */
+static void
+reset_cadet_task (void *cls);
+
+
+/**
+ * We had a serious error, tear down and re-create cadet from scratch,
+ * but do so asynchronously.
+ *
+ * @param mh cadet to reset
+ */
+static void
+reset_cadet_async (struct CadetHandle *mh)
+{
+  if (NULL != mh->reset_task)
+    GNUNET_SCHEDULER_cancel (mh->reset_task);
+  mh->reset_task = GNUNET_SCHEDULER_add_now (&reset_cadet_task, mh);
+}
+
+
+/**
+ * Closure for handle_reply().
+ */
+struct HandleReplyClosure
+{
+  /**
+   * Reply payload.
+   */
+  const void *data;
+
+  /**
+   * Expiration time for the block.
+   */
+  struct GNUNET_TIME_Absolute expiration;
+
+  /**
+   * Number of bytes in @e data.
+   */
+  size_t data_size;
+
+  /**
+   * Type of the block.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Did we have a matching query?
+   */
+  int found;
+};
+
+
+/**
+ * Iterator called on each entry in a waiting map to
+ * process a result.
+ *
+ * @param cls the `struct HandleReplyClosure`
+ * @param key the key of the entry in the map (the query)
+ * @param value the `struct GSF_CadetRequest` to handle result for
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+process_reply (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct HandleReplyClosure *hrc = cls;
+  struct GSF_CadetRequest *sr = value;
+
+  sr->proc (sr->proc_cls,
+            hrc->type,
+            hrc->expiration,
+            hrc->data_size,
+            hrc->data);
+  sr->proc = NULL;
+  GSF_cadet_query_cancel (sr);
+  hrc->found = GNUNET_YES;
+  return GNUNET_YES;
+}
+
+
+/**
+ * Iterator called on each entry in a waiting map to
+ * call the 'proc' continuation and release associated
+ * resources.
+ *
+ * @param cls the `struct CadetHandle`
+ * @param key the key of the entry in the map (the query)
+ * @param value the `struct GSF_CadetRequest` to clean up
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+free_waiting_entry (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct GSF_CadetRequest *sr = value;
+
+  GSF_cadet_query_cancel (sr);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Functions with this signature are called whenever a complete reply
+ * is received.
+ *
+ * @param cls closure with the `struct CadetHandle`
+ * @param srm the actual message
+ */
+static void
+handle_reply (void *cls, const struct CadetReplyMessage *srm)
+{
+  struct CadetHandle *mh = cls;
+  struct HandleReplyClosure hrc;
+  uint16_t msize;
+  enum GNUNET_BLOCK_Type type;
+  struct GNUNET_HashCode query;
+
+  msize = ntohs (srm->header.size) - sizeof(struct CadetReplyMessage);
+  type = (enum GNUNET_BLOCK_Type) ntohl (srm->type);
+  if (GNUNET_YES !=
+      GNUNET_BLOCK_get_key (GSF_block_ctx, type, &srm[1], msize, &query))
+  {
+    GNUNET_break_op (0);
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_WARNING,
+      "Received bogus reply of type %u with %u bytes via cadet from peer %s\n",
+      type,
+      msize,
+      GNUNET_i2s (&mh->target));
+    reset_cadet_async (mh);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received reply `%s' via cadet from peer %s\n",
+              GNUNET_h2s (&query),
+              GNUNET_i2s (&mh->target));
+  GNUNET_CADET_receive_done (mh->channel);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# replies received via cadet"),
+                            1,
+                            GNUNET_NO);
+  hrc.data = &srm[1];
+  hrc.data_size = msize;
+  hrc.expiration = GNUNET_TIME_absolute_ntoh (srm->expiration);
+  hrc.type = type;
+  hrc.found = GNUNET_NO;
+  GNUNET_CONTAINER_multihashmap_get_multiple (mh->waiting_map,
+                                              &query,
+                                              &process_reply,
+                                              &hrc);
+  if (GNUNET_NO == hrc.found)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# replies received via cadet dropped"),
+                              1,
+                              GNUNET_NO);
+  }
+}
+
+
+/**
+ * Function called by cadet when a client disconnects.
+ * Cleans up our `struct CadetClient` of that channel.
+ *
+ * @param cls our `struct CadetClient`
+ * @param channel channel of the disconnecting client
+ */
+static void
+disconnect_cb (void *cls, const struct GNUNET_CADET_Channel *channel)
+{
+  struct CadetHandle *mh = cls;
+  struct GSF_CadetRequest *sr;
+
+  if (NULL == mh->channel)
+    return; /* being destroyed elsewhere */
+  GNUNET_assert (channel == mh->channel);
+  mh->channel = NULL;
+  while (NULL != (sr = mh->pending_head))
+    GSF_cadet_query_cancel (sr);
+  /* first remove `mh` from the `cadet_map`, so that if the
+     callback from `free_waiting_entry()` happens to re-issue
+     the request, we don't immediately have it back in the
+     `waiting_map`. */
+  GNUNET_assert (GNUNET_OK == GNUNET_CONTAINER_multipeermap_remove (cadet_map,
+                                                                    &mh->target,
+                                                                    mh));
+  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map,
+                                         &free_waiting_entry,
+                                         mh);
+  if (NULL != mh->timeout_task)
+    GNUNET_SCHEDULER_cancel (mh->timeout_task);
+  if (NULL != mh->reset_task)
+    GNUNET_SCHEDULER_cancel (mh->reset_task);
+  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (mh->waiting_map));
+  GNUNET_CONTAINER_multihashmap_destroy (mh->waiting_map);
+  GNUNET_free (mh);
+}
+
+
+/**
+ * Function called whenever an MQ-channel's transmission window size changes.
+ *
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative..
+ */
+static void
+window_change_cb (void *cls,
+                  const struct GNUNET_CADET_Channel *channel,
+                  int window_size)
+{
+  /* FIXME: for flow control, implement? */
+#if 0
+  /* Something like this instead of the GNUNET_MQ_notify_sent() in
+     transmit_pending() might be good (once the window change CB works...) */
+  if (0 < window_size) /* test needed? */
+    transmit_pending (mh);
+#endif
+}
+
+
+/**
+ * We had a serious error, tear down and re-create cadet from scratch.
+ *
+ * @param mh cadet to reset
+ */
+static void
+reset_cadet (struct CadetHandle *mh)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Resetting cadet channel to %s\n",
+              GNUNET_i2s (&mh->target));
+  if (NULL != mh->channel)
+  {
+    GNUNET_CADET_channel_destroy (mh->channel);
+    mh->channel = NULL;
+  }
+  GNUNET_CONTAINER_multihashmap_iterate (mh->waiting_map, &move_to_pending, mh);
+  {
+    struct GNUNET_MQ_MessageHandler handlers[] =
+    { GNUNET_MQ_hd_var_size (reply,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY,
+                             struct CadetReplyMessage,
+                             mh),
+      GNUNET_MQ_handler_end () };
+    struct GNUNET_HashCode port;
+
+    GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
+                        strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
+                        &port);
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
+                                               mh,
+                                               &mh->target,
+                                               &port,
+                                               &window_change_cb,
+                                               &disconnect_cb,
+                                               handlers);
+  }
+  transmit_pending (mh);
+}
+
+
+/**
+ * Task called when it is time to destroy an inactive cadet channel.
+ *
+ * @param cls the `struct CadetHandle` to tear down
+ */
+static void
+cadet_timeout (void *cls)
+{
+  struct CadetHandle *mh = cls;
+  struct GNUNET_CADET_Channel *tun;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Timeout on cadet channel to %s\n",
+              GNUNET_i2s (&mh->target));
+  mh->timeout_task = NULL;
+  tun = mh->channel;
+  mh->channel = NULL;
+  if (NULL != tun)
+    GNUNET_CADET_channel_destroy (tun);
+}
+
+
+/**
+ * Task called when it is time to reset an cadet.
+ *
+ * @param cls the `struct CadetHandle` to tear down
+ */
+static void
+reset_cadet_task (void *cls)
+{
+  struct CadetHandle *mh = cls;
+
+  mh->reset_task = NULL;
+  reset_cadet (mh);
+}
+
+
+/**
+ * Transmit pending requests via the cadet.
+ *
+ * @param cls `struct CadetHandle` to process
+ */
+static void
+transmit_pending (void *cls)
+{
+  struct CadetHandle *mh = cls;
+  struct GNUNET_MQ_Handle *mq = GNUNET_CADET_get_mq (mh->channel);
+  struct GSF_CadetRequest *sr;
+  struct GNUNET_MQ_Envelope *env;
+  struct CadetQueryMessage *sqm;
+
+  if ((0 != GNUNET_MQ_get_length (mq)) || (NULL == (sr = mh->pending_head)))
+    return;
+  GNUNET_CONTAINER_DLL_remove (mh->pending_head, mh->pending_tail, sr);
+  GNUNET_assert (GNUNET_OK == GNUNET_CONTAINER_multihashmap_put (
+                   mh->waiting_map,
+                   &sr->query,
+                   sr,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+  sr->was_transmitted = GNUNET_YES;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending query for %s via cadet to %s\n",
+              GNUNET_h2s (&sr->query),
+              GNUNET_i2s (&mh->target));
+  env = GNUNET_MQ_msg (sqm, GNUNET_MESSAGE_TYPE_FS_CADET_QUERY);
+  GNUNET_MQ_env_set_options (env,
+                             GNUNET_MQ_PREF_GOODPUT
+                             | GNUNET_MQ_PREF_CORK_ALLOWED
+                             | GNUNET_MQ_PREF_OUT_OF_ORDER);
+  sqm->type = htonl (sr->type);
+  sqm->query = sr->query;
+  GNUNET_MQ_notify_sent (env, &transmit_pending, mh);
+  GNUNET_MQ_send (mq, env);
+}
+
+
+/**
+ * Get (or create) a cadet to talk to the given peer.
+ *
+ * @param target peer we want to communicate with
+ */
+static struct CadetHandle *
+get_cadet (const struct GNUNET_PeerIdentity *target)
+{
+  struct CadetHandle *mh;
+
+  mh = GNUNET_CONTAINER_multipeermap_get (cadet_map, target);
+  if (NULL != mh)
+  {
+    if (NULL != mh->timeout_task)
+    {
+      GNUNET_SCHEDULER_cancel (mh->timeout_task);
+      mh->timeout_task = NULL;
+    }
+    return mh;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating cadet channel to %s\n",
+              GNUNET_i2s (target));
+  mh = GNUNET_new (struct CadetHandle);
+  mh->reset_task =
+    GNUNET_SCHEDULER_add_delayed (CLIENT_RETRY_TIMEOUT, &reset_cadet_task, mh);
+  mh->waiting_map = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_YES);
+  mh->target = *target;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multipeermap_put (
+                   cadet_map,
+                   &mh->target,
+                   mh,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  {
+    struct GNUNET_MQ_MessageHandler handlers[] =
+    { GNUNET_MQ_hd_var_size (reply,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY,
+                             struct CadetReplyMessage,
+                             mh),
+      GNUNET_MQ_handler_end () };
+    struct GNUNET_HashCode port;
+
+    GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
+                        strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
+                        &port);
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
+                                               mh,
+                                               &mh->target,
+                                               &port,
+                                               &window_change_cb,
+                                               &disconnect_cb,
+                                               handlers);
+  }
+  return mh;
+}
+
+
+/**
+ * Look for a block by directly contacting a particular peer.
+ *
+ * @param target peer that should have the block
+ * @param query hash to query for the block
+ * @param type desired type for the block
+ * @param proc function to call with result
+ * @param proc_cls closure for @a proc
+ * @return handle to cancel the operation
+ */
+struct GSF_CadetRequest *
+GSF_cadet_query (const struct GNUNET_PeerIdentity *target,
+                 const struct GNUNET_HashCode *query,
+                 enum GNUNET_BLOCK_Type type,
+                 GSF_CadetReplyProcessor proc,
+                 void *proc_cls)
+{
+  struct CadetHandle *mh;
+  struct GSF_CadetRequest *sr;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Preparing to send query for %s via cadet to %s\n",
+              GNUNET_h2s (query),
+              GNUNET_i2s (target));
+  mh = get_cadet (target);
+  sr = GNUNET_new (struct GSF_CadetRequest);
+  sr->mh = mh;
+  sr->proc = proc;
+  sr->proc_cls = proc_cls;
+  sr->type = type;
+  sr->query = *query;
+  GNUNET_CONTAINER_DLL_insert (mh->pending_head, mh->pending_tail, sr);
+  transmit_pending (mh);
+  return sr;
+}
+
+
+/**
+ * Cancel an active request; must not be called after 'proc'
+ * was called.
+ *
+ * @param sr request to cancel
+ */
+void
+GSF_cadet_query_cancel (struct GSF_CadetRequest *sr)
+{
+  struct CadetHandle *mh = sr->mh;
+  GSF_CadetReplyProcessor p;
+
+  p = sr->proc;
+  sr->proc = NULL;
+  if (NULL != p)
+  {
+    /* signal failure / cancellation to callback */
+    p (sr->proc_cls, GNUNET_BLOCK_TYPE_ANY, GNUNET_TIME_UNIT_ZERO_ABS, 0, NULL);
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Cancelled query for %s via cadet to %s\n",
+              GNUNET_h2s (&sr->query),
+              GNUNET_i2s (&sr->mh->target));
+  if (GNUNET_YES == sr->was_transmitted)
+    GNUNET_assert (
+      GNUNET_OK ==
+      GNUNET_CONTAINER_multihashmap_remove (mh->waiting_map, &sr->query, sr));
+  else
+    GNUNET_CONTAINER_DLL_remove (mh->pending_head, mh->pending_tail, sr);
+  GNUNET_free (sr);
+  if ((0 == GNUNET_CONTAINER_multihashmap_size (mh->waiting_map)) &&
+      (NULL == mh->pending_head))
+    mh->timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
+                                                     &cadet_timeout,
+                                                     mh);
+}
+
+
+/**
+ * Function called on each active cadets to shut them down.
+ *
+ * @param cls NULL
+ * @param key target peer, unused
+ * @param value the `struct CadetHandle` to destroy
+ * @return #GNUNET_YES (continue to iterate)
+ */
+int
+GSF_cadet_release_clients (void *cls,
+                           const struct GNUNET_PeerIdentity *key,
+                           void *value)
+{
+  struct CadetHandle *mh = value;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Timeout on cadet channel to %s\n",
+              GNUNET_i2s (&mh->target));
+  if (NULL != mh->channel)
+  {
+    struct GNUNET_CADET_Channel *channel = mh->channel;
+
+    mh->channel = NULL;
+    GNUNET_CADET_channel_destroy (channel);
+  }
+  if (NULL != mh->reset_task)
+  {
+    GNUNET_SCHEDULER_cancel (mh->reset_task);
+    mh->reset_task = NULL;
+  }
+  return GNUNET_YES;
+}
+
+
+/* end of gnunet-service-fs_cadet_client.c */
diff --git a/src/service/fs/gnunet-service-fs_cadet_server.c b/src/service/fs/gnunet-service-fs_cadet_server.c
new file mode 100644
index 000000000..8bfe91cf0
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_cadet_server.c
@@ -0,0 +1,545 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012, 2013, 2017 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_cadet_server.c
+ * @brief non-anonymous file-transfer
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - PORT is set to old application type, unsure if we should keep
+ *   it that way (fine for now)
+ */
+#include "platform.h"
+#include "gnunet_constants.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_cadet_service.h"
+#include "gnunet_protocols.h"
+#include "gnunet_applications.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_indexing.h"
+#include "gnunet-service-fs_cadet.h"
+
+/**
+ * After how long do we termiante idle connections?
+ */
+#define IDLE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2)
+
+
+/**
+ * A message in the queue to be written to the cadet.
+ */
+struct WriteQueueItem
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct WriteQueueItem *next;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct WriteQueueItem *prev;
+
+  /**
+   * Number of bytes of payload, allocated at the end of this struct.
+   */
+  size_t msize;
+};
+
+
+/**
+ * Information we keep around for each active cadeting client.
+ */
+struct CadetClient
+{
+  /**
+   * DLL
+   */
+  struct CadetClient *next;
+
+  /**
+   * DLL
+   */
+  struct CadetClient *prev;
+
+  /**
+   * Channel for communication.
+   */
+  struct GNUNET_CADET_Channel *channel;
+
+  /**
+   * Head of write queue.
+   */
+  struct WriteQueueItem *wqi_head;
+
+  /**
+   * Tail of write queue.
+   */
+  struct WriteQueueItem *wqi_tail;
+
+  /**
+   * Current active request to the datastore, if we have one pending.
+   */
+  struct GNUNET_DATASTORE_QueueEntry *qe;
+
+  /**
+   * Task that is scheduled to asynchronously terminate the connection.
+   */
+  struct GNUNET_SCHEDULER_Task *terminate_task;
+
+  /**
+   * Task that is scheduled to terminate idle connections.
+   */
+  struct GNUNET_SCHEDULER_Task *timeout_task;
+
+  /**
+   * Size of the last write that was initiated.
+   */
+  size_t reply_size;
+};
+
+
+/**
+ * Listen port for incoming requests.
+ */
+static struct GNUNET_CADET_Port *cadet_port;
+
+/**
+ * Head of DLL of cadet clients.
+ */
+static struct CadetClient *sc_head;
+
+/**
+ * Tail of DLL of cadet clients.
+ */
+static struct CadetClient *sc_tail;
+
+/**
+ * Number of active cadet clients in the 'sc_*'-DLL.
+ */
+static unsigned int sc_count;
+
+/**
+ * Maximum allowed number of cadet clients.
+ */
+static unsigned long long sc_count_max;
+
+
+/**
+ * Task run to asynchronously terminate the cadet due to timeout.
+ *
+ * @param cls the 'struct CadetClient'
+ */
+static void
+timeout_cadet_task (void *cls)
+{
+  struct CadetClient *sc = cls;
+  struct GNUNET_CADET_Channel *tun;
+
+  sc->timeout_task = NULL;
+  tun = sc->channel;
+  sc->channel = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Timeout for inactive cadet client %p\n",
+              sc);
+  GNUNET_CADET_channel_destroy (tun);
+}
+
+
+/**
+ * Reset the timeout for the cadet client (due to activity).
+ *
+ * @param sc client handle to reset timeout for
+ */
+static void
+refresh_timeout_task (struct CadetClient *sc)
+{
+  if (NULL != sc->timeout_task)
+    GNUNET_SCHEDULER_cancel (sc->timeout_task);
+  sc->timeout_task = GNUNET_SCHEDULER_add_delayed (IDLE_TIMEOUT,
+                                                   &timeout_cadet_task,
+                                                   sc);
+}
+
+
+/**
+ * Check if we are done with the write queue, and if so tell CADET
+ * that we are ready to read more.
+ *
+ * @param cls where to process the write queue
+ */
+static void
+continue_writing (void *cls)
+{
+  struct CadetClient *sc = cls;
+  struct GNUNET_MQ_Handle *mq;
+
+  mq = GNUNET_CADET_get_mq (sc->channel);
+  if (0 != GNUNET_MQ_get_length (mq))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Write pending, waiting for it to complete\n");
+    return;
+  }
+  refresh_timeout_task (sc);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished processing cadet request from client %p, ready to receive the next one\n",
+              sc);
+  GNUNET_CADET_receive_done (sc->channel);
+}
+
+
+/**
+ * Process a datum that was stored in the datastore.
+ *
+ * @param cls closure with the `struct CadetClient` which sent the query
+ * @param key key for the content
+ * @param size number of bytes in @a data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+handle_datastore_reply (void *cls,
+                        const struct GNUNET_HashCode *key,
+                        size_t size,
+                        const void *data,
+                        enum GNUNET_BLOCK_Type type,
+                        uint32_t priority,
+                        uint32_t anonymity,
+                        uint32_t replication,
+                        struct GNUNET_TIME_Absolute expiration,
+                        uint64_t uid)
+{
+  struct CadetClient *sc = cls;
+  size_t msize = size + sizeof(struct CadetReplyMessage);
+  struct GNUNET_MQ_Envelope *env;
+  struct CadetReplyMessage *srm;
+
+  sc->qe = NULL;
+  if (NULL == data)
+  {
+    /* no result, this should not really happen, as for
+       non-anonymous routing only peers that HAVE the
+       answers should be queried; OTOH, this is not a
+       hard error as we might have had the answer in the
+       past and the user might have unindexed it. Hence
+       we log at level "INFO" for now. */if (NULL == key)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Have no answer and the query was NULL\n");
+    }
+    else
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Have no answer for query `%s'\n",
+                  GNUNET_h2s (key));
+    }
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# queries received via CADET not answered"),
+                              1,
+                              GNUNET_NO);
+    continue_writing (sc);
+    return;
+  }
+  if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Performing on-demand encoding for query %s\n",
+                GNUNET_h2s (key));
+    if (GNUNET_OK !=
+        GNUNET_FS_handle_on_demand_block (key,
+                                          size,
+                                          data,
+                                          type,
+                                          priority,
+                                          anonymity,
+                                          replication,
+                                          expiration,
+                                          uid,
+                                          &handle_datastore_reply,
+                                          sc))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "On-demand encoding request failed\n");
+      continue_writing (sc);
+    }
+    return;
+  }
+  if (msize > GNUNET_MAX_MESSAGE_SIZE)
+  {
+    GNUNET_break (0);
+    continue_writing (sc);
+    return;
+  }
+  GNUNET_break (GNUNET_BLOCK_TYPE_ANY != type);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Starting transmission of %u byte reply of type %d for query `%s' via cadet to %p\n",
+              (unsigned int) size,
+              (unsigned int) type,
+              GNUNET_h2s (key),
+              sc);
+  env = GNUNET_MQ_msg_extra (srm,
+                             size,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_REPLY);
+  srm->type = htonl (type);
+  srm->expiration = GNUNET_TIME_absolute_hton (expiration);
+  GNUNET_memcpy (&srm[1],
+                 data,
+                 size);
+  GNUNET_MQ_notify_sent (env,
+                         &continue_writing,
+                         sc);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Blocks transferred via cadet"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (sc->channel),
+                  env);
+}
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete query message is received.
+ *
+ * @param cls closure with the `struct CadetClient`
+ * @param sqm the actual message
+ */
+static void
+handle_request (void *cls,
+                const struct CadetQueryMessage *sqm)
+{
+  struct CadetClient *sc = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received query for `%s' via cadet from client %p\n",
+              GNUNET_h2s (&sqm->query),
+              sc);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# queries received via cadet"),
+                            1,
+                            GNUNET_NO);
+  refresh_timeout_task (sc);
+  sc->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
+                                     0 /* next_uid */,
+                                     false /* random */,
+                                     &sqm->query,
+                                     ntohl (sqm->type),
+                                     0 /* priority */,
+                                     GSF_datastore_queue_size,
+                                     &handle_datastore_reply,
+                                     sc);
+  if (NULL == sc->qe)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Queueing request with datastore failed (queue full?)\n");
+    continue_writing (sc);
+  }
+}
+
+
+/**
+ * Functions of this type are called upon new cadet connection from other peers.
+ *
+ * @param cls the closure from GNUNET_CADET_connect
+ * @param channel the channel representing the cadet
+ * @param initiator the identity of the peer who wants to establish a cadet
+ *            with us; NULL on binding error
+ * @return initial channel context (our `struct CadetClient`)
+ */
+static void *
+connect_cb (void *cls,
+            struct GNUNET_CADET_Channel *channel,
+            const struct GNUNET_PeerIdentity *initiator)
+{
+  struct CadetClient *sc;
+
+  GNUNET_assert (NULL != channel);
+  if (sc_count >= sc_count_max)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# cadet client connections rejected"),
+                              1,
+                              GNUNET_NO);
+    GNUNET_CADET_channel_destroy (channel);
+    return NULL;
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# cadet connections active"),
+                            1,
+                            GNUNET_NO);
+  sc = GNUNET_new (struct CadetClient);
+  sc->channel = channel;
+  GNUNET_CONTAINER_DLL_insert (sc_head,
+                               sc_tail,
+                               sc);
+  sc_count++;
+  refresh_timeout_task (sc);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Accepting inbound cadet connection from `%s' as client %p\n",
+              GNUNET_i2s (initiator),
+              sc);
+  return sc;
+}
+
+
+/**
+ * Function called by cadet when a client disconnects.
+ * Cleans up our `struct CadetClient` of that channel.
+ *
+ * @param cls  our `struct CadetClient`
+ * @param channel channel of the disconnecting client
+ */
+static void
+disconnect_cb (void *cls,
+               const struct GNUNET_CADET_Channel *channel)
+{
+  struct CadetClient *sc = cls;
+  struct WriteQueueItem *wqi;
+
+  if (NULL == sc)
+    return;
+  sc->channel = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Terminating cadet connection with client %p\n",
+              sc);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# cadet connections active"), -1,
+                            GNUNET_NO);
+  if (NULL != sc->terminate_task)
+    GNUNET_SCHEDULER_cancel (sc->terminate_task);
+  if (NULL != sc->timeout_task)
+    GNUNET_SCHEDULER_cancel (sc->timeout_task);
+  if (NULL != sc->qe)
+    GNUNET_DATASTORE_cancel (sc->qe);
+  while (NULL != (wqi = sc->wqi_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (sc->wqi_head,
+                                 sc->wqi_tail,
+                                 wqi);
+    GNUNET_free (wqi);
+  }
+  GNUNET_CONTAINER_DLL_remove (sc_head,
+                               sc_tail,
+                               sc);
+  sc_count--;
+  GNUNET_free (sc);
+}
+
+
+/**
+ * Function called whenever an MQ-channel's transmission window size changes.
+ *
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
+ *
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative..
+ */
+static void
+window_change_cb (void *cls,
+                  const struct GNUNET_CADET_Channel *channel,
+                  int window_size)
+{
+  /* FIXME: could do flow control here... */
+}
+
+
+/**
+ * Initialize subsystem for non-anonymous file-sharing.
+ */
+void
+GSF_cadet_start_server ()
+{
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (request,
+                             GNUNET_MESSAGE_TYPE_FS_CADET_QUERY,
+                             struct CadetQueryMessage,
+                             NULL),
+    GNUNET_MQ_handler_end ()
+  };
+  struct GNUNET_HashCode port;
+
+  if (GNUNET_YES !=
+      GNUNET_CONFIGURATION_get_value_number (GSF_cfg,
+                                             "fs",
+                                             "MAX_CADET_CLIENTS",
+                                             &sc_count_max))
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initializing cadet FS server with a limit of %llu connections\n",
+              sc_count_max);
+  cadet_map = GNUNET_CONTAINER_multipeermap_create (16, GNUNET_YES);
+  cadet_handle = GNUNET_CADET_connect (GSF_cfg);
+  GNUNET_assert (NULL != cadet_handle);
+  GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
+                      strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
+                      &port);
+  cadet_port = GNUNET_CADET_open_port (cadet_handle,
+                                       &port,
+                                       &connect_cb,
+                                       NULL,
+                                       &window_change_cb,
+                                       &disconnect_cb,
+                                       handlers);
+}
+
+
+/**
+ * Shutdown subsystem for non-anonymous file-sharing.
+ */
+void
+GSF_cadet_stop_server ()
+{
+  GNUNET_CONTAINER_multipeermap_iterate (cadet_map,
+                                         &GSF_cadet_release_clients,
+                                         NULL);
+  GNUNET_CONTAINER_multipeermap_destroy (cadet_map);
+  cadet_map = NULL;
+  if (NULL != cadet_port)
+  {
+    GNUNET_CADET_close_port (cadet_port);
+    cadet_port = NULL;
+  }
+  if (NULL != cadet_handle)
+  {
+    GNUNET_CADET_disconnect (cadet_handle);
+    cadet_handle = NULL;
+  }
+  GNUNET_assert (NULL == sc_head);
+  GNUNET_assert (0 == sc_count);
+}
+
+
+/* end of gnunet-service-fs_cadet.c */
diff --git a/src/service/fs/gnunet-service-fs_cp.c b/src/service/fs/gnunet-service-fs_cp.c
new file mode 100644
index 000000000..df934feaf
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_cp.c
@@ -0,0 +1,1667 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2016 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/gnunet-service-fs_cp.c
+ * @brief API to handle 'connected peers'
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_load_lib.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_cp.h"
+#include "gnunet-service-fs_pe.h"
+#include "gnunet-service-fs_pr.h"
+#include "gnunet-service-fs_push.h"
+#include "gnunet_peerstore_service.h"
+
+
+/**
+ * Ratio for moving average delay calculation.  The previous
+ * average goes in with a factor of (n-1) into the calculation.
+ * Must be > 0.
+ */
+#define RUNAVG_DELAY_N 16
+
+/**
+ * How often do we flush respect values to disk?
+ */
+#define RESPECT_FLUSH_FREQ GNUNET_TIME_relative_multiply ( \
+          GNUNET_TIME_UNIT_MINUTES, 5)
+
+/**
+ * After how long do we discard a reply?
+ */
+#define REPLY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, \
+                                                     2)
+
+/**
+ * Collect an instance number of statistics?  May cause excessive IPC.
+ */
+#define INSANE_STATISTICS GNUNET_NO
+
+
+/**
+ * Handle to cancel a transmission request.
+ */
+struct GSF_PeerTransmitHandle
+{
+  /**
+   * Kept in a doubly-linked list.
+   */
+  struct GSF_PeerTransmitHandle *next;
+
+  /**
+   * Kept in a doubly-linked list.
+   */
+  struct GSF_PeerTransmitHandle *prev;
+
+  /**
+   * Time when this transmission request was issued.
+   */
+  struct GNUNET_TIME_Absolute transmission_request_start_time;
+
+  /**
+   * Envelope with the actual message.
+   */
+  struct GNUNET_MQ_Envelope *env;
+
+  /**
+   * Peer this request targets.
+   */
+  struct GSF_ConnectedPeer *cp;
+
+  /**
+   * #GNUNET_YES if this is a query, #GNUNET_NO for content.
+   */
+  int is_query;
+
+  /**
+   * Priority of this request.
+   */
+  uint32_t priority;
+};
+
+
+/**
+ * Handle for an entry in our delay list.
+ */
+struct GSF_DelayedHandle
+{
+  /**
+   * Kept in a doubly-linked list.
+   */
+  struct GSF_DelayedHandle *next;
+
+  /**
+   * Kept in a doubly-linked list.
+   */
+  struct GSF_DelayedHandle *prev;
+
+  /**
+   * Peer this transmission belongs to.
+   */
+  struct GSF_ConnectedPeer *cp;
+
+  /**
+   * Envelope of the message that was delayed.
+   */
+  struct GNUNET_MQ_Envelope *env;
+
+  /**
+   * Task for the delay.
+   */
+  struct GNUNET_SCHEDULER_Task *delay_task;
+
+  /**
+   * Size of the message.
+   */
+  size_t msize;
+};
+
+
+/**
+ * Information per peer and request.
+ */
+struct PeerRequest
+{
+  /**
+   * Handle to generic request (generic: from peer or local client).
+   */
+  struct GSF_PendingRequest *pr;
+
+  /**
+   * Which specific peer issued this request?
+   */
+  struct GSF_ConnectedPeer *cp;
+
+  /**
+   * Task for asynchronous stopping of this request.
+   */
+  struct GNUNET_SCHEDULER_Task *kill_task;
+};
+
+
+/**
+ * A connected peer.
+ */
+struct GSF_ConnectedPeer
+{
+  /**
+   * Performance data for this peer.
+   */
+  struct GSF_PeerPerformanceData ppd;
+
+  /**
+   * Time until when we blocked this peer from migrating
+   * data to us.
+   */
+  struct GNUNET_TIME_Absolute last_migration_block;
+
+  /**
+   * Task scheduled to revive migration to this peer.
+   */
+  struct GNUNET_SCHEDULER_Task *mig_revive_task;
+
+  /**
+   * Messages (replies, queries, content migration) we would like to
+   * send to this peer in the near future.  Sorted by priority, head.
+   */
+  struct GSF_PeerTransmitHandle *pth_head;
+
+  /**
+   * Messages (replies, queries, content migration) we would like to
+   * send to this peer in the near future.  Sorted by priority, tail.
+   */
+  struct GSF_PeerTransmitHandle *pth_tail;
+
+  /**
+   * Messages (replies, queries, content migration) we would like to
+   * send to this peer in the near future.  Sorted by priority, head.
+   */
+  struct GSF_DelayedHandle *delayed_head;
+
+  /**
+   * Messages (replies, queries, content migration) we would like to
+   * send to this peer in the near future.  Sorted by priority, tail.
+   */
+  struct GSF_DelayedHandle *delayed_tail;
+
+  /**
+   * Task scheduled if we need to retry bandwidth reservation later.
+   */
+  struct GNUNET_SCHEDULER_Task *rc_delay_task;
+
+  /**
+   * Active requests from this neighbour, map of query to `struct PeerRequest`.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *request_map;
+
+  /**
+   * Handle for an active request for transmission to this
+   * peer.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Increase in traffic preference still to be submitted
+   * to the core service for this peer.
+   */
+  uint64_t inc_preference;
+
+  /**
+   * Number of entries in @e delayed_head DLL.
+   */
+  unsigned int delay_queue_size;
+
+  /**
+   * Respect rating for this peer on disk.
+   */
+  uint32_t disk_respect;
+
+  /**
+   * Which offset in @e last_p2p_replies will be updated next?
+   * (we go round-robin).
+   */
+  unsigned int last_p2p_replies_woff;
+
+  /**
+   * Which offset in @e last_client_replies will be updated next?
+   * (we go round-robin).
+   */
+  unsigned int last_client_replies_woff;
+
+  /**
+   * Current offset into @e last_request_times ring buffer.
+   */
+  unsigned int last_request_times_off;
+
+  /**
+   * Handle to the PEERSTORE iterate request for peer respect value
+   */
+  struct GNUNET_PEERSTORE_IterateContext *respect_iterate_req;
+};
+
+
+/**
+ * Map from peer identities to `struct GSF_ConnectPeer` entries.
+ */
+static struct GNUNET_CONTAINER_MultiPeerMap *cp_map;
+
+/**
+ * Handle to peerstore service.
+ */
+static struct GNUNET_PEERSTORE_Handle *peerstore;
+
+/**
+ * Task used to flush respect values to disk.
+ */
+static struct GNUNET_SCHEDULER_Task *fr_task;
+
+
+/**
+ * Update the latency information kept for the given peer.
+ *
+ * @param id peer record to update
+ * @param latency current latency value
+ */
+void
+GSF_update_peer_latency_ (const struct GNUNET_PeerIdentity *id,
+                          struct GNUNET_TIME_Relative latency)
+{
+  struct GSF_ConnectedPeer *cp;
+
+  cp = GSF_peer_get_ (id);
+  if (NULL == cp)
+    return; /* we're not yet connected at the core level, ignore */
+  GNUNET_LOAD_value_set_decline (cp->ppd.transmission_delay,
+                                 latency);
+}
+
+
+/**
+ * Return the performance data record for the given peer
+ *
+ * @param cp peer to query
+ * @return performance data record for the peer
+ */
+struct GSF_PeerPerformanceData *
+GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp)
+{
+  return &cp->ppd;
+}
+
+
+/**
+ * Core is ready to transmit to a peer, get the message.
+ *
+ * @param cp which peer to send a message to
+ */
+static void
+peer_transmit (struct GSF_ConnectedPeer *cp);
+
+
+/**
+ * If ready (bandwidth reserved), try to schedule transmission via
+ * core for the given handle.
+ *
+ * @param pth transmission handle to schedule
+ */
+static void
+schedule_transmission (struct GSF_PeerTransmitHandle *pth)
+{
+  struct GSF_ConnectedPeer *cp;
+  struct GNUNET_PeerIdentity target;
+
+  cp = pth->cp;
+  GNUNET_assert (0 != cp->ppd.pid);
+  GNUNET_PEER_resolve (cp->ppd.pid, &target);
+
+  peer_transmit (cp);
+}
+
+
+/**
+ * Core is ready to transmit to a peer, get the message.
+ *
+ * @param cp which peer to send a message to
+ */
+static void
+peer_transmit (struct GSF_ConnectedPeer *cp)
+{
+  struct GSF_PeerTransmitHandle *pth = cp->pth_head;
+  struct GSF_PeerTransmitHandle *pos;
+
+  if (NULL == pth)
+    return;
+  GNUNET_CONTAINER_DLL_remove (cp->pth_head,
+                               cp->pth_tail,
+                               pth);
+  if (GNUNET_YES == pth->is_query)
+  {
+    cp->ppd.last_request_times[(cp->last_request_times_off++)
+                               % MAX_QUEUE_PER_PEER] =
+      GNUNET_TIME_absolute_get ();
+    GNUNET_assert (0 < cp->ppd.pending_queries--);
+  }
+  else if (GNUNET_NO == pth->is_query)
+  {
+    GNUNET_assert (0 < cp->ppd.pending_replies--);
+  }
+  GNUNET_LOAD_update (cp->ppd.transmission_delay,
+                      GNUNET_TIME_absolute_get_duration
+                        (pth->transmission_request_start_time).rel_value_us);
+  GNUNET_MQ_send (cp->mq,
+                  pth->env);
+  GNUNET_free (pth);
+  if (NULL != (pos = cp->pth_head))
+  {
+    GNUNET_assert (pos != pth);
+    schedule_transmission (pos);
+  }
+}
+
+
+/**
+ * Function called by PEERSTORE with peer respect record
+ *
+ * @param cls handle to connected peer entry
+ * @param record peerstore record information
+ * @param emsg error message, or NULL if no errors
+ */
+static void
+peer_respect_cb (void *cls,
+                 const struct GNUNET_PEERSTORE_Record *record,
+                 const char *emsg)
+{
+  struct GSF_ConnectedPeer *cp = cls;
+
+  GNUNET_assert (NULL != cp->respect_iterate_req);
+  if ((NULL == record) && (NULL == emsg))
+  {
+    cp->respect_iterate_req = NULL;
+    return;
+  }
+  if ((NULL != record) &&
+      (sizeof(cp->disk_respect) == record->value_size))
+  {
+    cp->disk_respect = *((uint32_t *) record->value);
+    cp->ppd.respect += *((uint32_t *) record->value);
+  }
+  GSF_push_start_ (cp);
+  if (NULL != record)
+  {
+    GNUNET_PEERSTORE_iteration_stop (cp->respect_iterate_req);
+    cp->respect_iterate_req = NULL;
+    return;
+  }
+  GNUNET_PEERSTORE_iteration_next (cp->respect_iterate_req, 1);
+}
+
+
+/**
+ * Function called for each pending request whenever a new
+ * peer connects, giving us a chance to decide about submitting
+ * the existing request to the new peer.
+ *
+ * @param cls the `struct GSF_ConnectedPeer` of the new peer
+ * @param key query for the request
+ * @param pr handle to the pending request
+ * @return #GNUNET_YES to continue to iterate
+ */
+static int
+consider_peer_for_forwarding (void *cls,
+                              const struct GNUNET_HashCode *key,
+                              struct GSF_PendingRequest *pr)
+{
+  struct GSF_ConnectedPeer *cp = cls;
+  struct GNUNET_PeerIdentity pid;
+
+  if (GNUNET_YES !=
+      GSF_pending_request_test_active_ (pr))
+    return GNUNET_YES; /* request is not actually active, skip! */
+  GSF_connected_peer_get_identity_ (cp, &pid);
+  if (GNUNET_YES !=
+      GSF_pending_request_test_target_ (pr, &pid))
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# Loopback routes suppressed"),
+                              1,
+                              GNUNET_NO);
+    return GNUNET_YES;
+  }
+  GSF_plan_add_ (cp, pr);
+  return GNUNET_YES;
+}
+
+
+void *
+GSF_peer_connect_handler (void *cls,
+                          const struct GNUNET_PeerIdentity *peer,
+                          struct GNUNET_MQ_Handle *mq)
+{
+  struct GSF_ConnectedPeer *cp;
+
+  if (0 ==
+      GNUNET_memcmp (&GSF_my_id,
+                     peer))
+    return NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Connected to peer %s\n",
+              GNUNET_i2s (peer));
+  cp = GNUNET_new (struct GSF_ConnectedPeer);
+  cp->ppd.pid = GNUNET_PEER_intern (peer);
+  cp->ppd.peer = peer;
+  cp->mq = mq;
+  cp->ppd.transmission_delay = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_ZERO);
+
+  cp->request_map = GNUNET_CONTAINER_multihashmap_create (128,
+                                                          GNUNET_YES);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONTAINER_multipeermap_put (cp_map,
+                                                   GSF_connected_peer_get_identity2_ (
+                                                     cp),
+                                                   cp,
+                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  GNUNET_STATISTICS_set (GSF_stats,
+                         gettext_noop ("# peers connected"),
+                         GNUNET_CONTAINER_multipeermap_size (cp_map),
+                         GNUNET_NO);
+  cp->respect_iterate_req
+    = GNUNET_PEERSTORE_iteration_start (peerstore,
+                                      "fs",
+                                      peer,
+                                      "respect",
+                                      &peer_respect_cb,
+                                      cp);
+  GSF_iterate_pending_requests_ (&consider_peer_for_forwarding,
+                                 cp);
+  return cp;
+}
+
+
+/**
+ * It may be time to re-start migrating content to this
+ * peer.  Check, and if so, restart migration.
+ *
+ * @param cls the `struct GSF_ConnectedPeer`
+ */
+static void
+revive_migration (void *cls)
+{
+  struct GSF_ConnectedPeer *cp = cls;
+  struct GNUNET_TIME_Relative bt;
+
+  cp->mig_revive_task = NULL;
+  bt = GNUNET_TIME_absolute_get_remaining (cp->ppd.migration_blocked_until);
+  if (0 != bt.rel_value_us)
+  {
+    /* still time left... */
+    cp->mig_revive_task =
+      GNUNET_SCHEDULER_add_delayed (bt, &revive_migration, cp);
+    return;
+  }
+  GSF_push_start_ (cp);
+}
+
+
+struct GSF_ConnectedPeer *
+GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer)
+{
+  if (NULL == cp_map)
+    return NULL;
+  return GNUNET_CONTAINER_multipeermap_get (cp_map, peer);
+}
+
+
+/**
+ * Handle P2P #GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP message.
+ *
+ * @param cls closure, the `struct GSF_ConnectedPeer`
+ * @param msm the actual message
+ */
+void
+handle_p2p_migration_stop (void *cls,
+                           const struct MigrationStopMessage *msm)
+{
+  struct GSF_ConnectedPeer *cp = cls;
+  struct GNUNET_TIME_Relative bt;
+
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# migration stop messages received"),
+                            1, GNUNET_NO);
+  bt = GNUNET_TIME_relative_ntoh (msm->duration);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _ ("Migration of content to peer `%s' blocked for %s\n"),
+              GNUNET_i2s (cp->ppd.peer),
+              GNUNET_STRINGS_relative_time_to_string (bt, GNUNET_YES));
+  cp->ppd.migration_blocked_until = GNUNET_TIME_relative_to_absolute (bt);
+  if ((NULL == cp->mig_revive_task) &&
+      (NULL == cp->respect_iterate_req))
+  {
+    GSF_push_stop_ (cp);
+    cp->mig_revive_task =
+      GNUNET_SCHEDULER_add_delayed (bt,
+                                    &revive_migration, cp);
+  }
+}
+
+
+/**
+ * Free resources associated with the given peer request.
+ *
+ * @param peerreq request to free
+ */
+static void
+free_pending_request (struct PeerRequest *peerreq)
+{
+  struct GSF_ConnectedPeer *cp = peerreq->cp;
+  struct GSF_PendingRequestData *prd;
+
+  prd = GSF_pending_request_get_data_ (peerreq->pr);
+  if (NULL != peerreq->kill_task)
+  {
+    GNUNET_SCHEDULER_cancel (peerreq->kill_task);
+    peerreq->kill_task = NULL;
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# P2P searches active"),
+                            -1,
+                            GNUNET_NO);
+  GNUNET_break (GNUNET_YES ==
+                GNUNET_CONTAINER_multihashmap_remove (cp->request_map,
+                                                      &prd->query,
+                                                      peerreq));
+  GNUNET_free (peerreq);
+}
+
+
+/**
+ * Cancel all requests associated with the peer.
+ *
+ * @param cls unused
+ * @param query hash code of the request
+ * @param value the `struct GSF_PendingRequest`
+ * @return #GNUNET_YES (continue to iterate)
+ */
+static int
+cancel_pending_request (void *cls,
+                        const struct GNUNET_HashCode *query,
+                        void *value)
+{
+  struct PeerRequest *peerreq = value;
+  struct GSF_PendingRequest *pr = peerreq->pr;
+
+  free_pending_request (peerreq);
+  GSF_pending_request_cancel_ (pr,
+                               GNUNET_NO);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Free the given request.
+ *
+ * @param cls the request to free
+ */
+static void
+peer_request_destroy (void *cls)
+{
+  struct PeerRequest *peerreq = cls;
+  struct GSF_PendingRequest *pr = peerreq->pr;
+  struct GSF_PendingRequestData *prd;
+
+  peerreq->kill_task = NULL;
+  prd = GSF_pending_request_get_data_ (pr);
+  cancel_pending_request (NULL,
+                          &prd->query,
+                          peerreq);
+}
+
+
+/**
+ * The artificial delay is over, transmit the message now.
+ *
+ * @param cls the `struct GSF_DelayedHandle` with the message
+ */
+static void
+transmit_delayed_now (void *cls)
+{
+  struct GSF_DelayedHandle *dh = cls;
+  struct GSF_ConnectedPeer *cp = dh->cp;
+
+  GNUNET_CONTAINER_DLL_remove (cp->delayed_head,
+                               cp->delayed_tail,
+                               dh);
+  cp->delay_queue_size--;
+  GSF_peer_transmit_ (cp,
+                      GNUNET_NO,
+                      UINT32_MAX,
+                      dh->env);
+  GNUNET_free (dh);
+}
+
+
+/**
+ * Get the randomized delay a response should be subjected to.
+ *
+ * @return desired delay
+ */
+static struct GNUNET_TIME_Relative
+get_randomized_delay ()
+{
+  struct GNUNET_TIME_Relative ret;
+
+  ret =
+    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
+                                   GNUNET_CRYPTO_random_u32
+                                     (GNUNET_CRYPTO_QUALITY_WEAK,
+                                     2 * GSF_avg_latency.rel_value_us + 1));
+#if INSANE_STATISTICS
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop
+                              ("# artificial delays introduced (ms)"),
+                            ret.rel_value_us / 1000LL, GNUNET_NO);
+#endif
+  return ret;
+}
+
+
+/**
+ * Handle a reply to a pending request.  Also called if a request
+ * expires (then with data == NULL).  The handler may be called
+ * many times (depending on the request type), but will not be
+ * called during or after a call to GSF_pending_request_cancel
+ * and will also not be called anymore after a call signalling
+ * expiration.
+ *
+ * @param cls `struct PeerRequest` this is an answer for
+ * @param eval evaluation of the result
+ * @param pr handle to the original pending request
+ * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown"
+ * @param expiration when does @a data expire?
+ * @param last_transmission when did we last transmit a request for this block
+ * @param type type of the block
+ * @param data response data, NULL on request expiration
+ * @param data_len number of bytes in @a data
+ */
+static void
+handle_p2p_reply (void *cls,
+                  enum GNUNET_BLOCK_ReplyEvaluationResult eval,
+                  struct GSF_PendingRequest *pr,
+                  uint32_t reply_anonymity_level,
+                  struct GNUNET_TIME_Absolute expiration,
+                  struct GNUNET_TIME_Absolute last_transmission,
+                  enum GNUNET_BLOCK_Type type,
+                  const void *data,
+                  size_t data_len)
+{
+  struct PeerRequest *peerreq = cls;
+  struct GSF_ConnectedPeer *cp = peerreq->cp;
+  struct GSF_PendingRequestData *prd;
+  struct GNUNET_MQ_Envelope *env;
+  struct PutMessage *pm;
+  size_t msize;
+
+  GNUNET_assert (data_len + sizeof(struct PutMessage) <
+                 GNUNET_MAX_MESSAGE_SIZE);
+  GNUNET_assert (peerreq->pr == pr);
+  prd = GSF_pending_request_get_data_ (pr);
+  if (NULL == data)
+  {
+    free_pending_request (peerreq);
+    return;
+  }
+  GNUNET_break (GNUNET_BLOCK_TYPE_ANY != type);
+  if ( (prd->type != type) &&
+       (GNUNET_BLOCK_TYPE_ANY != prd->type) )
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              "# replies dropped due to type mismatch",
+                              1, GNUNET_NO);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Transmitting result for query `%s' to peer\n",
+              GNUNET_h2s (&prd->query));
+  GNUNET_STATISTICS_update (GSF_stats,
+                            "# replies received for other peers",
+                            1,
+                            GNUNET_NO);
+  msize = sizeof(struct PutMessage) + data_len;
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  if ( (UINT32_MAX != reply_anonymity_level) &&
+       (reply_anonymity_level > 1) )
+  {
+    if (reply_anonymity_level - 1 > GSF_cover_content_count)
+    {
+      GNUNET_STATISTICS_update (GSF_stats,
+                                "# replies dropped due to insufficient cover traffic",
+                                1, GNUNET_NO);
+      return;
+    }
+    GSF_cover_content_count -= (reply_anonymity_level - 1);
+  }
+
+  env = GNUNET_MQ_msg_extra (pm,
+                             data_len,
+                             GNUNET_MESSAGE_TYPE_FS_PUT);
+  pm->type = htonl (type);
+  pm->expiration = GNUNET_TIME_absolute_hton (expiration);
+  GNUNET_memcpy (&pm[1],
+                 data,
+                 data_len);
+  if ((UINT32_MAX != reply_anonymity_level) &&
+      (0 != reply_anonymity_level) &&
+      (GNUNET_YES == GSF_enable_randomized_delays))
+  {
+    struct GSF_DelayedHandle *dh;
+
+    dh = GNUNET_new (struct GSF_DelayedHandle);
+    dh->cp = cp;
+    dh->env = env;
+    dh->msize = msize;
+    GNUNET_CONTAINER_DLL_insert (cp->delayed_head,
+                                 cp->delayed_tail,
+                                 dh);
+    cp->delay_queue_size++;
+    dh->delay_task =
+      GNUNET_SCHEDULER_add_delayed (get_randomized_delay (),
+                                    &transmit_delayed_now,
+                                    dh);
+  }
+  else
+  {
+    GSF_peer_transmit_ (cp,
+                        GNUNET_NO,
+                        UINT32_MAX,
+                        env);
+  }
+  if (GNUNET_BLOCK_REPLY_OK_LAST != eval)
+    return;
+  if (NULL == peerreq->kill_task)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              "# P2P searches destroyed due to ultimate reply",
+                              1,
+                              GNUNET_NO);
+    peerreq->kill_task =
+      GNUNET_SCHEDULER_add_now (&peer_request_destroy,
+                                peerreq);
+  }
+}
+
+
+/**
+ * Increase the peer's respect by a value.
+ *
+ * @param cp which peer to change the respect value on
+ * @param value is the int value by which the
+ *  peer's credit is to be increased or decreased
+ * @returns the actual change in respect (positive or negative)
+ */
+static int
+change_peer_respect (struct GSF_ConnectedPeer *cp, int value)
+{
+  if (0 == value)
+    return 0;
+  GNUNET_assert (NULL != cp);
+  if (value > 0)
+  {
+    if (cp->ppd.respect + value < cp->ppd.respect)
+    {
+      value = UINT32_MAX - cp->ppd.respect;
+      cp->ppd.respect = UINT32_MAX;
+    }
+    else
+      cp->ppd.respect += value;
+  }
+  else
+  {
+    if (cp->ppd.respect < -value)
+    {
+      value = -cp->ppd.respect;
+      cp->ppd.respect = 0;
+    }
+    else
+      cp->ppd.respect += value;
+  }
+  return value;
+}
+
+
+/**
+ * We've received a request with the specified priority.  Bound it
+ * according to how much we respect the given peer.
+ *
+ * @param prio_in requested priority
+ * @param cp the peer making the request
+ * @return effective priority
+ */
+static int32_t
+bound_priority (uint32_t prio_in,
+                struct GSF_ConnectedPeer *cp)
+{
+#define N ((double) 128.0)
+  uint32_t ret;
+  double rret;
+  int ld;
+
+  ld = GSF_test_get_load_too_high_ (0);
+  if (GNUNET_SYSERR == ld)
+  {
+#if INSANE_STATISTICS
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                                ("# requests done for free (low load)"), 1,
+                              GNUNET_NO);
+#endif
+    return 0;                   /* excess resources */
+  }
+  if (prio_in > INT32_MAX)
+    prio_in = INT32_MAX;
+  ret = -change_peer_respect (cp, -(int) prio_in);
+  if (ret > 0)
+  {
+    if (ret > GSF_current_priorities + N)
+      rret = GSF_current_priorities + N;
+    else
+      rret = ret;
+    GSF_current_priorities = (GSF_current_priorities * (N - 1) + rret) / N;
+  }
+  if ((GNUNET_YES == ld) && (ret > 0))
+  {
+    /* try with charging */
+    ld = GSF_test_get_load_too_high_ (ret);
+  }
+  if (GNUNET_YES == ld)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                                ("# request dropped, priority insufficient"), 1,
+                              GNUNET_NO);
+    /* undo charge */
+    change_peer_respect (cp, (int) ret);
+    return -1;                  /* not enough resources */
+  }
+  else
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                                ("# requests done for a price (normal load)"),
+                              1,
+                              GNUNET_NO);
+  }
+#undef N
+  return ret;
+}
+
+
+/**
+ * The priority level imposes a bound on the maximum
+ * value for the ttl that can be requested.
+ *
+ * @param ttl_in requested ttl
+ * @param prio given priority
+ * @return @a ttl_in if @a ttl_in is below the limit,
+ *         otherwise the ttl-limit for the given @a prio
+ */
+static int32_t
+bound_ttl (int32_t ttl_in,
+           uint32_t prio)
+{
+  unsigned long long allowed;
+
+  if (ttl_in <= 0)
+    return ttl_in;
+  allowed = ((unsigned long long) prio) * TTL_DECREMENT / 1000;
+  if (ttl_in > allowed)
+  {
+    if (allowed >= (1 << 30))
+      return 1 << 30;
+    return allowed;
+  }
+  return ttl_in;
+}
+
+
+/**
+ * Closure for #test_exist_cb().
+ */
+struct TestExistClosure
+{
+  /**
+   * Priority of the incoming request.
+   */
+  int32_t priority;
+
+  /**
+   * Relative TTL of the incoming request.
+   */
+  int32_t ttl;
+
+  /**
+   * Type of the incoming request.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Set to #GNUNET_YES if we are done handling the query.
+   */
+  int finished;
+};
+
+
+/**
+ * Test if the query already exists.  If so, merge it, otherwise
+ * keep `finished` at #GNUNET_NO.
+ *
+ * @param cls our `struct TestExistClosure`
+ * @param hc the key of the query
+ * @param value the existing `struct PeerRequest`.
+ * @return #GNUNET_YES to continue to iterate,
+ *         #GNUNET_NO if we successfully merged
+ */
+static int
+test_exist_cb (void *cls,
+               const struct GNUNET_HashCode *hc,
+               void *value)
+{
+  struct TestExistClosure *tec = cls;
+  struct PeerRequest *peerreq = value;
+  struct GSF_PendingRequest *pr;
+  struct GSF_PendingRequestData *prd;
+
+  pr = peerreq->pr;
+  prd = GSF_pending_request_get_data_ (pr);
+  if (prd->type != tec->type)
+    return GNUNET_YES;
+  if (prd->ttl.abs_value_us >=
+      GNUNET_TIME_absolute_get ().abs_value_us + tec->ttl * 1000LL)
+  {
+    /* existing request has higher TTL, drop new one! */
+    prd->priority += tec->priority;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Have existing request with higher TTL, dropping new request.\n");
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                                ("# requests dropped due to higher-TTL request"),
+                              1, GNUNET_NO);
+    tec->finished = GNUNET_YES;
+    return GNUNET_NO;
+  }
+  /* existing request has lower TTL, drop old one! */
+  tec->priority += prd->priority;
+  free_pending_request (peerreq);
+  GSF_pending_request_cancel_ (pr,
+                               GNUNET_YES);
+  return GNUNET_NO;
+}
+
+
+/**
+ * Handle P2P "QUERY" message.  Creates the pending request entry
+ * and sets up all of the data structures to that we will
+ * process replies properly.  Does not initiate forwarding or
+ * local database lookups.
+ *
+ * @param cls the other peer involved (sender of the message)
+ * @param gm the GET message
+ */
+void
+handle_p2p_get (void *cls,
+                const struct GetMessage *gm)
+{
+  struct GSF_ConnectedPeer *cps = cls;
+  struct PeerRequest *peerreq;
+  struct GSF_PendingRequest *pr;
+  struct GSF_ConnectedPeer *cp;
+  const struct GNUNET_PeerIdentity *target;
+  enum GSF_PendingRequestOptions options;
+  uint16_t msize;
+  unsigned int bits;
+  const struct GNUNET_PeerIdentity *opt;
+  uint32_t bm;
+  size_t bfsize;
+  uint32_t ttl_decrement;
+  struct TestExistClosure tec;
+  GNUNET_PEER_Id spid;
+  const struct GSF_PendingRequestData *prd;
+
+  msize = ntohs (gm->header.size);
+  tec.type = ntohl (gm->type);
+  bm = ntohl (gm->hash_bitmap);
+  bits = 0;
+  while (bm > 0)
+  {
+    if (1 == (bm & 1))
+      bits++;
+    bm >>= 1;
+  }
+  opt = (const struct GNUNET_PeerIdentity *) &gm[1];
+  bfsize = msize - sizeof(struct GetMessage) - bits * sizeof(struct
+                                                             GNUNET_PeerIdentity);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop
+                              ("# GET requests received (from other peers)"),
+                            1,
+                            GNUNET_NO);
+  GSF_cover_query_count++;
+  bm = ntohl (gm->hash_bitmap);
+  bits = 0;
+  if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO))
+    cp = GSF_peer_get_ (&opt[bits++]);
+  else
+    cp = cps;
+  if (NULL == cp)
+  {
+    if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO))
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Failed to find RETURN-TO peer `%s' in connection set. Dropping query.\n",
+                  GNUNET_i2s (&opt[bits - 1]));
+
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Failed to find peer `%s' in connection set. Dropping query.\n",
+                  GNUNET_i2s (cps->ppd.peer));
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              (
+                                "# requests dropped due to missing reverse route"),
+                              1,
+                              GNUNET_NO);
+    return;
+  }
+  unsigned int queue_size = GNUNET_MQ_get_length (cp->mq);
+  queue_size += cp->ppd.pending_replies + cp->delay_queue_size;
+  if (queue_size > MAX_QUEUE_PER_PEER)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Peer `%s' has too many replies queued already. Dropping query.\n",
+                GNUNET_i2s (cps->ppd.peer));
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# requests dropped due to full reply queue"),
+                              1,
+                              GNUNET_NO);
+    return;
+  }
+  /* note that we can really only check load here since otherwise
+   * peers could find out that we are overloaded by not being
+   * disconnected after sending us a malformed query... */
+  tec.priority = bound_priority (ntohl (gm->priority),
+                                 cps);
+  if (tec.priority < 0)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Dropping query from `%s', this peer is too busy.\n",
+                GNUNET_i2s (cps->ppd.peer));
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received request for `%s' of type %u from peer `%s' with flags %u\n",
+              GNUNET_h2s (&gm->query),
+              (unsigned int) tec.type,
+              GNUNET_i2s (cps->ppd.peer),
+              (unsigned int) bm);
+  target =
+    (0 !=
+     (bm & GET_MESSAGE_BIT_TRANSMIT_TO)) ? (&opt[bits++]) : NULL;
+  options = GSF_PRO_DEFAULTS;
+  spid = 0;
+  if ((GNUNET_LOAD_get_load (cp->ppd.transmission_delay) > 3 * (1
+                                                                + tec.priority))
+      || (GNUNET_LOAD_get_average (cp->ppd.transmission_delay) >
+          GNUNET_CONSTANTS_MAX_CORK_DELAY.rel_value_us * 2
+          + GNUNET_LOAD_get_average (GSF_rt_entry_lifetime)))
+  {
+    /* don't have BW to send to peer, or would likely take longer than we have for it,
+     * so at best indirect the query */
+    tec.priority = 0;
+    options |= GSF_PRO_FORWARD_ONLY;
+    spid = GNUNET_PEER_intern (cps->ppd.peer);
+    GNUNET_assert (0 != spid);
+  }
+  tec.ttl = bound_ttl (ntohl (gm->ttl),
+                       tec.priority);
+  /* decrement ttl (always) */
+  ttl_decrement =
+    2 * TTL_DECREMENT + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                                  TTL_DECREMENT);
+  if ((tec.ttl < 0) &&
+      (((int32_t) (tec.ttl - ttl_decrement)) > 0))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Dropping query from `%s' due to TTL underflow (%d - %u).\n",
+                GNUNET_i2s (cps->ppd.peer),
+                tec.ttl,
+                ttl_decrement);
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                                ("# requests dropped due TTL underflow"), 1,
+                              GNUNET_NO);
+    /* integer underflow => drop (should be very rare)! */
+    return;
+  }
+  tec.ttl -= ttl_decrement;
+
+  /* test if the request already exists */
+  tec.finished = GNUNET_NO;
+  GNUNET_CONTAINER_multihashmap_get_multiple (cp->request_map,
+                                              &gm->query,
+                                              &test_exist_cb,
+                                              &tec);
+  if (GNUNET_YES == tec.finished)
+    return; /* merged into existing request, we're done */
+
+  peerreq = GNUNET_new (struct PeerRequest);
+  peerreq->cp = cp;
+  pr = GSF_pending_request_create_ (options,
+                                    tec.type,
+                                    &gm->query,
+                                    target,
+                                    (bfsize > 0)
+                                    ? (const char *) &opt[bits]
+                                    : NULL,
+                                    bfsize,
+                                    1 /* anonymity */,
+                                    (uint32_t) tec.priority,
+                                    tec.ttl,
+                                    spid,
+                                    GNUNET_PEER_intern (cps->ppd.peer),
+                                    NULL, 0,        /* replies_seen */
+                                    &handle_p2p_reply,
+                                    peerreq);
+  GNUNET_assert (NULL != pr);
+  prd = GSF_pending_request_get_data_ (pr);
+  peerreq->pr = pr;
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONTAINER_multihashmap_put (cp->request_map,
+                                                   &prd->query,
+                                                   peerreq,
+                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop (
+                              "# P2P query messages received and processed"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# P2P searches active"),
+                            1,
+                            GNUNET_NO);
+  GSF_pending_request_get_data_ (pr)->has_started = GNUNET_YES;
+  GSF_local_lookup_ (pr,
+                     &GSF_consider_forwarding,
+                     NULL);
+}
+
+
+/**
+ * Transmit a message to the given peer as soon as possible.
+ * If the peer disconnects before the transmission can happen,
+ * the callback is invoked with a `NULL` @a buffer.
+ *
+ * @param cp target peer
+ * @param is_query is this a query (#GNUNET_YES) or content (#GNUNET_NO) or neither (#GNUNET_SYSERR)
+ * @param priority how important is this request?
+ * @param env message to send
+ */
+void
+GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp,
+                    int is_query,
+                    uint32_t priority,
+                    struct GNUNET_MQ_Envelope *env)
+{
+  struct GSF_PeerTransmitHandle *pth;
+  struct GSF_PeerTransmitHandle *pos;
+  struct GSF_PeerTransmitHandle *prev;
+
+  pth = GNUNET_new (struct GSF_PeerTransmitHandle);
+  pth->transmission_request_start_time = GNUNET_TIME_absolute_get ();
+  pth->env = env;
+  pth->is_query = is_query;
+  pth->priority = priority;
+  pth->cp = cp;
+  /* insertion sort (by priority, descending) */
+  prev = NULL;
+  pos = cp->pth_head;
+  while ((NULL != pos) && (pos->priority > priority))
+  {
+    prev = pos;
+    pos = pos->next;
+  }
+  GNUNET_CONTAINER_DLL_insert_after (cp->pth_head,
+                                     cp->pth_tail,
+                                     prev,
+                                     pth);
+  if (GNUNET_YES == is_query)
+    cp->ppd.pending_queries++;
+  else if (GNUNET_NO == is_query)
+    cp->ppd.pending_replies++;
+  schedule_transmission (pth);
+}
+
+
+/**
+ * Report on receiving a reply; update the performance record of the given peer.
+ *
+ * @param cp responding peer (will be updated)
+ * @param request_time time at which the original query was transmitted
+ * @param request_priority priority of the original request
+ */
+void
+GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp,
+                              struct GNUNET_TIME_Absolute request_time,
+                              uint32_t request_priority)
+{
+  struct GNUNET_TIME_Relative delay;
+
+  delay = GNUNET_TIME_absolute_get_duration (request_time);
+  cp->ppd.avg_reply_delay.rel_value_us =
+    (cp->ppd.avg_reply_delay.rel_value_us * (RUNAVG_DELAY_N - 1)
+     + delay.rel_value_us) / RUNAVG_DELAY_N;
+  cp->ppd.avg_priority =
+    (cp->ppd.avg_priority * (RUNAVG_DELAY_N - 1)
+     + request_priority) / RUNAVG_DELAY_N;
+}
+
+
+/**
+ * Report on receiving a reply in response to an initiating client.
+ * Remember that this peer is good for this client.
+ *
+ * @param cp responding peer (will be updated)
+ * @param initiator_client local client on responsible for query
+ */
+void
+GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp,
+                                   struct GSF_LocalClient *initiator_client)
+{
+  cp->ppd.last_client_replies[cp->last_client_replies_woff++
+                              % CS2P_SUCCESS_LIST_SIZE] = initiator_client;
+}
+
+
+/**
+ * Report on receiving a reply in response to an initiating peer.
+ * Remember that this peer is good for this initiating peer.
+ *
+ * @param cp responding peer (will be updated)
+ * @param initiator_peer other peer responsible for query
+ */
+void
+GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp,
+                                 const struct GSF_ConnectedPeer *initiator_peer)
+{
+  unsigned int woff;
+
+  woff = cp->last_p2p_replies_woff % P2P_SUCCESS_LIST_SIZE;
+  GNUNET_PEER_change_rc (cp->ppd.last_p2p_replies[woff], -1);
+  cp->ppd.last_p2p_replies[woff] = initiator_peer->ppd.pid;
+  GNUNET_PEER_change_rc (initiator_peer->ppd.pid, 1);
+  cp->last_p2p_replies_woff = (woff + 1) % P2P_SUCCESS_LIST_SIZE;
+}
+
+
+/**
+ * Write peer-respect information to a file - flush the buffer entry!
+ *
+ * @param cls unused
+ * @param key peer identity
+ * @param value the `struct GSF_ConnectedPeer` to flush
+ * @return #GNUNET_OK to continue iteration
+ */
+static int
+flush_respect (void *cls,
+               const struct GNUNET_PeerIdentity *key,
+               void *value)
+{
+  struct GSF_ConnectedPeer *cp = value;
+  struct GNUNET_PeerIdentity pid;
+
+  if (cp->ppd.respect == cp->disk_respect)
+    return GNUNET_OK;           /* unchanged */
+  GNUNET_assert (0 != cp->ppd.pid);
+  GNUNET_PEER_resolve (cp->ppd.pid, &pid);
+  GNUNET_PEERSTORE_store (peerstore, "fs", &pid, "respect", &cp->ppd.respect,
+                          sizeof(cp->ppd.respect),
+                          GNUNET_TIME_UNIT_FOREVER_ABS,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  return GNUNET_OK;
+}
+
+
+void
+GSF_peer_disconnect_handler (void *cls,
+                             const struct GNUNET_PeerIdentity *peer,
+                             void *internal_cls)
+{
+  struct GSF_ConnectedPeer *cp = internal_cls;
+  struct GSF_PeerTransmitHandle *pth;
+  struct GSF_DelayedHandle *dh;
+
+  if (NULL == cp)
+    return;  /* must have been disconnect from core with
+              * 'peer' == my_id, ignore */
+  flush_respect (NULL,
+                 peer,
+                 cp);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (cp_map,
+                                                       peer,
+                                                       cp));
+  GNUNET_STATISTICS_set (GSF_stats,
+                         gettext_noop ("# peers connected"),
+                         GNUNET_CONTAINER_multipeermap_size (cp_map),
+                         GNUNET_NO);
+  if (NULL != cp->respect_iterate_req)
+  {
+    GNUNET_PEERSTORE_iteration_stop (cp->respect_iterate_req);
+    cp->respect_iterate_req = NULL;
+  }
+  GNUNET_CONTAINER_multihashmap_iterate (cp->request_map,
+                                         &cancel_pending_request,
+                                         cp);
+  GNUNET_CONTAINER_multihashmap_destroy (cp->request_map);
+  cp->request_map = NULL;
+  GSF_plan_notify_peer_disconnect_ (cp);
+  GNUNET_LOAD_value_free (cp->ppd.transmission_delay);
+  GNUNET_PEER_decrement_rcs (cp->ppd.last_p2p_replies,
+                             P2P_SUCCESS_LIST_SIZE);
+  memset (cp->ppd.last_p2p_replies,
+          0,
+          sizeof(cp->ppd.last_p2p_replies));
+  GSF_push_stop_ (cp);
+  while (NULL != (pth = cp->pth_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (cp->pth_head,
+                                 cp->pth_tail,
+                                 pth);
+    if (GNUNET_YES == pth->is_query)
+      GNUNET_assert (0 < cp->ppd.pending_queries--);
+    else if (GNUNET_NO == pth->is_query)
+      GNUNET_assert (0 < cp->ppd.pending_replies--);
+    GNUNET_free (pth);
+  }
+  while (NULL != (dh = cp->delayed_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (cp->delayed_head,
+                                 cp->delayed_tail,
+                                 dh);
+    GNUNET_MQ_discard (dh->env);
+    cp->delay_queue_size--;
+    GNUNET_SCHEDULER_cancel (dh->delay_task);
+    GNUNET_free (dh);
+  }
+  GNUNET_PEER_change_rc (cp->ppd.pid, -1);
+  if (NULL != cp->mig_revive_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->mig_revive_task);
+    cp->mig_revive_task = NULL;
+  }
+  GNUNET_break (0 == cp->ppd.pending_queries);
+  GNUNET_break (0 == cp->ppd.pending_replies);
+  GNUNET_free (cp);
+}
+
+
+/**
+ * Closure for #call_iterator().
+ */
+struct IterationContext
+{
+  /**
+   * Function to call on each entry.
+   */
+  GSF_ConnectedPeerIterator it;
+
+  /**
+   * Closure for @e it.
+   */
+  void *it_cls;
+};
+
+
+/**
+ * Function that calls the callback for each peer.
+ *
+ * @param cls the `struct IterationContext *`
+ * @param key identity of the peer
+ * @param value the `struct GSF_ConnectedPeer *`
+ * @return #GNUNET_YES to continue iteration
+ */
+static int
+call_iterator (void *cls,
+               const struct GNUNET_PeerIdentity *key,
+               void *value)
+{
+  struct IterationContext *ic = cls;
+  struct GSF_ConnectedPeer *cp = value;
+
+  ic->it (ic->it_cls,
+          key, cp,
+          &cp->ppd);
+  return GNUNET_YES;
+}
+
+
+void
+GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it,
+                              void *it_cls)
+{
+  struct IterationContext ic;
+
+  ic.it = it;
+  ic.it_cls = it_cls;
+  GNUNET_CONTAINER_multipeermap_iterate (cp_map,
+                                         &call_iterator,
+                                         &ic);
+}
+
+
+/**
+ * Obtain the identity of a connected peer.
+ *
+ * @param cp peer to get identity of
+ * @param id identity to set (written to)
+ */
+void
+GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp,
+                                  struct GNUNET_PeerIdentity *id)
+{
+  GNUNET_assert (0 != cp->ppd.pid);
+  GNUNET_PEER_resolve (cp->ppd.pid, id);
+}
+
+
+/**
+ * Obtain the identity of a connected peer.
+ *
+ * @param cp peer to get identity of
+ * @return reference to peer identity, valid until peer disconnects (!)
+ */
+const struct GNUNET_PeerIdentity *
+GSF_connected_peer_get_identity2_ (const struct GSF_ConnectedPeer *cp)
+{
+  GNUNET_assert (0 != cp->ppd.pid);
+  return GNUNET_PEER_resolve2 (cp->ppd.pid);
+}
+
+
+/**
+ * Ask a peer to stop migrating data to us until the given point
+ * in time.
+ *
+ * @param cp peer to ask
+ * @param block_time until when to block
+ */
+void
+GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp,
+                           struct GNUNET_TIME_Absolute block_time)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct MigrationStopMessage *msm;
+
+  if (cp->last_migration_block.abs_value_us > block_time.abs_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Migration already blocked for another %s\n",
+                GNUNET_STRINGS_relative_time_to_string (
+                  GNUNET_TIME_absolute_get_remaining
+                    (cp->
+                    last_migration_block), GNUNET_YES));
+    return;                     /* already blocked */
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Asking to stop migration for %s\n",
+              GNUNET_STRINGS_relative_time_to_string (
+                GNUNET_TIME_absolute_get_remaining (block_time),
+                GNUNET_YES));
+  cp->last_migration_block = block_time;
+  env = GNUNET_MQ_msg (msm,
+                       GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP);
+  msm->reserved = htonl (0);
+  msm->duration
+    = GNUNET_TIME_relative_hton (GNUNET_TIME_absolute_get_remaining
+                                   (cp->last_migration_block));
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# migration stop messages sent"),
+                            1,
+                            GNUNET_NO);
+  GSF_peer_transmit_ (cp,
+                      GNUNET_SYSERR,
+                      UINT32_MAX,
+                      env);
+}
+
+
+/**
+ * Notify core about a preference we have for the given peer
+ * (to allocate more resources towards it).  The change will
+ * be communicated the next time we reserve bandwidth with
+ * core (not instantly).
+ *
+ * @param cp peer to reserve bandwidth from
+ * @param pref preference change
+ */
+void
+GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp,
+                                       uint64_t pref)
+{
+  cp->inc_preference += pref;
+}
+
+
+/**
+ * Call this method periodically to flush respect information to disk.
+ *
+ * @param cls closure, not used
+ */
+static void
+cron_flush_respect (void *cls)
+{
+  fr_task = NULL;
+  GNUNET_CONTAINER_multipeermap_iterate (cp_map,
+                                         &flush_respect,
+                                         NULL);
+  fr_task = GNUNET_SCHEDULER_add_delayed_with_priority (RESPECT_FLUSH_FREQ,
+                                                        GNUNET_SCHEDULER_PRIORITY_HIGH,
+                                                        &cron_flush_respect,
+                                                        NULL);
+}
+
+
+/**
+ * Initialize peer management subsystem.
+ */
+void
+GSF_connected_peer_init_ ()
+{
+  cp_map = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_YES);
+  peerstore = GNUNET_PEERSTORE_connect (GSF_cfg);
+  fr_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_HIGH,
+                                                &cron_flush_respect, NULL);
+}
+
+
+/**
+ * Shutdown peer management subsystem.
+ */
+void
+GSF_connected_peer_done_ ()
+{
+  GNUNET_CONTAINER_multipeermap_iterate (cp_map,
+                                         &flush_respect,
+                                         NULL);
+  GNUNET_SCHEDULER_cancel (fr_task);
+  fr_task = NULL;
+  GNUNET_CONTAINER_multipeermap_destroy (cp_map);
+  cp_map = NULL;
+  GNUNET_PEERSTORE_disconnect (peerstore);
+}
+
+
+/**
+ * Iterator to remove references to LC entry.
+ *
+ * @param cls the `struct GSF_LocalClient *` to look for
+ * @param key current key code
+ * @param value value in the hash map (peer entry)
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+clean_local_client (void *cls,
+                    const struct GNUNET_PeerIdentity *key,
+                    void *value)
+{
+  const struct GSF_LocalClient *lc = cls;
+  struct GSF_ConnectedPeer *cp = value;
+  unsigned int i;
+
+  for (i = 0; i < CS2P_SUCCESS_LIST_SIZE; i++)
+    if (cp->ppd.last_client_replies[i] == lc)
+      cp->ppd.last_client_replies[i] = NULL;
+  return GNUNET_YES;
+}
+
+
+/**
+ * Notification that a local client disconnected.  Clean up all of our
+ * references to the given handle.
+ *
+ * @param lc handle to the local client (henceforth invalid)
+ */
+void
+GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc)
+{
+  if (NULL == cp_map)
+    return;                     /* already cleaned up */
+  GNUNET_CONTAINER_multipeermap_iterate (cp_map,
+                                         &clean_local_client,
+                                         (void *) lc);
+}
+
+
+/* end of gnunet-service-fs_cp.c */
diff --git a/src/service/fs/gnunet-service-fs_cp.h b/src/service/fs/gnunet-service-fs_cp.h
new file mode 100644
index 000000000..cea82b10c
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_cp.h
@@ -0,0 +1,415 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_cp.h
+ * @brief API to handle 'connected peers'
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_CP_H
+#define GNUNET_SERVICE_FS_CP_H
+
+#include "fs.h"
+#include "gnunet-service-fs.h"
+
+
+/**
+ * Maximum number of outgoing messages we queue per peer.
+ *
+ * Performance measurements for 2 peer setup for 50 MB file
+ * (using perf_gnunet_service_fs_p2p):
+ *
+ *  24: 2-3 MB/s # ~ 24 MB RAM
+ * 256:   8 MB/s # ~256 MB RAM
+ *
+ * Conclusion: 24 should suffice (reasonable
+ * performance, no excessive memory use).
+ */
+#define MAX_QUEUE_PER_PEER 24
+
+/**
+ * Length of the P2P success tracker.  Note that having a very long
+ * list can also hurt performance.
+ */
+#define P2P_SUCCESS_LIST_SIZE 8
+
+/**
+ * Length of the CS-2-P success tracker.  Note that
+ * having a very long list can also hurt performance.
+ */
+#define CS2P_SUCCESS_LIST_SIZE 8
+
+
+/**
+ * Performance data kept for a peer.
+ */
+struct GSF_PeerPerformanceData
+{
+  /**
+   * List of the last clients for which this peer successfully
+   * answered a query.
+   */
+  struct GSF_LocalClient *last_client_replies[CS2P_SUCCESS_LIST_SIZE];
+
+  /**
+   * List of the last PIDs for which
+   * this peer successfully answered a query;
+   * We use 0 to indicate no successful reply.
+   */
+  GNUNET_PEER_Id last_p2p_replies[P2P_SUCCESS_LIST_SIZE];
+
+  /**
+   * Average delay between sending the peer a request and
+   * getting a reply (only calculated over the requests for
+   * which we actually got a reply).   Calculated
+   * as a moving average: new_delay = ((n-1)*last_delay+curr_delay) / n
+   */
+  struct GNUNET_TIME_Relative avg_reply_delay;
+
+  /**
+   * If we get content we already have from this peer, for how
+   * long do we block it?  Adjusted based on the fraction of
+   * redundant data we receive, between 1s and 1h.
+   */
+  struct GNUNET_TIME_Relative migration_delay;
+
+  /**
+   * Point in time until which this peer does not want us to migrate content
+   * to it.
+   */
+  struct GNUNET_TIME_Absolute migration_blocked_until;
+
+  /**
+   * Transmission times for the last MAX_QUEUE_PER_PEER
+   * requests for this peer.  Used as a ring buffer, current
+   * offset is stored in 'last_request_times_off'.  If the
+   * oldest entry is more recent than the 'avg_delay', we should
+   * not send any more requests right now.
+   */
+  struct GNUNET_TIME_Absolute last_request_times[MAX_QUEUE_PER_PEER];
+
+  /**
+   * How long does it typically take for us to transmit a message
+   * to this peer?  (delay between the request being issued and
+   * the callback being invoked).
+   */
+  struct GNUNET_LOAD_Value *transmission_delay;
+
+  /**
+   * Average priority of successful replies.  Calculated
+   * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n
+   */
+  double avg_priority;
+
+  /**
+   * The peer's identity (interned version).
+   */
+  GNUNET_PEER_Id pid;
+
+  /**
+   * The peer's identity (pointer).
+   */
+  const struct GNUNET_PeerIdentity *peer;
+
+  /**
+   * Respect rating for this peer
+   */
+  uint32_t respect;
+
+  /**
+   * Number of pending queries (replies are not counted)
+   */
+  unsigned int pending_queries;
+
+  /**
+   * Number of pending replies (queries are not counted)
+   */
+  unsigned int pending_replies;
+};
+
+
+/**
+ * Signature of function called on a connected peer.
+ *
+ * @param cls closure
+ * @param peer identity of the peer
+ * @param cp handle to the connected peer record
+ * @param perf peer performance data
+ */
+typedef void
+(*GSF_ConnectedPeerIterator) (void *cls,
+                              const struct GNUNET_PeerIdentity *peer,
+                              struct GSF_ConnectedPeer *cp,
+                              const struct GSF_PeerPerformanceData *ppd);
+
+
+/**
+ * Function called to get a message for transmission.
+ *
+ * @param cls closure
+ * @param buf_size number of bytes available in @a buf
+ * @param buf where to copy the message, NULL on error (peer disconnect)
+ * @return number of bytes copied to @a buf, can be 0 (without indicating an error)
+ */
+typedef size_t
+(*GSF_GetMessageCallback) (void *cls,
+                           size_t buf_size,
+                           void *buf);
+
+
+/**
+ * Signature of function called on a reservation success or failure.
+ *
+ * @param cls closure
+ * @param cp handle to the connected peer record
+ * @param success #GNUNET_YES on success, #GNUNET_NO on failure
+ */
+typedef void
+(*GSF_PeerReserveCallback) (void *cls,
+                            struct GSF_ConnectedPeer *cp,
+                            int success);
+
+
+/**
+ * Handle to cancel a transmission request.
+ */
+struct GSF_PeerTransmitHandle;
+
+
+/**
+ * A peer connected to us.  Setup the connected peer
+ * records.
+ *
+ * @param cls NULL
+ * @param peer identity of peer that connected
+ * @param mq message queue for talking to @a peer
+ * @return internal handle for the peer
+ */
+void *
+GSF_peer_connect_handler (void *cls,
+                          const struct GNUNET_PeerIdentity *peer,
+                          struct GNUNET_MQ_Handle *mq);
+
+
+/**
+ * Get a handle for a connected peer.
+ *
+ * @param peer peer's identity
+ * @return NULL if this peer is not currently connected
+ */
+struct GSF_ConnectedPeer *
+GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer);
+
+
+/**
+ * Update the latency information kept for the given peer.
+ *
+ * @param id peer record to update
+ * @param latency current latency value
+ */
+void
+GSF_update_peer_latency_ (const struct GNUNET_PeerIdentity *id,
+                          struct GNUNET_TIME_Relative latency);
+
+
+/**
+ * Transmit a message to the given peer as soon as possible.
+ * If the peer disconnects before the transmission can happen,
+ * the callback is invoked with a 'NULL' buffer.
+ *
+ * @param cp target peer
+ * @param is_query is this a query (#GNUNET_YES) or content (#GNUNET_NO)
+ * @param priority how important is this request?
+ * @param env envelope of message to send
+ */
+void
+GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp,
+                    int is_query,
+                    uint32_t priority,
+                    struct GNUNET_MQ_Envelope *env);
+
+
+/**
+ * Report on receiving a reply; update the performance record of the given peer.
+ *
+ * @param cp responding peer (will be updated)
+ * @param request_time time at which the original query was transmitted
+ * @param request_priority priority of the original request
+ */
+void
+GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp,
+                              struct GNUNET_TIME_Absolute request_time,
+                              uint32_t request_priority);
+
+
+/**
+ * Report on receiving a reply in response to an initiating client.
+ * Remember that this peer is good for this client.
+ *
+ * @param cp responding peer (will be updated)
+ * @param initiator_client local client on responsible for query
+ */
+void
+GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp,
+                                   struct GSF_LocalClient *initiator_client);
+
+
+/**
+ * Report on receiving a reply in response to an initiating peer.
+ * Remember that this peer is good for this initiating peer.
+ *
+ * @param cp responding peer (will be updated)
+ * @param initiator_peer other peer responsible for query
+ */
+void
+GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp,
+                                 const struct GSF_ConnectedPeer
+                                 *initiator_peer);
+
+
+/**
+ * Handle P2P #GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP message.
+ *
+ * @param cls closure, the `struct GSF_ConnectedPeer`
+ * @param msm the actual message
+ */
+void
+handle_p2p_migration_stop (void *cls,
+                           const struct MigrationStopMessage *message);
+
+
+/**
+ * Handle P2P "QUERY" message.
+ *
+ * @param cls the `struct GSF_ConnectedPeer` of the other sender
+ * @param gm the actual message
+ */
+void
+handle_p2p_get (void *cls,
+                const struct GetMessage *gm);
+
+
+/**
+ * Return the performance data record for the given peer
+ *
+ * @param cp peer to query
+ * @return performance data record for the peer
+ */
+struct GSF_PeerPerformanceData *
+GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp);
+
+
+/**
+ * Ask a peer to stop migrating data to us until the given point
+ * in time.
+ *
+ * @param cp peer to ask
+ * @param block_time until when to block
+ */
+void
+GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp,
+                           struct GNUNET_TIME_Absolute block_time);
+
+
+/**
+ * A peer disconnected from us.  Tear down the connected peer
+ * record.
+ *
+ * @param cls unused
+ * @param peer identity of peer that disconnected
+ * @param internal_cls the corresponding `struct GSF_ConnectedPeer` 
+ */
+void
+GSF_peer_disconnect_handler (void *cls,
+                             const struct GNUNET_PeerIdentity *peer,
+                             void *internal_cls);
+
+
+/**
+ * Notification that a local client disconnected.  Clean up all of our
+ * references to the given handle.
+ *
+ * @param lc handle to the local client (henceforth invalid)
+ */
+void
+GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc);
+
+
+/**
+ * Notify core about a preference we have for the given peer
+ * (to allocate more resources towards it).  The change will
+ * be communicated the next time we reserve bandwidth with
+ * core (not instantly).
+ *
+ * @param cp peer to reserve bandwidth from
+ * @param pref preference change
+ */
+void
+GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp,
+                                       uint64_t pref);
+
+
+/**
+ * Obtain the identity of a connected peer.
+ *
+ * @param cp peer to get identity of
+ * @param id identity to set (written to)
+ */
+void
+GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp,
+                                  struct GNUNET_PeerIdentity *id);
+
+
+/**
+ * Obtain the identity of a connected peer.
+ *
+ * @param cp peer to get identity of
+ * @return reference to peer identity, valid until peer disconnects (!)
+ */
+const struct GNUNET_PeerIdentity *
+GSF_connected_peer_get_identity2_ (const struct GSF_ConnectedPeer *cp);
+
+
+/**
+ * Iterate over all connected peers.
+ *
+ * @param it function to call for each peer
+ * @param it_cls closure for @a it
+ */
+void
+GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it, void *it_cls);
+
+
+/**
+ * Initialize peer management subsystem.
+ */
+void
+GSF_connected_peer_init_ (void);
+
+
+/**
+ * Shutdown peer management subsystem.
+ */
+void
+GSF_connected_peer_done_ (void);
+
+
+#endif
+/* end of gnunet-service-fs_cp.h */
diff --git a/src/service/fs/gnunet-service-fs_indexing.c b/src/service/fs/gnunet-service-fs_indexing.c
new file mode 100644
index 000000000..8fb34c067
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_indexing.c
@@ -0,0 +1,495 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_indexing.c
+ * @brief program that provides indexing functions of the file-sharing service
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include <float.h>
+#include "gnunet_core_service.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_protocols.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_indexing.h"
+#include "fs.h"
+
+/**
+ * In-memory information about indexed files (also available
+ * on-disk).
+ */
+struct IndexInfo
+{
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexInfo *next;
+
+  /**
+   * This is a doubly linked list.
+   */
+  struct IndexInfo *prev;
+
+  /**
+   * Name of the indexed file.  Memory allocated
+   * at the end of this struct (do not free).
+   */
+  const char *filename;
+
+  /**
+   * Context for transmitting confirmation to client,
+   * NULL if we've done this already.
+   */
+  struct GNUNET_SERVER_TransmitContext *tc;
+
+  /**
+   * Context for hashing of the file.
+   */
+  struct GNUNET_CRYPTO_FileHashContext *fhc;
+
+  /**
+   * Hash of the contents of the file.
+   */
+  struct GNUNET_HashCode file_id;
+};
+
+
+/**
+ * Head of linked list of indexed files.
+ * FIXME: we don't need both a DLL and a hashmap here!
+ */
+static struct IndexInfo *indexed_files_head;
+
+/**
+ * Tail of linked list of indexed files.
+ */
+static struct IndexInfo *indexed_files_tail;
+
+/**
+ * Maps hash over content of indexed files to the respective 'struct IndexInfo'.
+ * The filenames are pointers into the indexed_files linked list and
+ * do not need to be freed.
+ */
+static struct GNUNET_CONTAINER_MultiHashMap *ifm;
+
+/**
+ * Our configuration.
+ */
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Datastore handle.  Created and destroyed by code in
+ * gnunet-service-fs (this is an alias).
+ */
+static struct GNUNET_DATASTORE_Handle *dsh;
+
+
+/**
+ * Write the current index information list to disk.
+ */
+static void
+write_index_list ()
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  char *fn;
+  struct IndexInfo *pos;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                               "fs",
+                               "INDEXDB");
+    return;
+  }
+  wh = GNUNET_BIO_write_open_file (fn);
+  if (NULL == wh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _ ("Could not open `%s'.\n"),
+                fn);
+    GNUNET_free (fn);
+    return;
+  }
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+    if ((GNUNET_OK != GNUNET_BIO_write (wh,
+                                        "fs-indexing-file-id",
+                                        &pos->file_id,
+                                        sizeof(struct GNUNET_HashCode))) ||
+        (GNUNET_OK != GNUNET_BIO_write_string (wh,
+                                               "fs-indexing-filename",
+                                               pos->filename)))
+      break;
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _ ("Error writing `%s'.\n"),
+                fn);
+    GNUNET_free (fn);
+    return;
+  }
+  GNUNET_free (fn);
+}
+
+
+/**
+ * Read index information from disk.
+ */
+static void
+read_index_list ()
+{
+  struct GNUNET_BIO_ReadHandle *rh;
+  char *fn;
+  struct IndexInfo *pos;
+  char *fname;
+  struct GNUNET_HashCode hc;
+  size_t slen;
+  char *emsg;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                               "fs",
+                               "INDEXDB");
+    return;
+  }
+  if (GNUNET_NO == GNUNET_DISK_file_test (fn))
+  {
+    /* no index info yet */
+    GNUNET_free (fn);
+    return;
+  }
+  rh = GNUNET_BIO_read_open_file (fn);
+  if (NULL == rh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                _ ("Could not open `%s'.\n"),
+                fn);
+    GNUNET_free (fn);
+    return;
+  }
+  while (
+    (GNUNET_OK == GNUNET_BIO_read (rh,
+                                   "Hash of indexed file",
+                                   &hc,
+                                   sizeof(struct GNUNET_HashCode))) &&
+    (GNUNET_OK ==
+     GNUNET_BIO_read_string (rh, "Name of indexed file", &fname, 1024 * 16)) &&
+    (fname != NULL))
+  {
+    slen = strlen (fname) + 1;
+    pos = GNUNET_malloc (sizeof(struct IndexInfo) + slen);
+    pos->file_id = hc;
+    pos->filename = (const char *) &pos[1];
+    GNUNET_memcpy (&pos[1], fname, slen);
+    if (GNUNET_SYSERR == GNUNET_CONTAINER_multihashmap_put (
+          ifm,
+          &pos->file_id,
+          pos,
+          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+    {
+      GNUNET_free (pos);
+    }
+    else
+    {
+      GNUNET_CONTAINER_DLL_insert (indexed_files_head, indexed_files_tail, pos);
+    }
+    GNUNET_free (fname);
+  }
+  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
+    GNUNET_free (emsg);
+  GNUNET_free (fn);
+}
+
+
+/**
+ * Continuation called from datastore's remove
+ * function.
+ *
+ * @param cls unused
+ * @param success did the deletion work?
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg error message
+ */
+static void
+remove_cont (void *cls,
+             int success,
+             struct GNUNET_TIME_Absolute min_expiration,
+             const char *msg)
+{
+  if (GNUNET_OK != success)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Failed to delete bogus block: %s\n"),
+                msg);
+}
+
+
+int
+GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key,
+                                  uint32_t size,
+                                  const void *data,
+                                  enum GNUNET_BLOCK_Type type,
+                                  uint32_t priority,
+                                  uint32_t anonymity,
+                                  uint32_t replication,
+                                  struct GNUNET_TIME_Absolute expiration,
+                                  uint64_t uid,
+                                  GNUNET_DATASTORE_DatumProcessor cont,
+                                  void *cont_cls)
+{
+  const struct OnDemandBlock *odb;
+  struct GNUNET_HashCode nkey;
+  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
+  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
+  struct GNUNET_HashCode query;
+  ssize_t nsize;
+  char ndata[DBLOCK_SIZE];
+  char edata[DBLOCK_SIZE];
+  const char *fn;
+  struct GNUNET_DISK_FileHandle *fh;
+  uint64_t off;
+  struct IndexInfo *ii;
+
+  if (size != sizeof(struct OnDemandBlock))
+  {
+    GNUNET_break (0);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  odb = (const struct OnDemandBlock *) data;
+  off = GNUNET_ntohll (odb->offset);
+  ii = GNUNET_CONTAINER_multihashmap_get (ifm, &odb->file_id);
+  if (NULL == ii)
+  {
+    GNUNET_break (0);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to find index %s\n",
+                GNUNET_h2s (&odb->file_id));
+    return GNUNET_SYSERR;
+  }
+  fn = ii->filename;
+  if ((NULL == fn) || (0 != access (fn, R_OK)))
+  {
+    GNUNET_STATISTICS_update (
+      GSF_stats,
+      gettext_noop ("# index blocks removed: original file inaccessible"),
+      1,
+      GNUNET_YES);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  if ((NULL == (fh = GNUNET_DISK_file_open (fn,
+                                            GNUNET_DISK_OPEN_READ,
+                                            GNUNET_DISK_PERM_NONE))) ||
+      (off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)) ||
+      (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof(ndata)))))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ (
+                  "Could not access indexed file `%s' (%s) at offset %llu: %s\n"),
+                GNUNET_h2s (&odb->file_id),
+                fn,
+                (unsigned long long) off,
+                (fn == NULL) ? _ ("not indexed") : strerror (errno));
+    if (fh != NULL)
+      GNUNET_DISK_file_close (fh);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_DISK_file_close (fh);
+  GNUNET_CRYPTO_hash (ndata, nsize, &nkey);
+  GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
+  GNUNET_CRYPTO_symmetric_encrypt (ndata, nsize, &skey, &iv, edata);
+  GNUNET_CRYPTO_hash (edata, nsize, &query);
+  if (0 != memcmp (&query, key, sizeof(struct GNUNET_HashCode)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _ ("Indexed file `%s' changed at offset %llu\n"),
+                fn,
+                (unsigned long long) off);
+    GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "On-demand encoded block for query `%s'\n",
+              GNUNET_h2s (key));
+  cont (cont_cls,
+        key,
+        nsize,
+        edata,
+        GNUNET_BLOCK_TYPE_FS_DBLOCK,
+        priority,
+        anonymity,
+        replication,
+        expiration,
+        uid);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Transmit information about indexed files to @a mq.
+ *
+ * @param mq message queue to send information to
+ */
+void
+GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct IndexInfoMessage *iim;
+  struct GNUNET_MessageHeader *iem;
+  size_t slen;
+  const char *fn;
+  struct IndexInfo *pos;
+
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+  {
+    fn = pos->filename;
+    slen = strlen (fn) + 1;
+    if (slen + sizeof(struct IndexInfoMessage) >= GNUNET_MAX_MESSAGE_SIZE)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    env =
+      GNUNET_MQ_msg_extra (iim, slen, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY);
+    iim->reserved = 0;
+    iim->file_id = pos->file_id;
+    GNUNET_memcpy (&iim[1], fn, slen);
+    GNUNET_MQ_send (mq, env);
+  }
+  env = GNUNET_MQ_msg (iem, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END);
+  GNUNET_MQ_send (mq, env);
+}
+
+
+/**
+ * Remove a file from the index.
+ *
+ * @param fid identifier of the file to remove
+ * @return #GNUNET_YES if the @a fid was found
+ */
+int
+GNUNET_FS_indexing_do_unindex (const struct GNUNET_HashCode *fid)
+{
+  struct IndexInfo *pos;
+
+  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
+  {
+    if (0 == memcmp (&pos->file_id, fid, sizeof(struct GNUNET_HashCode)))
+    {
+      GNUNET_CONTAINER_DLL_remove (indexed_files_head, indexed_files_tail, pos);
+      GNUNET_break (
+        GNUNET_OK ==
+        GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id, pos));
+      GNUNET_free (pos);
+      write_index_list ();
+      return GNUNET_YES;
+    }
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Add the given file to the list of indexed files.
+ *
+ * @param filename name of the file
+ * @param file_id hash identifier for @a filename
+ */
+void
+GNUNET_FS_add_to_index (const char *filename,
+                        const struct GNUNET_HashCode *file_id)
+{
+  struct IndexInfo *ii;
+  size_t slen;
+
+  ii = GNUNET_CONTAINER_multihashmap_get (ifm, file_id);
+  if (NULL != ii)
+  {
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_INFO,
+      _ (
+        "Index request received for file `%s' is already indexed as `%s'.  Permitting anyway.\n"),
+      filename,
+      ii->filename);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Adding file %s to index as %s\n",
+              filename,
+              GNUNET_h2s (file_id));
+  slen = strlen (filename) + 1;
+  ii = GNUNET_malloc (sizeof(struct IndexInfo) + slen);
+  ii->file_id = *file_id;
+  ii->filename = (const char *) &ii[1];
+  GNUNET_memcpy (&ii[1], filename, slen);
+  GNUNET_CONTAINER_DLL_insert (indexed_files_head, indexed_files_tail, ii);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (
+                   ifm,
+                   &ii->file_id,
+                   ii,
+                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  write_index_list ();
+}
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GNUNET_FS_indexing_done ()
+{
+  struct IndexInfo *pos;
+
+  while (NULL != (pos = indexed_files_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (indexed_files_head, indexed_files_tail, pos);
+    if (pos->fhc != NULL)
+      GNUNET_CRYPTO_hash_file_cancel (pos->fhc);
+    GNUNET_break (
+      GNUNET_OK ==
+      GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id, pos));
+    GNUNET_free (pos);
+  }
+  GNUNET_CONTAINER_multihashmap_destroy (ifm);
+  ifm = NULL;
+  cfg = NULL;
+}
+
+
+int
+GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c,
+                         struct GNUNET_DATASTORE_Handle *d)
+{
+  cfg = c;
+  dsh = d;
+  ifm = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_YES);
+  read_index_list ();
+  return GNUNET_OK;
+}
+
+
+/* end of gnunet-service-fs_indexing.c */
diff --git a/src/service/fs/gnunet-service-fs_indexing.h b/src/service/fs/gnunet-service-fs_indexing.h
new file mode 100644
index 000000000..244a51900
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_indexing.h
@@ -0,0 +1,120 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010, 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_indexing.h
+ * @brief indexing for the file-sharing service
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_INDEXING_H
+#define GNUNET_SERVICE_FS_INDEXING_H
+
+#include "gnunet_block_lib.h"
+#include "gnunet_core_service.h"
+#include "gnunet_datastore_service.h"
+#include "gnunet_protocols.h"
+#include "gnunet_signatures.h"
+#include "gnunet_util_lib.h"
+
+
+/**
+ * We've received an on-demand encoded block from the datastore.
+ * Attempt to do on-demand encoding and (if successful), call the
+ * continuation with the resulting block.  On error, clean up and ask
+ * the datastore for more results.
+ *
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ * @param cont function to call with the actual block (at most once, on success)
+ * @param cont_cls closure for @a cont
+ * @return #GNUNET_OK on success
+ */
+int
+GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key,
+                                  uint32_t size,
+                                  const void *data,
+                                  enum GNUNET_BLOCK_Type type,
+                                  uint32_t priority,
+                                  uint32_t anonymity,
+                                  uint32_t replication,
+                                  struct GNUNET_TIME_Absolute expiration,
+                                  uint64_t uid,
+                                  GNUNET_DATASTORE_DatumProcessor cont,
+                                  void *cont_cls);
+
+
+/**
+ * Transmit information about indexed files to @a mq.
+ *
+ * @param mq message queue to send information to
+ */
+void
+GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq);
+
+
+/**
+ * Remove a file from the index.
+ *
+ * @param fid identifier of the file to remove
+ * @return #GNUNET_YES if the @a fid was found
+ */
+int
+GNUNET_FS_indexing_do_unindex (const struct GNUNET_HashCode *fid);
+
+
+/**
+ * Add the given file to the list of indexed files.
+ *
+ * @param filename name of the file
+ * @param file_id hash identifier for @a filename
+ */
+void
+GNUNET_FS_add_to_index (const char *filename,
+                        const struct GNUNET_HashCode *file_id);
+
+
+/**
+ * Initialize the indexing submodule.
+ *
+ * @param c configuration to use
+ * @param d datastore to use
+ * @return GNUNET_OK on success
+ */
+int
+GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c,
+                         struct GNUNET_DATASTORE_Handle *d);
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GNUNET_FS_indexing_done (void);
+
+
+#endif
diff --git a/src/service/fs/gnunet-service-fs_pe.c b/src/service/fs/gnunet-service-fs_pe.c
new file mode 100644
index 000000000..60dd0ab70
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_pe.c
@@ -0,0 +1,814 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_pe.c
+ * @brief API to manage query plan
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_cp.h"
+#include "gnunet-service-fs_pe.h"
+#include "gnunet-service-fs_pr.h"
+
+/**
+ * Collect an instance number of statistics?  May cause excessive IPC.
+ */
+#define INSANE_STATISTICS GNUNET_NO
+
+/**
+ * List of GSF_PendingRequests this request plan
+ * participates with.
+ */
+struct PendingRequestList;
+
+/**
+ * Transmission plan for a peer.
+ */
+struct PeerPlan;
+
+
+/**
+ * M:N binding of plans to pending requests.
+ * Each pending request can be in a number of plans,
+ * and each plan can have a number of pending requests.
+ * Objects of this type indicate a mapping of a plan to
+ * a particular pending request.
+ *
+ * The corresponding head and tail of the "PE" MDLL
+ * are stored in a `struct GSF_RequestPlan`. (We need
+ * to be able to lookup all pending requests corresponding
+ * to a given plan entry.)
+ *
+ * Similarly head and tail of the "PR" MDLL are stored
+ * with the `struct GSF_PendingRequest`.  (We need
+ * to be able to lookup all plan entries corresponding
+ * to a given pending request.)
+ */
+struct GSF_PendingRequestPlanBijection
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_PendingRequestPlanBijection *next_PR;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_PendingRequestPlanBijection *prev_PR;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_PendingRequestPlanBijection *next_PE;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_PendingRequestPlanBijection *prev_PE;
+
+  /**
+   * Associated request plan (tells us one of the peers that
+   * we plan to forward the request to).
+   */
+  struct GSF_RequestPlan *rp;
+
+  /**
+   * Associated pending request (identifies request details
+   * and one of the origins of the request).
+   */
+  struct GSF_PendingRequest *pr;
+};
+
+
+/**
+ * Information we keep per request per peer.  This is a doubly-linked
+ * list (with head and tail in the `struct GSF_PendingRequestData`)
+ * with one entry in each heap of each `struct PeerPlan`.  Each
+ * entry tracks information relevant for this request and this peer.
+ */
+struct GSF_RequestPlan
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_RequestPlan *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct GSF_RequestPlan *prev;
+
+  /**
+   * Heap node associated with this request and this peer.
+   */
+  struct GNUNET_CONTAINER_HeapNode *hn;
+
+  /**
+   * The transmission plan for a peer that this request is associated with.
+   */
+  struct PeerPlan *pp;
+
+  /**
+   * Head of list of associated pending requests.  This tells us
+   * which incoming requests from other peers this plan entry
+   * corresponds to.
+   */
+  struct GSF_PendingRequestPlanBijection *pe_head;
+
+  /**
+   * Tail of list of associated pending requests.
+   */
+  struct GSF_PendingRequestPlanBijection *pe_tail;
+
+  /**
+   * Earliest time we'd be happy to (re)transmit this request.
+   */
+  struct GNUNET_TIME_Absolute earliest_transmission;
+
+  /**
+   * When was the last time we transmitted this request to this peer? 0 for never.
+   */
+  struct GNUNET_TIME_Absolute last_transmission;
+
+  /**
+   * Current priority for this request for this target.
+   */
+  uint64_t priority;
+
+  /**
+   * How often did we transmit this request to this peer?
+   */
+  unsigned int transmission_counter;
+};
+
+
+/**
+ * Transmission plan for a peer.
+ */
+struct PeerPlan
+{
+  /**
+   * Heap with pending queries (`struct GSF_RequestPlan`), higher weights mean higher priority.
+   */
+  struct GNUNET_CONTAINER_Heap *priority_heap;
+
+  /**
+   * Heap with pending queries (`struct GSF_RequestPlan`), by transmission time, lowest first.
+   */
+  struct GNUNET_CONTAINER_Heap *delay_heap;
+
+  /**
+   * Map of queries to plan entries.  All entries in the @e priority_heap
+   * or @e delay_heap should be in the @e plan_map.  Note that it is
+   * possible for the @e plan_map to have multiple entries for the same
+   * query.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *plan_map;
+
+  /**
+   * Peer for which this is the plan.
+   */
+  struct GSF_ConnectedPeer *cp;
+
+  /**
+   * Current task for executing the plan.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Current message under transmission for the plan.
+   */
+  struct GNUNET_MQ_Envelope *env;
+};
+
+
+/**
+ * Hash map from peer identities to PeerPlans.
+ */
+static struct GNUNET_CONTAINER_MultiPeerMap *plans;
+
+/**
+ * Sum of all transmission counters (equals total delay for all plan entries).
+ */
+static unsigned long long total_delay;
+
+/**
+ * Number of plan entries.
+ */
+static unsigned long long plan_count;
+
+
+/**
+ * Return the query (key in the plan_map) for the given request plan.
+ * Note that this key may change as there can be multiple pending
+ * requests for the same key and we just return _one_ of them; this
+ * particular one might complete while another one might still be
+ * active, hence the lifetime of the returned hash code is NOT
+ * necessarily identical to that of the `struct GSF_RequestPlan`
+ * given.
+ *
+ * @param rp a request plan
+ * @return the associated query
+ */
+static const struct GNUNET_HashCode *
+get_rp_key (struct GSF_RequestPlan *rp)
+{
+  return &GSF_pending_request_get_data_ (rp->pe_head->pr)->query;
+}
+
+
+/**
+ * Insert the given request plan into the heap with the appropriate weight.
+ *
+ * @param pp associated peer's plan
+ * @param rp request to plan
+ */
+static void
+plan (struct PeerPlan *pp,
+      struct GSF_RequestPlan *rp)
+{
+#define N ((double) 128.0)
+  /**
+   * Running average delay we currently impose.
+   */
+  static double avg_delay;
+
+  struct GSF_PendingRequestData *prd;
+  struct GNUNET_TIME_Relative delay;
+
+  GNUNET_assert (rp->pp == pp);
+  GNUNET_STATISTICS_set (GSF_stats,
+                         gettext_noop ("# average retransmission delay (ms)"),
+                         total_delay * 1000LL / plan_count, GNUNET_NO);
+  prd = GSF_pending_request_get_data_ (rp->pe_head->pr);
+
+  if (rp->transmission_counter < 8)
+    delay =
+      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                     rp->transmission_counter);
+  else if (rp->transmission_counter < 32)
+    delay =
+      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                     8
+                                     + (1LL << (rp->transmission_counter - 8)));
+  else
+    delay =
+      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                     8 + (1LL << 24));
+  delay.rel_value_us =
+    GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                              delay.rel_value_us + 1);
+  /* Add 0.01 to avg_delay to avoid division-by-zero later */
+  avg_delay = (((avg_delay * (N - 1.0)) + delay.rel_value_us) / N) + 0.01;
+
+  /*
+   * For the priority, we need to consider a few basic rules:
+   * 1) if we just started requesting (delay is small), we should
+   * virtually always have a priority of zero.
+   * 2) for requests with average latency, our priority should match
+   * the average priority observed on the network
+   * 3) even the longest-running requests should not be WAY out of
+   * the observed average (thus we bound by a factor of 2)
+   * 4) we add +1 to the observed average priority to avoid everyone
+   * staying put at zero (2 * 0 = 0...).
+   *
+   * Using the specific calculation below, we get:
+   *
+   * delay = 0 => priority = 0;
+   * delay = avg delay => priority = running-average-observed-priority;
+   * delay >> avg_delay => priority = 2 * running-average-observed-priority;
+   *
+   * which satisfies all of the rules above.
+   *
+   * Note: M_PI_4 = PI/4 = arctan(1)
+   */rp->priority =
+    round ((GSF_current_priorities
+            + 1.0) * atan (delay.rel_value_us / avg_delay)) / M_PI_4;
+  /* Note: usage of 'round' and 'atan' requires -lm */
+
+  if (rp->transmission_counter != 0)
+    delay.rel_value_us += TTL_DECREMENT * 1000;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Considering (re)transmission number %u in %s\n",
+              (unsigned int) rp->transmission_counter,
+              GNUNET_STRINGS_relative_time_to_string (delay,
+                                                      GNUNET_YES));
+  rp->earliest_transmission = GNUNET_TIME_relative_to_absolute (delay);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Earliest (re)transmission for `%s' in %us\n",
+              GNUNET_h2s (&prd->query),
+              rp->transmission_counter);
+  GNUNET_assert (rp->hn == NULL);
+  if (0 == GNUNET_TIME_absolute_get_remaining (
+        rp->earliest_transmission).rel_value_us)
+    rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap,
+                                           rp,
+                                           rp->priority);
+  else
+    rp->hn =
+      GNUNET_CONTAINER_heap_insert (pp->delay_heap,
+                                    rp,
+                                    rp->earliest_transmission.abs_value_us);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_contains_value (pp->plan_map,
+                                                               get_rp_key (rp),
+                                                               rp));
+#undef N
+}
+
+
+/**
+ * Get the pending request with the highest TTL from the given plan.
+ *
+ * @param rp plan to investigate
+ * @return pending request with highest TTL
+ */
+struct GSF_PendingRequest *
+get_latest (const struct GSF_RequestPlan *rp)
+{
+  struct GSF_PendingRequest *ret;
+  struct GSF_PendingRequestPlanBijection *bi;
+  const struct GSF_PendingRequestData *rprd;
+  const struct GSF_PendingRequestData *prd;
+
+  bi = rp->pe_head;
+  if (NULL == bi)
+    return NULL; /* should never happen */
+  ret = bi->pr;
+  rprd = GSF_pending_request_get_data_ (ret);
+  for (bi = bi->next_PE; NULL != bi; bi = bi->next_PE)
+  {
+    GNUNET_break (GNUNET_YES ==
+                  GSF_pending_request_test_active_ (bi->pr));
+    prd = GSF_pending_request_get_data_ (bi->pr);
+    if (prd->ttl.abs_value_us > rprd->ttl.abs_value_us)
+    {
+      ret = bi->pr;
+      rprd = prd;
+    }
+  }
+  return ret;
+}
+
+
+/**
+ * Figure out when and how to transmit to the given peer.
+ *
+ * @param cls the `struct PeerPlan`
+ */
+static void
+schedule_peer_transmission (void *cls)
+{
+  struct PeerPlan *pp = cls;
+  struct GSF_RequestPlan *rp;
+  struct GNUNET_TIME_Relative delay;
+
+  if (NULL != pp->task)
+  {
+    pp->task = NULL;
+  }
+  else
+  {
+    GNUNET_assert (NULL != pp->env);
+    pp->env = NULL;
+  }
+  /* move ready requests to priority queue */
+  while ((NULL != (rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap))) &&
+         (0 == GNUNET_TIME_absolute_get_remaining
+            (rp->earliest_transmission).rel_value_us))
+  {
+    GNUNET_assert (rp == GNUNET_CONTAINER_heap_remove_root (pp->delay_heap));
+    rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap,
+                                           rp,
+                                           rp->priority);
+  }
+  if (0 == GNUNET_CONTAINER_heap_get_size (pp->priority_heap))
+  {
+    /* priority heap (still) empty, check for delay... */
+    rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap);
+    if (NULL == rp)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "No active requests for plan %p.\n",
+                  pp);
+      return;                   /* both queues empty */
+    }
+    delay = GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Sleeping for %s before retrying requests on plan %p.\n",
+                GNUNET_STRINGS_relative_time_to_string (delay,
+                                                        GNUNET_YES),
+                pp);
+    GNUNET_STATISTICS_set (GSF_stats,
+                           gettext_noop ("# delay heap timeout (ms)"),
+                           delay.rel_value_us / 1000LL, GNUNET_NO);
+
+    pp->task
+      = GNUNET_SCHEDULER_add_at (rp->earliest_transmission,
+                                 &schedule_peer_transmission,
+                                 pp);
+    return;
+  }
+#if INSANE_STATISTICS
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# query plans executed"),
+                            1,
+                            GNUNET_NO);
+#endif
+  /* process from priority heap */
+  rp = GNUNET_CONTAINER_heap_remove_root (pp->priority_heap);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Executing query plan %p\n",
+              rp);
+  GNUNET_assert (NULL != rp);
+  rp->hn = NULL;
+  rp->last_transmission = GNUNET_TIME_absolute_get ();
+  rp->transmission_counter++;
+  total_delay++;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Executing plan %p executed %u times, planning retransmission\n",
+              rp,
+              rp->transmission_counter);
+  GNUNET_assert (NULL == pp->env);
+  pp->env = GSF_pending_request_get_message_ (get_latest (rp));
+  GNUNET_MQ_notify_sent (pp->env,
+                         &schedule_peer_transmission,
+                         pp);
+  GSF_peer_transmit_ (pp->cp,
+                      GNUNET_YES,
+                      rp->priority,
+                      pp->env);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop (
+                              "# query messages sent to other peers"),
+                            1,
+                            GNUNET_NO);
+  plan (pp,
+        rp);
+}
+
+
+/**
+ * Closure for merge_pr().
+ */
+struct MergeContext
+{
+  /**
+   * Request we are trying to merge.
+   */
+  struct GSF_PendingRequest *pr;
+
+  /**
+   * Set to #GNUNET_YES if we succeeded to merge.
+   */
+  int merged;
+};
+
+
+/**
+ * Iterator that checks if an equivalent request is already
+ * present for this peer.
+ *
+ * @param cls closure
+ * @param query the query
+ * @param element request plan stored at the node
+ * @return #GNUNET_YES if we should continue to iterate,
+ *         #GNUNET_NO if not (merge success)
+ */
+static int
+merge_pr (void *cls,
+          const struct GNUNET_HashCode *query,
+          void *element)
+{
+  struct MergeContext *mpr = cls;
+  struct GSF_RequestPlan *rp = element;
+  struct GSF_PendingRequestData *prd;
+  struct GSF_PendingRequestPlanBijection *bi;
+  struct GSF_PendingRequest *latest;
+
+  GNUNET_break (GNUNET_YES ==
+                GSF_pending_request_test_active_ (mpr->pr));
+  if (GNUNET_OK !=
+      GSF_pending_request_is_compatible_ (mpr->pr,
+                                          rp->pe_head->pr))
+    return GNUNET_YES;
+  /* merge new request with existing request plan */
+  bi = GNUNET_new (struct GSF_PendingRequestPlanBijection);
+  bi->rp = rp;
+  bi->pr = mpr->pr;
+  prd = GSF_pending_request_get_data_ (mpr->pr);
+  GNUNET_CONTAINER_MDLL_insert (PR,
+                                prd->pr_head,
+                                prd->pr_tail,
+                                bi);
+  GNUNET_CONTAINER_MDLL_insert (PE,
+                                rp->pe_head,
+                                rp->pe_tail,
+                                bi);
+  mpr->merged = GNUNET_YES;
+#if INSANE_STATISTICS
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# requests merged"),
+                            1,
+                            GNUNET_NO);
+#endif
+  latest = get_latest (rp);
+  if (GSF_pending_request_get_data_ (latest)->ttl.abs_value_us <
+      prd->ttl.abs_value_us)
+  {
+#if INSANE_STATISTICS
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# requests refreshed"),
+                              1,
+                              GNUNET_NO);
+#endif
+    rp->transmission_counter = 0;       /* reset */
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Create a new query plan entry.
+ *
+ * @param cp peer with the entry
+ * @param pr request with the entry
+ */
+void
+GSF_plan_add_ (struct GSF_ConnectedPeer *cp,
+               struct GSF_PendingRequest *pr)
+{
+  const struct GNUNET_PeerIdentity *id;
+  struct PeerPlan *pp;
+  struct GSF_PendingRequestData *prd;
+  struct GSF_RequestPlan *rp;
+  struct GSF_PendingRequestPlanBijection *bi;
+  struct MergeContext mpc;
+
+  GNUNET_assert (GNUNET_YES ==
+                 GSF_pending_request_test_active_ (pr));
+  GNUNET_assert (NULL != cp);
+  id = GSF_connected_peer_get_identity2_ (cp);
+  pp = GNUNET_CONTAINER_multipeermap_get (plans, id);
+  if (NULL == pp)
+  {
+    pp = GNUNET_new (struct PeerPlan);
+    pp->plan_map = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_NO);
+    pp->priority_heap =
+      GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MAX);
+    pp->delay_heap =
+      GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
+    pp->cp = cp;
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_CONTAINER_multipeermap_put (plans,
+                                                      id,
+                                                      pp,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+    pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission,
+                                         pp);
+  }
+  mpc.merged = GNUNET_NO;
+  mpc.pr = pr;
+  prd = GSF_pending_request_get_data_ (pr);
+  GNUNET_CONTAINER_multihashmap_get_multiple (pp->plan_map,
+                                              &prd->query,
+                                              &merge_pr,
+                                              &mpc);
+  if (GNUNET_NO != mpc.merged)
+    return;
+  plan_count++;
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# query plan entries"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Planning transmission of query `%s' to peer `%s'\n",
+              GNUNET_h2s (&prd->query),
+              GNUNET_i2s (id));
+  rp = GNUNET_new (struct GSF_RequestPlan);
+  bi = GNUNET_new (struct GSF_PendingRequestPlanBijection);
+  bi->rp = rp;
+  bi->pr = pr;
+  GNUNET_CONTAINER_MDLL_insert (PR,
+                                prd->pr_head,
+                                prd->pr_tail,
+                                bi);
+  GNUNET_CONTAINER_MDLL_insert (PE,
+                                rp->pe_head,
+                                rp->pe_tail,
+                                bi);
+  rp->pp = pp;
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_put (pp->plan_map,
+                                                    get_rp_key (rp),
+                                                    rp,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+  plan (pp,
+        rp);
+}
+
+
+/**
+ * Notify the plan about a peer being no longer available;
+ * destroy all entries associated with this peer.
+ *
+ * @param cp connected peer
+ */
+void
+GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp)
+{
+  const struct GNUNET_PeerIdentity *id;
+  struct PeerPlan *pp;
+  struct GSF_RequestPlan *rp;
+  struct GSF_PendingRequestData *prd;
+  struct GSF_PendingRequestPlanBijection *bi;
+
+  id = GSF_connected_peer_get_identity2_ (cp);
+  pp = GNUNET_CONTAINER_multipeermap_get (plans, id);
+  if (NULL == pp)
+    return;                     /* nothing was ever planned for this peer */
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (plans, id,
+                                                       pp));
+  if (NULL != pp->task)
+  {
+    GNUNET_SCHEDULER_cancel (pp->task);
+    pp->task = NULL;
+  }
+  while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->priority_heap)))
+  {
+    GNUNET_break (GNUNET_YES ==
+                  GNUNET_CONTAINER_multihashmap_remove (pp->plan_map,
+                                                        get_rp_key (rp),
+                                                        rp));
+    while (NULL != (bi = rp->pe_head))
+    {
+      GNUNET_CONTAINER_MDLL_remove (PE,
+                                    rp->pe_head,
+                                    rp->pe_tail,
+                                    bi);
+      prd = GSF_pending_request_get_data_ (bi->pr);
+      GNUNET_CONTAINER_MDLL_remove (PR,
+                                    prd->pr_head,
+                                    prd->pr_tail,
+                                    bi);
+      GNUNET_free (bi);
+    }
+    plan_count--;
+    GNUNET_free (rp);
+  }
+  GNUNET_CONTAINER_heap_destroy (pp->priority_heap);
+  while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->delay_heap)))
+  {
+    GNUNET_break (GNUNET_YES ==
+                  GNUNET_CONTAINER_multihashmap_remove (pp->plan_map,
+                                                        get_rp_key (rp),
+                                                        rp));
+    while (NULL != (bi = rp->pe_head))
+    {
+      prd = GSF_pending_request_get_data_ (bi->pr);
+      GNUNET_CONTAINER_MDLL_remove (PE,
+                                    rp->pe_head,
+                                    rp->pe_tail,
+                                    bi);
+      GNUNET_CONTAINER_MDLL_remove (PR,
+                                    prd->pr_head,
+                                    prd->pr_tail,
+                                    bi);
+      GNUNET_free (bi);
+    }
+    plan_count--;
+    GNUNET_free (rp);
+  }
+  GNUNET_STATISTICS_set (GSF_stats,
+                         gettext_noop ("# query plan entries"),
+                         plan_count,
+                         GNUNET_NO);
+  GNUNET_CONTAINER_heap_destroy (pp->delay_heap);
+  GNUNET_CONTAINER_multihashmap_destroy (pp->plan_map);
+  GNUNET_free (pp);
+}
+
+
+/**
+ * Get the last transmission attempt time for the request plan list
+ * referenced by @a pr_head, that was sent to @a sender
+ *
+ * @param pr_head request plan reference list to check.
+ * @param sender the peer that we've sent the request to.
+ * @param result the timestamp to fill, set to #GNUNET_TIME_UNIT_FOREVER_ABS if never transmitted
+ * @return #GNUNET_YES if @a result was changed, #GNUNET_NO otherwise.
+ */
+int
+GSF_request_plan_reference_get_last_transmission_ (struct
+                                                   GSF_PendingRequestPlanBijection
+                                                   *pr_head,
+                                                   struct GSF_ConnectedPeer *
+                                                   sender,
+                                                   struct GNUNET_TIME_Absolute *
+                                                   result)
+{
+  struct GSF_PendingRequestPlanBijection *bi;
+
+  for (bi = pr_head; NULL != bi; bi = bi->next_PR)
+  {
+    if (bi->rp->pp->cp == sender)
+    {
+      if (0 == bi->rp->last_transmission.abs_value_us)
+        *result = GNUNET_TIME_UNIT_FOREVER_ABS;
+      else
+        *result = bi->rp->last_transmission;
+      return GNUNET_YES;
+    }
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Notify the plan about a request being done; destroy all entries
+ * associated with this request.
+ *
+ * @param pr request that is done
+ */
+void
+GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr)
+{
+  struct GSF_RequestPlan *rp;
+  struct GSF_PendingRequestData *prd;
+  struct GSF_PendingRequestPlanBijection *bi;
+
+  prd = GSF_pending_request_get_data_ (pr);
+  while (NULL != (bi = prd->pr_head))
+  {
+    rp = bi->rp;
+    GNUNET_CONTAINER_MDLL_remove (PR,
+                                  prd->pr_head,
+                                  prd->pr_tail,
+                                  bi);
+    GNUNET_CONTAINER_MDLL_remove (PE,
+                                  rp->pe_head,
+                                  rp->pe_tail,
+                                  bi);
+    GNUNET_assert (bi->pr == pr);
+    if (NULL == rp->pe_head)
+    {
+      GNUNET_CONTAINER_heap_remove_node (rp->hn);
+      plan_count--;
+      GNUNET_break (GNUNET_YES ==
+                    GNUNET_CONTAINER_multihashmap_remove (rp->pp->plan_map,
+                                                          &prd->query,
+                                                          rp));
+      GNUNET_free (rp);
+    }
+    GNUNET_free (bi);
+  }
+  GNUNET_STATISTICS_set (GSF_stats,
+                         gettext_noop ("# query plan entries"),
+                         plan_count,
+                         GNUNET_NO);
+}
+
+
+/**
+ * Initialize plan subsystem.
+ */
+void
+GSF_plan_init ()
+{
+  plans = GNUNET_CONTAINER_multipeermap_create (256,
+                                                GNUNET_YES);
+}
+
+
+/**
+ * Shutdown plan subsystem.
+ */
+void
+GSF_plan_done ()
+{
+  GNUNET_assert (0 == GNUNET_CONTAINER_multipeermap_size (plans));
+  GNUNET_CONTAINER_multipeermap_destroy (plans);
+}
+
+
+/* end of gnunet-service-fs_pe.h */
diff --git a/src/service/fs/gnunet-service-fs_pe.h b/src/service/fs/gnunet-service-fs_pe.h
new file mode 100644
index 000000000..d532b6b71
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_pe.h
@@ -0,0 +1,95 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_pe.h
+ * @brief API to manage query plan
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_PE_H
+#define GNUNET_SERVICE_FS_PE_H
+
+#include "gnunet-service-fs.h"
+
+
+/**
+ * Create a new query plan entry.
+ *
+ * @param cp peer with the entry
+ * @param pr request with the entry
+ */
+void
+GSF_plan_add_ (struct GSF_ConnectedPeer *cp,
+               struct GSF_PendingRequest *pr);
+
+
+/**
+ * Notify the plan about a peer being no longer available;
+ * destroy all entries associated with this peer.
+ *
+ * @param cp connected peer
+ */
+void
+GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp);
+
+
+/**
+ * Notify the plan about a request being done;
+ * destroy all entries associated with this request.
+ *
+ * @param pr request that is done
+ */
+void
+GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr);
+
+/**
+ * Get the last transmission attempt time for the request plan list
+ * referenced by 'rpr_head', that was sent to 'sender'
+ *
+ * @param pr_head request plan reference list to check.
+ * @param sender the peer that we've sent the request to.
+ * @param result the timestamp to fill.
+ * @return GNUNET_YES if 'result' was changed, GNUNET_NO otherwise.
+ */
+int
+GSF_request_plan_reference_get_last_transmission_ (struct
+                                                   GSF_PendingRequestPlanBijection
+                                                   *pr_head,
+                                                   struct GSF_ConnectedPeer *
+                                                   sender,
+                                                   struct GNUNET_TIME_Absolute *
+                                                   result);
+
+/**
+ * Initialize plan subsystem.
+ */
+void
+GSF_plan_init (void);
+
+
+/**
+ * Shutdown plan subsystem.
+ */
+void
+GSF_plan_done (void);
+
+
+#endif
+/* end of gnunet-service-fs_pe.h */
diff --git a/src/service/fs/gnunet-service-fs_pr.c b/src/service/fs/gnunet-service-fs_pr.c
new file mode 100644
index 000000000..f192c017d
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_pr.c
@@ -0,0 +1,1887 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_pr.c
+ * @brief API to handle pending requests
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_load_lib.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_cp.h"
+#include "gnunet-service-fs_indexing.h"
+#include "gnunet-service-fs_pe.h"
+#include "gnunet-service-fs_pr.h"
+#include "gnunet-service-fs_cadet.h"
+
+
+/**
+ * Desired replication level for GETs.
+ */
+#define DHT_GET_REPLICATION 5
+
+/**
+ * Maximum size of the datastore queue for P2P operations.  Needs to
+ * be large enough to queue #MAX_QUEUE_PER_PEER operations for roughly
+ * the number of active (connected) peers.
+ */
+#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER)
+
+/**
+ * Bandwidth value of a 0-priority content (must be fairly high
+ * compared to query since content is typically significantly larger
+ * -- and more valuable since it can take many queries to get one
+ * piece of content).
+ */
+#define CONTENT_BANDWIDTH_VALUE 800
+
+/**
+ * Hard limit on the number of results we may get from the datastore per query.
+ */
+#define MAX_RESULTS (100 * 1024)
+
+/**
+ * Collect an instance number of statistics?  May cause excessive IPC.
+ */
+#define INSANE_STATISTICS GNUNET_NO
+
+/**
+ * If obtaining a block via cadet fails, how often do we retry it before
+ * giving up for good (and sticking to non-anonymous transfer)?
+ */
+#define CADET_RETRY_MAX 3
+
+
+/**
+ * An active request.
+ */
+struct GSF_PendingRequest
+{
+  /**
+   * Public data for the request.
+   */
+  struct GSF_PendingRequestData public_data;
+
+  /**
+   * Function to call if we encounter a reply.
+   */
+  GSF_PendingRequestReplyHandler rh;
+
+  /**
+   * Closure for @e rh
+   */
+  void *rh_cls;
+
+  /**
+   * Array of hash codes of replies we've already seen.
+   */
+  struct GNUNET_HashCode *replies_seen;
+
+  /**
+   * Block group for filtering replies we've already seen.
+   */
+  struct GNUNET_BLOCK_Group *bg;
+
+  /**
+   * Entry for this pending request in the expiration heap, or NULL.
+   */
+  struct GNUNET_CONTAINER_HeapNode *hnode;
+
+  /**
+   * Datastore queue entry for this request (or NULL for none).
+   */
+  struct GNUNET_DATASTORE_QueueEntry *qe;
+
+  /**
+   * DHT request handle for this request (or NULL for none).
+   */
+  struct GNUNET_DHT_GetHandle *gh;
+
+  /**
+   * Cadet request handle for this request (or NULL for none).
+   */
+  struct GSF_CadetRequest *cadet_request;
+
+  /**
+   * Function to call upon completion of the local get
+   * request, or NULL for none.
+   */
+  GSF_LocalLookupContinuation llc_cont;
+
+  /**
+   * Closure for @e llc_cont.
+   */
+  void *llc_cont_cls;
+
+  /**
+   * Last result from the local datastore lookup evaluation.
+   */
+  enum GNUNET_BLOCK_ReplyEvaluationResult local_result;
+
+  /**
+   * Identity of the peer that we should use for the 'sender'
+   * (recipient of the response) when forwarding (0 for none).
+   */
+  GNUNET_PEER_Id sender_pid;
+
+  /**
+   * Identity of the peer that we should never forward this query
+   * to since it originated this query (0 for none).
+   */
+  GNUNET_PEER_Id origin_pid;
+
+  /**
+   * Time we started the last datastore lookup.
+   */
+  struct GNUNET_TIME_Absolute qe_start;
+
+  /**
+   * Task that warns us if the local datastore lookup takes too long.
+   */
+  struct GNUNET_SCHEDULER_Task *warn_task;
+
+  /**
+   * Do we have a first UID yet?
+   */
+  bool have_first_uid;
+
+  /**
+   * Have we seen a NULL result yet?
+   */
+  bool seen_null;
+
+  /**
+   * Unique ID of the first result from the local datastore;
+   * used to terminate the loop.
+   */
+  uint64_t first_uid;
+
+  /**
+   * Result count.
+   */
+  size_t result_count;
+
+  /**
+   * How often have we retried this request via 'cadet'?
+   * (used to bound overall retries).
+   */
+  unsigned int cadet_retry_count;
+
+  /**
+   * Number of valid entries in the 'replies_seen' array.
+   */
+  unsigned int replies_seen_count;
+
+  /**
+   * Length of the 'replies_seen' array.
+   */
+  unsigned int replies_seen_size;
+};
+
+
+/**
+ * All pending requests, ordered by the query.  Entries
+ * are of type 'struct GSF_PendingRequest*'.
+ */
+static struct GNUNET_CONTAINER_MultiHashMap *pr_map;
+
+
+/**
+ * Datastore 'PUT' load tracking.
+ */
+static struct GNUNET_LOAD_Value *datastore_put_load;
+
+
+/**
+ * Are we allowed to migrate content to this peer.
+ */
+static int active_to_migration;
+
+
+/**
+ * Heap with the request that will expire next at the top.  Contains
+ * pointers of type "struct PendingRequest*"; these will *also* be
+ * aliased from the "requests_by_peer" data structures and the
+ * "requests_by_query" table.  Note that requests from our clients
+ * don't expire and are thus NOT in the "requests_by_expiration"
+ * (or the "requests_by_peer" tables).
+ */
+static struct GNUNET_CONTAINER_Heap *requests_by_expiration_heap;
+
+
+/**
+ * Maximum number of requests (from other peers, overall) that we're
+ * willing to have pending at any given point in time.  Can be changed
+ * via the configuration file (32k is just the default).
+ */
+static unsigned long long max_pending_requests = (32 * 1024);
+
+
+/**
+ * Recalculate our bloom filter for filtering replies.  This function
+ * will create a new bloom filter from scratch, so it should only be
+ * called if we have no bloomfilter at all (and hence can create a
+ * fresh one of minimal size without problems) OR if our peer is the
+ * initiator (in which case we may resize to larger than minimum size).
+ *
+ * @param type type of the request
+ * @param pr request for which the BF is to be recomputed
+ */
+static void
+refresh_bloomfilter (enum GNUNET_BLOCK_Type type,
+                     struct GSF_PendingRequest *pr)
+{
+  if (NULL != pr->bg)
+  {
+    GNUNET_BLOCK_group_destroy (pr->bg);
+    pr->bg = NULL;
+  }
+  if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type)
+    return; /* no need */
+  pr->bg =
+    GNUNET_BLOCK_group_create (GSF_block_ctx,
+                               type,
+                               NULL,
+                               0,
+                               "seen-set-size",
+                               pr->replies_seen_count,
+                               NULL);
+  if (NULL == pr->bg)
+    return;
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_BLOCK_group_set_seen (pr->bg,
+                                             pr->replies_seen,
+                                             pr->replies_seen_count));
+}
+
+
+struct GSF_PendingRequest *
+GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
+                             enum GNUNET_BLOCK_Type type,
+                             const struct GNUNET_HashCode *query,
+                             const struct GNUNET_PeerIdentity *target,
+                             const char *bf_data,
+                             size_t bf_size,
+                             uint32_t anonymity_level,
+                             uint32_t priority,
+                             int32_t ttl,
+                             GNUNET_PEER_Id sender_pid,
+                             GNUNET_PEER_Id origin_pid,
+                             const struct GNUNET_HashCode *replies_seen,
+                             unsigned int replies_seen_count,
+                             GSF_PendingRequestReplyHandler rh,
+                             void *rh_cls)
+{
+  struct GSF_PendingRequest *pr;
+  struct GSF_PendingRequest *dpr;
+  size_t extra;
+  struct GNUNET_HashCode *eptr;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Creating request handle for `%s' of type %d\n",
+              GNUNET_h2s (query),
+              type);
+#if INSANE_STATISTICS
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Pending requests created"),
+                            1,
+                            GNUNET_NO);
+#endif
+  extra = 0;
+  if (NULL != target)
+    extra += sizeof(struct GNUNET_PeerIdentity);
+  pr = GNUNET_malloc (sizeof(struct GSF_PendingRequest) + extra);
+  pr->public_data.query = *query;
+  eptr = (struct GNUNET_HashCode *) &pr[1];
+  if (NULL != target)
+  {
+    pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr;
+    GNUNET_memcpy (eptr, target, sizeof(struct GNUNET_PeerIdentity));
+  }
+  pr->public_data.anonymity_level = anonymity_level;
+  pr->public_data.priority = priority;
+  pr->public_data.original_priority = priority;
+  pr->public_data.options = options;
+  pr->public_data.type = type;
+  pr->public_data.start_time = GNUNET_TIME_absolute_get ();
+  pr->sender_pid = sender_pid;
+  pr->origin_pid = origin_pid;
+  pr->rh = rh;
+  pr->rh_cls = rh_cls;
+  GNUNET_assert ((sender_pid != 0) || (0 == (options & GSF_PRO_FORWARD_ONLY)));
+  if (ttl >= 0)
+    pr->public_data.ttl = GNUNET_TIME_relative_to_absolute (
+      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, (uint32_t) ttl));
+  else
+    pr->public_data.ttl = GNUNET_TIME_absolute_subtract (
+      pr->public_data.start_time,
+      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                     (uint32_t) (-ttl)));
+  if (replies_seen_count > 0)
+  {
+    pr->replies_seen_size = replies_seen_count;
+    pr->replies_seen =
+      GNUNET_new_array (pr->replies_seen_size, struct GNUNET_HashCode);
+    GNUNET_memcpy (pr->replies_seen,
+                   replies_seen,
+                   replies_seen_count * sizeof(struct GNUNET_HashCode));
+    pr->replies_seen_count = replies_seen_count;
+  }
+  if ((NULL != bf_data) &&
+      (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type))
+  {
+    pr->bg = GNUNET_BLOCK_group_create (GSF_block_ctx,
+                                        pr->public_data.type,
+                                        bf_data,
+                                        bf_size,
+                                        "seen-set-size",
+                                        0,
+                                        NULL);
+  }
+  else if ((replies_seen_count > 0) &&
+           (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)))
+  {
+    refresh_bloomfilter (pr->public_data.type, pr);
+  }
+  GNUNET_CONTAINER_multihashmap_put (pr_map,
+                                     &pr->public_data.query,
+                                     pr,
+                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  if (0 == (options & GSF_PRO_REQUEST_NEVER_EXPIRES))
+  {
+    pr->hnode = GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap,
+                                              pr,
+                                              pr->public_data.ttl.abs_value_us);
+    /* make sure we don't track too many requests */
+    while (GNUNET_CONTAINER_heap_get_size (requests_by_expiration_heap) >
+           max_pending_requests)
+    {
+      dpr = GNUNET_CONTAINER_heap_peek (requests_by_expiration_heap);
+      GNUNET_assert (NULL != dpr);
+      if (pr == dpr)
+        break;     /* let the request live briefly... */
+      if (NULL != dpr->rh)
+        dpr->rh (dpr->rh_cls,
+                 GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED,
+                 dpr,
+                 UINT32_MAX,
+                 GNUNET_TIME_UNIT_FOREVER_ABS,
+                 GNUNET_TIME_UNIT_FOREVER_ABS,
+                 GNUNET_BLOCK_TYPE_ANY,
+                 NULL,
+                 0);
+      GSF_pending_request_cancel_ (dpr, GNUNET_YES);
+    }
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Pending requests active"),
+                            1,
+                            GNUNET_NO);
+  return pr;
+}
+
+
+/**
+ * Obtain the public data associated with a pending request
+ *
+ * @param pr pending request
+ * @return associated public data
+ */
+struct GSF_PendingRequestData *
+GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr)
+{
+  return &pr->public_data;
+}
+
+
+/**
+ * Test if two pending requests are compatible (would generate
+ * the same query modulo filters and should thus be processed
+ * jointly).
+ *
+ * @param pra a pending request
+ * @param prb another pending request
+ * @return #GNUNET_OK if the requests are compatible
+ */
+int
+GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra,
+                                    struct GSF_PendingRequest *prb)
+{
+  if ((pra->public_data.type != prb->public_data.type) ||
+      (0 != memcmp (&pra->public_data.query,
+                    &prb->public_data.query,
+                    sizeof(struct GNUNET_HashCode))))
+    return GNUNET_NO;
+  return GNUNET_OK;
+}
+
+
+void
+GSF_pending_request_update_ (struct GSF_PendingRequest *pr,
+                             const struct GNUNET_HashCode *replies_seen,
+                             unsigned int replies_seen_count)
+{
+  if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count)
+    return; /* integer overflow */
+  if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))
+  {
+    /* we're responsible for the BF, full refresh */
+    if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size)
+      GNUNET_array_grow (pr->replies_seen,
+                         pr->replies_seen_size,
+                         replies_seen_count + pr->replies_seen_count);
+    GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count],
+                   replies_seen,
+                   sizeof(struct GNUNET_HashCode) * replies_seen_count);
+    pr->replies_seen_count += replies_seen_count;
+    refresh_bloomfilter (pr->public_data.type, pr);
+  }
+  else
+  {
+    if (NULL == pr->bg)
+    {
+      /* we're not the initiator, but the initiator did not give us
+       * any bloom-filter, so we need to create one on-the-fly */
+      refresh_bloomfilter (pr->public_data.type, pr);
+    }
+    else
+    {
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_BLOCK_group_set_seen (pr->bg,
+                                                 replies_seen,
+                                                 pr->replies_seen_count));
+    }
+  }
+  if (NULL != pr->gh)
+    GNUNET_DHT_get_filter_known_results (pr->gh,
+                                         replies_seen_count,
+                                         replies_seen);
+}
+
+
+/**
+ * Generate the message corresponding to the given pending request for
+ * transmission to other peers.
+ *
+ * @param pr request to generate the message for
+ * @return envelope with the request message
+ */
+struct GNUNET_MQ_Envelope *
+GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GetMessage *gm;
+  struct GNUNET_PeerIdentity *ext;
+  unsigned int k;
+  uint32_t bm;
+  uint32_t prio;
+  size_t bf_size;
+  struct GNUNET_TIME_Absolute now;
+  int64_t ttl;
+  int do_route;
+  void *bf_data;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Building request message for `%s' of type %d\n",
+              GNUNET_h2s (&pr->public_data.query),
+              pr->public_data.type);
+  k = 0;
+  bm = 0;
+  do_route = (0 == (pr->public_data.options & GSF_PRO_FORWARD_ONLY));
+  if ((! do_route) && (pr->sender_pid == 0))
+  {
+    GNUNET_break (0);
+    do_route = GNUNET_YES;
+  }
+  if (! do_route)
+  {
+    bm |= GET_MESSAGE_BIT_RETURN_TO;
+    k++;
+  }
+  if (NULL != pr->public_data.target)
+  {
+    bm |= GET_MESSAGE_BIT_TRANSMIT_TO;
+    k++;
+  }
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_group_serialize (pr->bg,
+                                    &bf_data,
+                                    &bf_size))
+  {
+    bf_size = 0;
+    bf_data = NULL;
+  }
+  env = GNUNET_MQ_msg_extra (gm,
+                             bf_size + k * sizeof(struct GNUNET_PeerIdentity),
+                             GNUNET_MESSAGE_TYPE_FS_GET);
+  gm->type = htonl (pr->public_data.type);
+  if (do_route)
+    prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                     pr->public_data.priority + 1);
+  else
+    prio = 0;
+  pr->public_data.priority -= prio;
+  pr->public_data.num_transmissions++;
+  pr->public_data.respect_offered += prio;
+  gm->priority = htonl (prio);
+  now = GNUNET_TIME_absolute_get ();
+  ttl = (int64_t) (pr->public_data.ttl.abs_value_us - now.abs_value_us);
+  gm->ttl = htonl (ttl / 1000LL / 1000LL);
+  gm->reserved = htonl (0);
+  gm->hash_bitmap = htonl (bm);
+  gm->query = pr->public_data.query;
+  ext = (struct GNUNET_PeerIdentity *) &gm[1];
+  k = 0;
+  if (! do_route)
+    GNUNET_PEER_resolve (pr->sender_pid, &ext[k++]);
+  if (NULL != pr->public_data.target)
+    ext[k++] = *pr->public_data.target;
+  GNUNET_memcpy (&ext[k], bf_data, bf_size);
+  GNUNET_free (bf_data);
+  return env;
+}
+
+
+/**
+ * Iterator to free pending requests.
+ *
+ * @param cls closure, unused
+ * @param key current key code
+ * @param value value in the hash map (pending request)
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+clean_request (void *cls, const struct GNUNET_HashCode *key, void *value)
+{
+  struct GSF_PendingRequest *pr = value;
+  GSF_LocalLookupContinuation cont;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Cleaning up pending request for `%s'.\n",
+              GNUNET_h2s (key));
+  if (NULL != pr->cadet_request)
+  {
+    pr->cadet_retry_count = CADET_RETRY_MAX;
+    GSF_cadet_query_cancel (pr->cadet_request);
+    pr->cadet_request = NULL;
+  }
+  if (NULL != (cont = pr->llc_cont))
+  {
+    pr->llc_cont = NULL;
+    cont (pr->llc_cont_cls,
+          pr,
+          pr->local_result);
+  }
+  GSF_plan_notify_request_done_ (pr);
+  GNUNET_free (pr->replies_seen);
+  GNUNET_BLOCK_group_destroy (pr->bg);
+  pr->bg = NULL;
+  GNUNET_PEER_change_rc (pr->sender_pid, -1);
+  pr->sender_pid = 0;
+  GNUNET_PEER_change_rc (pr->origin_pid, -1);
+  pr->origin_pid = 0;
+  if (NULL != pr->hnode)
+  {
+    GNUNET_CONTAINER_heap_remove_node (pr->hnode);
+    pr->hnode = NULL;
+  }
+  if (NULL != pr->qe)
+  {
+    GNUNET_DATASTORE_cancel (pr->qe);
+    pr->qe = NULL;
+  }
+  if (NULL != pr->gh)
+  {
+    GNUNET_DHT_get_stop (pr->gh);
+    pr->gh = NULL;
+  }
+  if (NULL != pr->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (pr->warn_task);
+    pr->warn_task = NULL;
+  }
+  GNUNET_assert (
+    GNUNET_OK ==
+    GNUNET_CONTAINER_multihashmap_remove (pr_map, &pr->public_data.query, pr));
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Pending requests active"),
+                            -1,
+                            GNUNET_NO);
+  GNUNET_free (pr);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Explicitly cancel a pending request.
+ *
+ * @param pr request to cancel
+ * @param full_cleanup fully purge the request
+ */
+void
+GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, int full_cleanup)
+{
+  GSF_LocalLookupContinuation cont;
+
+  if (NULL == pr_map)
+    return; /* already cleaned up! */
+  if (GNUNET_NO == full_cleanup)
+  {
+    /* make request inactive (we're no longer interested in more results),
+     * but do NOT remove from our data-structures, we still need it there
+     * to prevent the request from looping */
+    pr->rh = NULL;
+    if (NULL != pr->cadet_request)
+    {
+      pr->cadet_retry_count = CADET_RETRY_MAX;
+      GSF_cadet_query_cancel (pr->cadet_request);
+      pr->cadet_request = NULL;
+    }
+    if (NULL != (cont = pr->llc_cont))
+    {
+      pr->llc_cont = NULL;
+      cont (pr->llc_cont_cls,
+            pr,
+            pr->local_result);
+    }
+    GSF_plan_notify_request_done_ (pr);
+    if (NULL != pr->qe)
+    {
+      GNUNET_DATASTORE_cancel (pr->qe);
+      pr->qe = NULL;
+    }
+    if (NULL != pr->gh)
+    {
+      GNUNET_DHT_get_stop (pr->gh);
+      pr->gh = NULL;
+    }
+    if (NULL != pr->warn_task)
+    {
+      GNUNET_SCHEDULER_cancel (pr->warn_task);
+      pr->warn_task = NULL;
+    }
+    return;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 clean_request (NULL, &pr->public_data.query, pr));
+}
+
+
+void
+GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls)
+{
+  GNUNET_CONTAINER_multihashmap_iterate (
+    pr_map,
+    (GNUNET_CONTAINER_MultiHashMapIteratorCallback) it,
+    cls);
+}
+
+
+/**
+ * Closure for process_reply() function.
+ */
+struct ProcessReplyClosure
+{
+  /**
+   * The data for the reply.
+   */
+  const void *data;
+
+  /**
+   * Who gave us this reply? NULL for local host (or DHT)
+   */
+  struct GSF_ConnectedPeer *sender;
+
+  /**
+   * When the reply expires.
+   */
+  struct GNUNET_TIME_Absolute expiration;
+
+  /**
+   * Size of data.
+   */
+  size_t size;
+
+  /**
+   * Type of the block.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * How much was this reply worth to us?
+   */
+  uint32_t priority;
+
+  /**
+   * Anonymity requirements for this reply.
+   */
+  uint32_t anonymity_level;
+
+  /**
+   * Evaluation result (returned).
+   */
+  enum GNUNET_BLOCK_ReplyEvaluationResult eval;
+
+  /**
+   * Did we find a matching request?
+   */
+  int request_found;
+};
+
+
+/**
+ * Update the performance data for the sender (if any) since
+ * the sender successfully answered one of our queries.
+ *
+ * @param prq information about the sender
+ * @param pr request that was satisfied
+ */
+static void
+update_request_performance_data (struct ProcessReplyClosure *prq,
+                                 struct GSF_PendingRequest *pr)
+{
+  if (prq->sender == NULL)
+    return;
+  GSF_peer_update_performance_ (prq->sender,
+                                pr->public_data.start_time,
+                                prq->priority);
+}
+
+
+/**
+ * We have received a reply; handle it!
+ *
+ * @param cls response (a `struct ProcessReplyClosure`)
+ * @param key our query
+ * @param value value in the hash map (info about the query)
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static enum GNUNET_GenericReturnValue
+process_reply (void *cls,
+               const struct GNUNET_HashCode *key,
+               void *value)
+{
+  struct ProcessReplyClosure *prq = cls;
+  struct GSF_PendingRequest *pr = value;
+  struct GNUNET_HashCode chash;
+  struct GNUNET_TIME_Absolute last_transmission;
+
+  if (NULL == pr->rh)
+    return GNUNET_YES;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Matched result (type %u) for query `%s' with pending request\n",
+              (unsigned int) prq->type,
+              GNUNET_h2s (key));
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# replies received and matched"),
+                            1,
+                            GNUNET_NO);
+  prq->eval = GNUNET_BLOCK_check_reply (GSF_block_ctx,
+                                        prq->type,
+                                        pr->bg,
+                                        key,
+                                        NULL, 0,
+                                        prq->data,
+                                        prq->size);
+  switch (prq->eval)
+  {
+  case GNUNET_BLOCK_REPLY_OK_MORE:
+    update_request_performance_data (prq, pr);
+    break;
+  case GNUNET_BLOCK_REPLY_OK_LAST:
+    /* short cut: stop processing early, no BF-update, etc. */
+    update_request_performance_data (prq, pr);
+    GNUNET_LOAD_update (GSF_rt_entry_lifetime,
+                        GNUNET_TIME_absolute_get_duration (
+                          pr->public_data.start_time)
+                        .rel_value_us);
+    if (GNUNET_YES !=
+        GSF_request_plan_reference_get_last_transmission_ (pr->public_data
+                                                           .pr_head,
+                                                           prq->sender,
+                                                           &last_transmission))
+      last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS;
+    /* pass on to other peers / local clients */
+    pr->rh (pr->rh_cls,
+            prq->eval,
+            pr,
+            prq->anonymity_level,
+            prq->expiration,
+            last_transmission,
+            prq->type,
+            prq->data,
+            prq->size);
+    return GNUNET_YES;
+  case GNUNET_BLOCK_REPLY_OK_DUPLICATE:
+#if INSANE_STATISTICS
+    GNUNET_STATISTICS_update (GSF_stats,
+                              "# duplicate replies discarded (bloomfilter)",
+                              1,
+                              GNUNET_NO);
+#endif
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Duplicate response, discarding.\n");
+    return GNUNET_YES;   /* duplicate */
+  case GNUNET_BLOCK_REPLY_IRRELEVANT:
+    GNUNET_STATISTICS_update (GSF_stats,
+                              "# irrelevant replies discarded",
+                              1,
+                              GNUNET_NO);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Irrelevant response, ignoring.\n");
+    return GNUNET_YES;
+  case GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED:
+    GNUNET_break (0); /* bad installation? */
+    return GNUNET_NO;
+  }
+  /* update bloomfilter */
+  GNUNET_CRYPTO_hash (prq->data,
+                      prq->size,
+                      &chash);
+  GSF_pending_request_update_ (pr,
+                               &chash,
+                               1);
+  if (NULL == prq->sender)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Found result for query `%s' in local datastore\n",
+                GNUNET_h2s (key));
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# results found locally"),
+                              1,
+                              GNUNET_NO);
+  }
+  else
+  {
+    GSF_dht_lookup_ (pr);
+  }
+  prq->priority += pr->public_data.original_priority;
+  pr->public_data.priority = 0;
+  pr->public_data.original_priority = 0;
+  pr->public_data.results_found++;
+  prq->request_found = GNUNET_YES;
+  /* finally, pass on to other peer / local client */
+  if (! GSF_request_plan_reference_get_last_transmission_ (pr->public_data
+                                                           .pr_head,
+                                                           prq->sender,
+                                                           &last_transmission))
+    last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS;
+  pr->rh (pr->rh_cls,
+          prq->eval,
+          pr,
+          prq->anonymity_level,
+          prq->expiration,
+          last_transmission,
+          prq->type,
+          prq->data,
+          prq->size);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Context for put_migration_continuation().
+ */
+struct PutMigrationContext
+{
+  /**
+   * Start time for the operation.
+   */
+  struct GNUNET_TIME_Absolute start;
+
+  /**
+   * Request origin.
+   */
+  struct GNUNET_PeerIdentity origin;
+
+  /**
+   * #GNUNET_YES if we had a matching request for this block,
+   * #GNUNET_NO if not.
+   */
+  int requested;
+};
+
+
+/**
+ * Continuation called to notify client about result of the
+ * operation.
+ *
+ * @param cls closure
+ * @param success #GNUNET_SYSERR on failure
+ * @param min_expiration minimum expiration time required for content to be stored
+ * @param msg NULL on success, otherwise an error message
+ */
+static void
+put_migration_continuation (void *cls,
+                            int success,
+                            struct GNUNET_TIME_Absolute min_expiration,
+                            const char *msg)
+{
+  struct PutMigrationContext *pmc = cls;
+  struct GSF_ConnectedPeer *cp;
+  struct GNUNET_TIME_Relative mig_pause;
+  struct GSF_PeerPerformanceData *ppd;
+
+  if (NULL != datastore_put_load)
+  {
+    if (GNUNET_SYSERR != success)
+    {
+      GNUNET_LOAD_update (datastore_put_load,
+                          GNUNET_TIME_absolute_get_duration (pmc->start)
+                          .rel_value_us);
+    }
+    else
+    {
+      /* on queue failure / timeout, increase the put load dramatically */
+      GNUNET_LOAD_update (datastore_put_load,
+                          GNUNET_TIME_UNIT_MINUTES.rel_value_us);
+    }
+  }
+  cp = GSF_peer_get_ (&pmc->origin);
+  if (GNUNET_OK == success)
+  {
+    if (NULL != cp)
+    {
+      ppd = GSF_get_peer_performance_data_ (cp);
+      ppd->migration_delay.rel_value_us /= 2;
+    }
+    GNUNET_free (pmc);
+    return;
+  }
+  if ((GNUNET_NO == success) && (GNUNET_NO == pmc->requested) && (NULL != cp))
+  {
+    ppd = GSF_get_peer_performance_data_ (cp);
+    if (min_expiration.abs_value_us > 0)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Asking to stop migration for %s because datastore is full\n",
+                  GNUNET_STRINGS_relative_time_to_string (
+                    GNUNET_TIME_absolute_get_remaining (min_expiration),
+                    GNUNET_YES));
+      GSF_block_peer_migration_ (cp, min_expiration);
+    }
+    else
+    {
+      ppd->migration_delay = GNUNET_TIME_relative_max (GNUNET_TIME_UNIT_SECONDS,
+                                                       ppd->migration_delay);
+      ppd->migration_delay =
+        GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_HOURS, ppd->migration_delay);
+      mig_pause.rel_value_us =
+        GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                  ppd->migration_delay.rel_value_us);
+      ppd->migration_delay =
+        GNUNET_TIME_relative_saturating_multiply (ppd->migration_delay, 2);
+      GNUNET_log (
+        GNUNET_ERROR_TYPE_DEBUG,
+        "Replicated content already exists locally, asking to stop migration for %s\n",
+        GNUNET_STRINGS_relative_time_to_string (mig_pause, GNUNET_YES));
+      GSF_block_peer_migration_ (cp,
+                                 GNUNET_TIME_relative_to_absolute (mig_pause));
+    }
+  }
+  GNUNET_free (pmc);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Datastore `PUT' failures"),
+                            1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Test if the DATABASE (PUT) load on this peer is too high
+ * to even consider processing the query at
+ * all.
+ *
+ * @param priority the priority of the item
+ * @return #GNUNET_YES if the load is too high to do anything (load high)
+ *         #GNUNET_NO to process normally (load normal or low)
+ */
+static int
+test_put_load_too_high (uint32_t priority)
+{
+  double ld;
+
+  if (NULL == datastore_put_load)
+    return GNUNET_NO;
+  if (GNUNET_LOAD_get_average (datastore_put_load) < 50)
+    return GNUNET_NO; /* very fast */
+  ld = GNUNET_LOAD_get_load (datastore_put_load);
+  if (ld < 2.0 * (1 + priority))
+    return GNUNET_NO;
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop (
+                              "# storage requests dropped due to high load"),
+                            1,
+                            GNUNET_NO);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Iterator called on each result obtained for a DHT
+ * operation that expects a reply
+ *
+ * @param cls closure
+ * @param exp when will this value expire
+ * @param key key of the result
+ * @param trunc_peer truncated peer, NULL for none
+ * @param get_path peers on reply path (or NULL if not recorded)
+ * @param get_path_length number of entries in @a get_path
+ * @param put_path peers on the PUT path (or NULL if not recorded)
+ * @param put_path_length number of entries in @a get_path
+ * @param type type of the result
+ * @param size number of bytes in @a data
+ * @param data pointer to the result data
+ */
+static void
+handle_dht_reply (void *cls,
+                  struct GNUNET_TIME_Absolute exp,
+                  const struct GNUNET_HashCode *key,
+                  const struct GNUNET_PeerIdentity *trunc_peer,
+                  const struct GNUNET_DHT_PathElement *get_path,
+                  unsigned int get_path_length,
+                  const struct GNUNET_DHT_PathElement *put_path,
+                  unsigned int put_path_length,
+                  enum GNUNET_BLOCK_Type type,
+                  size_t size,
+                  const void *data)
+{
+  struct GSF_PendingRequest *pr = cls;
+  struct ProcessReplyClosure prq;
+  struct PutMigrationContext *pmc;
+
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Replies received from DHT"),
+                            1,
+                            GNUNET_NO);
+  memset (&prq, 0, sizeof(prq));
+  prq.data = data;
+  prq.expiration = exp;
+  /* do not allow migrated content to live longer than 1 year */
+  prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (
+                                               GNUNET_TIME_UNIT_YEARS),
+                                             prq.expiration);
+  prq.size = size;
+  prq.type = type;
+  process_reply (&prq,
+                 key,
+                 pr);
+  if ((GNUNET_YES == active_to_migration) &&
+      (GNUNET_NO == test_put_load_too_high (prq.priority)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Replicating result for query `%s' with priority %u\n",
+                GNUNET_h2s (key),
+                prq.priority);
+    pmc = GNUNET_new (struct PutMigrationContext);
+    pmc->start = GNUNET_TIME_absolute_get ();
+    pmc->requested = GNUNET_YES;
+    if (NULL == GNUNET_DATASTORE_put (GSF_dsh,
+                                      0,
+                                      key,
+                                      size,
+                                      data,
+                                      type,
+                                      prq.priority,
+                                      1 /* anonymity */,
+                                      0 /* replication */,
+                                      exp,
+                                      1 + prq.priority,
+                                      MAX_DATASTORE_QUEUE,
+                                      &put_migration_continuation,
+                                      pmc))
+    {
+      put_migration_continuation (pmc,
+                                  GNUNET_SYSERR,
+                                  GNUNET_TIME_UNIT_ZERO_ABS,
+                                  NULL);
+    }
+  }
+}
+
+
+/**
+ * Consider looking up the data in the DHT (anonymity-level permitting).
+ *
+ * @param pr the pending request to process
+ */
+void
+GSF_dht_lookup_ (struct GSF_PendingRequest *pr)
+{
+  const void *xquery;
+  size_t xquery_size;
+  struct GNUNET_PeerIdentity pi;
+  char buf[sizeof(struct GNUNET_HashCode) * 2] GNUNET_ALIGN;
+
+  if (0 != pr->public_data.anonymity_level)
+    return;
+  if (NULL != pr->gh)
+  {
+    GNUNET_DHT_get_stop (pr->gh);
+    pr->gh = NULL;
+  }
+  xquery = NULL;
+  xquery_size = 0;
+  if (0 != (pr->public_data.options & GSF_PRO_FORWARD_ONLY))
+  {
+    GNUNET_assert (0 != pr->sender_pid);
+    GNUNET_PEER_resolve (pr->sender_pid, &pi);
+    GNUNET_memcpy (&buf[xquery_size], &pi, sizeof(struct GNUNET_PeerIdentity));
+    xquery_size += sizeof(struct GNUNET_PeerIdentity);
+  }
+  pr->gh = GNUNET_DHT_get_start (GSF_dht,
+                                 pr->public_data.type,
+                                 &pr->public_data.query,
+                                 DHT_GET_REPLICATION,
+                                 GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                 xquery,
+                                 xquery_size,
+                                 &handle_dht_reply,
+                                 pr);
+  if ((NULL != pr->gh) && (0 != pr->replies_seen_count))
+    GNUNET_DHT_get_filter_known_results (pr->gh,
+                                         pr->replies_seen_count,
+                                         pr->replies_seen);
+}
+
+
+/**
+ * Function called with a reply from the cadet.
+ *
+ * @param cls the pending request struct
+ * @param type type of the block, ANY on error
+ * @param expiration expiration time for the block
+ * @param data_size number of bytes in @a data, 0 on error
+ * @param data reply block data, NULL on error
+ */
+static void
+cadet_reply_proc (void *cls,
+                  enum GNUNET_BLOCK_Type type,
+                  struct GNUNET_TIME_Absolute expiration,
+                  size_t data_size,
+                  const void *data)
+{
+  struct GSF_PendingRequest *pr = cls;
+  struct ProcessReplyClosure prq;
+  struct GNUNET_HashCode query;
+
+  pr->cadet_request = NULL;
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_check_block (GSF_block_ctx,
+                                type,
+                                data,
+                                data_size))
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  if (GNUNET_BLOCK_TYPE_ANY == type)
+  {
+    GNUNET_break (NULL == data);
+    GNUNET_break (0 == data_size);
+    pr->cadet_retry_count++;
+    if (pr->cadet_retry_count >= CADET_RETRY_MAX)
+      return;   /* give up on cadet */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error retrieiving block via cadet\n");
+    /* retry -- without delay, as this is non-anonymous
+       and cadet/cadet connect will take some time anyway */
+    pr->cadet_request = GSF_cadet_query (pr->public_data.target,
+                                         &pr->public_data.query,
+                                         pr->public_data.type,
+                                         &cadet_reply_proc,
+                                         pr);
+    return;
+  }
+  if (GNUNET_YES !=
+      GNUNET_BLOCK_get_key (GSF_block_ctx,
+                            type,
+                            data,
+                            data_size,
+                            &query))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Failed to derive key for block of type %d\n",
+                (int) type);
+    GNUNET_break_op (0);
+    return;
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Replies received from CADET"),
+                            1,
+                            GNUNET_NO);
+  memset (&prq, 0, sizeof(prq));
+  prq.data = data;
+  prq.expiration = expiration;
+  /* do not allow migrated content to live longer than 1 year */
+  prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (
+                                               GNUNET_TIME_UNIT_YEARS),
+                                             prq.expiration);
+  prq.size = data_size;
+  prq.type = type;
+  process_reply (&prq,
+                 &query,
+                 pr);
+}
+
+
+/**
+ * Consider downloading via cadet (if possible)
+ *
+ * @param pr the pending request to process
+ */
+void
+GSF_cadet_lookup_ (struct GSF_PendingRequest *pr)
+{
+  if (0 != pr->public_data.anonymity_level)
+    return;
+  if (0 == pr->public_data.target)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Cannot do cadet-based download, target peer not known\n");
+    return;
+  }
+  if (NULL != pr->cadet_request)
+    return;
+  pr->cadet_request = GSF_cadet_query (pr->public_data.target,
+                                       &pr->public_data.query,
+                                       pr->public_data.type,
+                                       &cadet_reply_proc,
+                                       pr);
+}
+
+
+/**
+ * Task that issues a warning if the datastore lookup takes too long.
+ *
+ * @param cls the `struct GSF_PendingRequest`
+ */
+static void
+warn_delay_task (void *cls)
+{
+  struct GSF_PendingRequest *pr = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+              _ ("Datastore lookup already took %s!\n"),
+              GNUNET_STRINGS_relative_time_to_string (
+                GNUNET_TIME_absolute_get_duration (pr->qe_start),
+                GNUNET_YES));
+  pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                                &warn_delay_task,
+                                                pr);
+}
+
+
+/**
+ * Task that issues a warning if the datastore lookup takes too long.
+ *
+ * @param cls the `struct GSF_PendingRequest`
+ */
+static void
+odc_warn_delay_task (void *cls)
+{
+  struct GSF_PendingRequest *pr = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              _ ("On-demand lookup already took %s!\n"),
+              GNUNET_STRINGS_relative_time_to_string (
+                GNUNET_TIME_absolute_get_duration (pr->qe_start),
+                GNUNET_YES));
+  pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                                &odc_warn_delay_task,
+                                                pr);
+}
+
+
+/* Call our continuation (if we have any) */
+static void
+call_continuation (struct GSF_PendingRequest *pr)
+{
+  GSF_LocalLookupContinuation cont = pr->llc_cont;
+
+  GNUNET_assert (NULL == pr->qe);
+  if (NULL != pr->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (pr->warn_task);
+    pr->warn_task = NULL;
+  }
+  if (NULL == cont)
+    return; /* no continuation */
+  pr->llc_cont = NULL;
+  if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options))
+  {
+    if (GNUNET_BLOCK_REPLY_OK_LAST != pr->local_result)
+    {
+      /* Signal that we are done and that there won't be any
+         additional results to allow client to clean up state. */
+      pr->rh (pr->rh_cls,
+              GNUNET_BLOCK_REPLY_OK_LAST,
+              pr,
+              UINT32_MAX,
+              GNUNET_TIME_UNIT_ZERO_ABS,
+              GNUNET_TIME_UNIT_FOREVER_ABS,
+              GNUNET_BLOCK_TYPE_ANY,
+              NULL,
+              0);
+    }
+    /* Finally, call our continuation to signal that we are
+       done with local processing of this request; i.e. to
+       start reading again from the client. */
+    cont (pr->llc_cont_cls,
+          NULL,
+          GNUNET_BLOCK_REPLY_OK_LAST);
+    return;
+  }
+
+  cont (pr->llc_cont_cls,
+        pr,
+        pr->local_result);
+}
+
+
+/* Update stats and call continuation */
+static void
+no_more_local_results (struct GSF_PendingRequest *pr)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
+              "No further local responses available.\n");
+#if INSANE_STATISTICS
+  if ((GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) ||
+      (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type))
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# requested DBLOCK or IBLOCK not found"),
+                              1,
+                              GNUNET_NO);
+#endif
+  call_continuation (pr);
+}
+
+
+/* forward declaration */
+static void
+process_local_reply (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     size_t size,
+                     const void *data,
+                     enum GNUNET_BLOCK_Type type,
+                     uint32_t priority,
+                     uint32_t anonymity,
+                     uint32_t replication,
+                     struct GNUNET_TIME_Absolute expiration,
+                     uint64_t uid);
+
+
+/* Start a local query */
+static void
+start_local_query (struct GSF_PendingRequest *pr,
+                   uint64_t next_uid,
+                   bool random)
+{
+  pr->qe_start = GNUNET_TIME_absolute_get ();
+  pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                                &warn_delay_task,
+                                                pr);
+  pr->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
+                                     next_uid,
+                                     random,
+                                     &pr->public_data.query,
+                                     pr->public_data.type ==
+                                     GNUNET_BLOCK_TYPE_FS_DBLOCK
+                                     ? GNUNET_BLOCK_TYPE_ANY
+                                     : pr->public_data.type,
+                                     (0 != (GSF_PRO_PRIORITY_UNLIMITED
+                                            & pr->public_data.options))
+                                     ? UINT_MAX
+                                     : 1
+                                     /* queue priority */,
+                                     (0 != (GSF_PRO_PRIORITY_UNLIMITED
+                                            & pr->public_data.options))
+                                     ? UINT_MAX
+                                     : GSF_datastore_queue_size
+                                     /* max queue size */,
+                                     &process_local_reply,
+                                     pr);
+  if (NULL != pr->qe)
+    return;
+  GNUNET_log (
+    GNUNET_ERROR_TYPE_DEBUG,
+    "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n",
+    GNUNET_h2s (&pr->public_data.query),
+    pr->public_data.type,
+    (unsigned long long) next_uid);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop (
+                              "# Datastore lookups concluded (error queueing)"),
+                            1,
+                            GNUNET_NO);
+  call_continuation (pr);
+}
+
+
+/**
+ * We're processing (local) results for a search request
+ * from another peer.  Pass applicable results to the
+ * peer and if we are done either clean up (operation
+ * complete) or forward to other peers (more results possible).
+ *
+ * @param cls our closure (`struct GSF_PendingRequest *`)
+ * @param key key for the content
+ * @param size number of bytes in @a data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+process_local_reply (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     size_t size,
+                     const void *data,
+                     enum GNUNET_BLOCK_Type type,
+                     uint32_t priority,
+                     uint32_t anonymity,
+                     uint32_t replication,
+                     struct GNUNET_TIME_Absolute expiration,
+                     uint64_t uid)
+{
+  struct GSF_PendingRequest *pr = cls;
+  struct ProcessReplyClosure prq;
+  struct GNUNET_HashCode query;
+  unsigned int old_rf;
+
+  GNUNET_SCHEDULER_cancel (pr->warn_task);
+  pr->warn_task = NULL;
+  if (NULL == pr->qe)
+    goto called_from_on_demand;
+  pr->qe = NULL;
+  if (
+    (NULL == key) && pr->seen_null &&
+    ! pr->have_first_uid)  /* We have hit the end for the 2nd time with no results */
+  {
+    /* No results */
+#if INSANE_STATISTICS
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# Datastore lookups concluded (no results)"),
+                              1,
+                              GNUNET_NO);
+#endif
+    no_more_local_results (pr);
+    return;
+  }
+  if (((NULL == key) &&
+       pr->seen_null) || /* We have hit the end for the 2nd time OR */
+      (pr->seen_null && pr->have_first_uid &&
+       (uid >= pr->first_uid))) /* We have hit the end and past first UID */
+  {
+    /* Seen all results */
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# Datastore lookups concluded (seen all)"),
+                              1,
+                              GNUNET_NO);
+    no_more_local_results (pr);
+    return;
+  }
+  if (NULL == key)
+  {
+    GNUNET_assert (! pr->seen_null);
+    pr->seen_null = true;
+    start_local_query (pr, 0 /* next_uid */, false /* random */);
+    return;
+  }
+  if (! pr->have_first_uid)
+  {
+    pr->first_uid = uid;
+    pr->have_first_uid = true;
+  }
+  pr->result_count++;
+  if (pr->result_count > MAX_RESULTS)
+  {
+    GNUNET_STATISTICS_update (
+      GSF_stats,
+      gettext_noop ("# Datastore lookups aborted (more than MAX_RESULTS)"),
+      1,
+      GNUNET_NO);
+    no_more_local_results (pr);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received reply for `%s' of type %d with UID %llu from datastore.\n",
+              GNUNET_h2s (key),
+              type,
+              (unsigned long long) uid);
+  if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Found ONDEMAND block, performing on-demand encoding\n");
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# on-demand blocks matched requests"),
+                              1,
+                              GNUNET_NO);
+    pr->qe_start = GNUNET_TIME_absolute_get ();
+    pr->warn_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                                  &odc_warn_delay_task,
+                                                  pr);
+    if (GNUNET_OK == GNUNET_FS_handle_on_demand_block (key,
+                                                       size,
+                                                       data,
+                                                       type,
+                                                       priority,
+                                                       anonymity,
+                                                       replication,
+                                                       expiration,
+                                                       uid,
+                                                       &process_local_reply,
+                                                       pr))
+    {
+      GNUNET_STATISTICS_update (GSF_stats,
+                                gettext_noop (
+                                  "# on-demand lookups performed successfully"),
+                                1,
+                                GNUNET_NO);
+      return;     /* we're done */
+    }
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# on-demand lookups failed"),
+                              1,
+                              GNUNET_NO);
+    GNUNET_SCHEDULER_cancel (pr->warn_task);
+    start_local_query (pr, uid + 1 /* next_uid */, false /* random */);
+    return;
+  }
+called_from_on_demand:
+  old_rf = pr->public_data.results_found;
+  memset (&prq, 0, sizeof(prq));
+  prq.data = data;
+  prq.expiration = expiration;
+  prq.size = size;
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_get_key (GSF_block_ctx,
+                            type,
+                            data,
+                            size,
+                            &query))
+  {
+    GNUNET_break (0);
+    GNUNET_DATASTORE_remove (GSF_dsh,
+                             key,
+                             size,
+                             data,
+                             UINT_MAX,
+                             UINT_MAX,
+                             NULL,
+                             NULL);
+    start_local_query (pr, uid + 1 /* next_uid */, false /* random */);
+    return;
+  }
+  prq.type = type;
+  prq.priority = priority;
+  prq.request_found = GNUNET_NO;
+  prq.anonymity_level = anonymity;
+  if ((0 == old_rf) && (0 == pr->public_data.results_found))
+    GSF_update_datastore_delay_ (pr->public_data.start_time);
+  process_reply (&prq,
+                 key,
+                 pr);
+  pr->local_result = prq.eval;
+  if (GNUNET_BLOCK_REPLY_OK_LAST == prq.eval)
+  {
+    GNUNET_STATISTICS_update (
+      GSF_stats,
+      gettext_noop ("# Datastore lookups concluded (found last result)"),
+      1,
+      GNUNET_NO);
+    call_continuation (pr);
+    return;
+  }
+  if ((0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) &&
+      ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) ||
+       (pr->public_data.results_found > 5 + 2 * pr->public_data.priority)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Load too high, done with request\n");
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop (
+                                "# Datastore lookups concluded (load too high)"),
+                              1,
+                              GNUNET_NO);
+    call_continuation (pr);
+    return;
+  }
+  start_local_query (pr, uid + 1 /* next_uid */, false /* random */);
+}
+
+
+/**
+ * Is the given target a legitimate peer for forwarding the given request?
+ *
+ * @param pr request
+ * @param target
+ * @return #GNUNET_YES if this request could be forwarded to the given peer
+ */
+int
+GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr,
+                                  const struct GNUNET_PeerIdentity *target)
+{
+  struct GNUNET_PeerIdentity pi;
+
+  if (0 == pr->origin_pid)
+    return GNUNET_YES;
+  GNUNET_PEER_resolve (pr->origin_pid, &pi);
+  return (0 == memcmp (&pi, target, sizeof(struct GNUNET_PeerIdentity)))
+         ? GNUNET_NO
+         : GNUNET_YES;
+}
+
+
+/**
+ * Look up the request in the local datastore.
+ *
+ * @param pr the pending request to process
+ * @param cont function to call at the end
+ * @param cont_cls closure for @a cont
+ */
+void
+GSF_local_lookup_ (struct GSF_PendingRequest *pr,
+                   GSF_LocalLookupContinuation cont,
+                   void *cont_cls)
+{
+  GNUNET_assert (NULL == pr->gh);
+  GNUNET_assert (NULL == pr->cadet_request);
+  GNUNET_assert (NULL == pr->llc_cont);
+  pr->llc_cont = cont;
+  pr->llc_cont_cls = cont_cls;
+#if INSANE_STATISTICS
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Datastore lookups initiated"),
+                            1,
+                            GNUNET_NO);
+#endif
+  start_local_query (pr, 0 /* next_uid */, true /* random */);
+}
+
+
+/**
+ * Handle P2P "CONTENT" message.  Checks that the message is
+ * well-formed and then checks if there are any pending requests for
+ * this content and possibly passes it on (to local clients or other
+ * peers).  Does NOT perform migration (content caching at this peer).
+ *
+ * @param cls the other peer involved
+ * @param put the actual message
+ */
+void
+handle_p2p_put (void *cls,
+                const struct PutMessage *put)
+{
+  struct GSF_ConnectedPeer *cp = cls;
+  uint16_t msize;
+  size_t dsize;
+  enum GNUNET_BLOCK_Type type;
+  struct GNUNET_TIME_Absolute expiration;
+  struct GNUNET_HashCode query;
+  struct ProcessReplyClosure prq;
+  struct GNUNET_TIME_Relative block_time;
+  double putl;
+  struct PutMigrationContext *pmc;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received P2P PUT from %s\n",
+              GNUNET_i2s (GSF_get_peer_performance_data_ (cp)->peer));
+  GSF_cover_content_count++;
+  msize = ntohs (put->header.size);
+  dsize = msize - sizeof(struct PutMessage);
+  type = ntohl (put->type);
+  expiration = GNUNET_TIME_absolute_ntoh (put->expiration);
+  /* do not allow migrated content to live longer than 1 year */
+  expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (
+                                           GNUNET_TIME_UNIT_YEARS),
+                                         expiration);
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_check_block (GSF_block_ctx,
+                                type,
+                                &put[1],
+                                dsize))
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  if (GNUNET_OK !=
+      GNUNET_BLOCK_get_key (GSF_block_ctx,
+                            type,
+                            &put[1],
+                            dsize,
+                            &query))
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# GAP PUT messages received"),
+                            1,
+                            GNUNET_NO);
+  /* now, lookup 'query' */
+  prq.data = (const void *) &put[1];
+  prq.sender = cp;
+  prq.size = dsize;
+  prq.type = type;
+  prq.expiration = expiration;
+  prq.priority = 0;
+  prq.anonymity_level = UINT32_MAX;
+  prq.request_found = GNUNET_NO;
+  GNUNET_CONTAINER_multihashmap_get_multiple (pr_map,
+                                              &query,
+                                              &process_reply,
+                                              &prq);
+  if (NULL != cp)
+  {
+    GSF_connected_peer_change_preference_ (cp,
+                                           CONTENT_BANDWIDTH_VALUE
+                                           + 1000 * prq.priority);
+    GSF_get_peer_performance_data_ (cp)->respect += prq.priority;
+  }
+  if ((GNUNET_YES == active_to_migration) && (NULL != cp) &&
+      (GNUNET_NO == test_put_load_too_high (prq.priority)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Replicating result for query `%s' with priority %u\n",
+                GNUNET_h2s (&query),
+                prq.priority);
+    pmc = GNUNET_new (struct PutMigrationContext);
+    pmc->start = GNUNET_TIME_absolute_get ();
+    pmc->requested = prq.request_found;
+    GNUNET_assert (0 != GSF_get_peer_performance_data_ (cp)->pid);
+    GNUNET_PEER_resolve (GSF_get_peer_performance_data_ (cp)->pid,
+                         &pmc->origin);
+    if (NULL == GNUNET_DATASTORE_put (GSF_dsh,
+                                      0,
+                                      &query,
+                                      dsize,
+                                      &put[1],
+                                      type,
+                                      prq.priority,
+                                      1 /* anonymity */,
+                                      0 /* replication */,
+                                      expiration,
+                                      1 + prq.priority,
+                                      MAX_DATASTORE_QUEUE,
+                                      &put_migration_continuation,
+                                      pmc))
+    {
+      put_migration_continuation (pmc,
+                                  GNUNET_SYSERR,
+                                  GNUNET_TIME_UNIT_ZERO_ABS,
+                                  NULL);
+    }
+  }
+  else if (NULL != cp)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Choosing not to keep content `%s' (%d/%d)\n",
+                GNUNET_h2s (&query),
+                active_to_migration,
+                test_put_load_too_high (prq.priority));
+  }
+  putl = GNUNET_LOAD_get_load (datastore_put_load);
+  if ((NULL != cp) && (GNUNET_NO == prq.request_found) &&
+      ((GNUNET_YES != active_to_migration) ||
+       (putl > 2.5 * (1 + prq.priority))))
+  {
+    if (GNUNET_YES != active_to_migration)
+      putl = 1.0 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 5);
+    block_time = GNUNET_TIME_relative_multiply (
+      GNUNET_TIME_UNIT_MILLISECONDS,
+      5000 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       (unsigned int) (60000 * putl * putl)));
+    GNUNET_log (
+      GNUNET_ERROR_TYPE_DEBUG,
+      "Asking to stop migration for %s because of load %f and events %d/%d\n",
+      GNUNET_STRINGS_relative_time_to_string (block_time, GNUNET_YES),
+      putl,
+      active_to_migration,
+      (GNUNET_NO == prq.request_found));
+    GSF_block_peer_migration_ (cp,
+                               GNUNET_TIME_relative_to_absolute (block_time));
+  }
+}
+
+
+/**
+ * Check if the given request is still active.
+ *
+ * @param pr pending request
+ * @return #GNUNET_YES if the request is still active
+ */
+int
+GSF_pending_request_test_active_ (struct GSF_PendingRequest *pr)
+{
+  return (NULL != pr->rh) ? GNUNET_YES : GNUNET_NO;
+}
+
+
+/**
+ * Setup the subsystem.
+ */
+void
+GSF_pending_request_init_ ()
+{
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (GSF_cfg,
+                                             "fs",
+                                             "MAX_PENDING_REQUESTS",
+                                             &max_pending_requests))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_INFO,
+                               "fs",
+                               "MAX_PENDING_REQUESTS");
+  }
+  active_to_migration =
+    GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_CACHING");
+  datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE);
+  pr_map = GNUNET_CONTAINER_multihashmap_create (32 * 1024, GNUNET_YES);
+  requests_by_expiration_heap =
+    GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
+}
+
+
+/**
+ * Shutdown the subsystem.
+ */
+void
+GSF_pending_request_done_ ()
+{
+  GNUNET_CONTAINER_multihashmap_iterate (pr_map, &clean_request, NULL);
+  GNUNET_CONTAINER_multihashmap_destroy (pr_map);
+  pr_map = NULL;
+  GNUNET_CONTAINER_heap_destroy (requests_by_expiration_heap);
+  requests_by_expiration_heap = NULL;
+  GNUNET_LOAD_value_free (datastore_put_load);
+  datastore_put_load = NULL;
+}
+
+
+/* end of gnunet-service-fs_pr.c */
diff --git a/src/service/fs/gnunet-service-fs_pr.h b/src/service/fs/gnunet-service-fs_pr.h
new file mode 100644
index 000000000..339e409c5
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_pr.h
@@ -0,0 +1,422 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2010, 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_pr.h
+ * @brief API to handle pending requests
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_PR_H
+#define GNUNET_SERVICE_FS_PR_H
+
+#include "gnunet-service-fs.h"
+
+
+/**
+ * Options for pending requests (bits to be ORed).
+ */
+enum GSF_PendingRequestOptions
+{
+  /**
+   * No special options (P2P-default).
+   */
+  GSF_PRO_DEFAULTS = 0,
+
+  /**
+   * Request must only be processed locally.
+   */
+  GSF_PRO_LOCAL_ONLY = 1,
+
+  /**
+   * Request must only be forwarded (no routing)
+   */
+  GSF_PRO_FORWARD_ONLY = 2,
+
+  /**
+   * Request persists indefinitely (no expiration).
+   */
+  GSF_PRO_REQUEST_NEVER_EXPIRES = 4,
+
+  /**
+   * Request is allowed to refresh bloomfilter and change mingle value.
+   */
+  GSF_PRO_BLOOMFILTER_FULL_REFRESH = 8,
+
+  /**
+   * Request priority is allowed to be exceeded.
+   */
+  GSF_PRO_PRIORITY_UNLIMITED = 16,
+
+  /**
+   * Option mask for typical local requests.
+   */
+  GSF_PRO_LOCAL_REQUEST =
+    (GSF_PRO_BLOOMFILTER_FULL_REFRESH | GSF_PRO_PRIORITY_UNLIMITED
+     | GSF_PRO_REQUEST_NEVER_EXPIRES)
+};
+
+
+/**
+ * Public data (in the sense of not encapsulated within
+ * 'gnunet-service-fs_pr', not in the sense of network-wide
+ * known) associated with each pending request.
+ */
+struct GSF_PendingRequestData
+{
+  /**
+   * Primary query hash for this request.
+   */
+  struct GNUNET_HashCode query;
+
+  /**
+   * Identity of a peer hosting the content, otherwise NULl.
+   * Allocated after struct only if needed. Do not free!
+   */
+  const struct GNUNET_PeerIdentity *target;
+
+  /**
+   * Fields for the plan module to track a DLL with the request.
+   */
+  struct GSF_PendingRequestPlanBijection *pr_head;
+
+  /**
+   * Fields for the plan module to track a DLL with the request.
+   */
+  struct GSF_PendingRequestPlanBijection *pr_tail;
+
+  /**
+   * Current TTL for the request.
+   */
+  struct GNUNET_TIME_Absolute ttl;
+
+  /**
+   * When did we start with the request.
+   */
+  struct GNUNET_TIME_Absolute start_time;
+
+  /**
+   * Desired anonymity level.
+   */
+  uint32_t anonymity_level;
+
+  /**
+   * Priority that this request (still) has for us.
+   */
+  uint32_t priority;
+
+  /**
+   * Priority that this request (originally) had for us.
+   */
+  uint32_t original_priority;
+
+  /**
+   * Counter for how often this request has been transmitted (estimate,
+   * because we might have the same request pending for multiple clients,
+   * and of course because a transmission may have failed at a lower
+   * layer).
+   */
+  uint32_t num_transmissions;
+
+  /**
+   * How much respect did we (in total) offer for this request so far (estimate,
+   * because we might have the same request pending for multiple clients,
+   * and of course because a transmission may have failed at a lower
+   * layer).
+   */
+  uint32_t respect_offered;
+
+  /**
+   * Options for the request.
+   */
+  enum GSF_PendingRequestOptions options;
+
+  /**
+   * Type of the requested block.
+   */
+  enum GNUNET_BLOCK_Type type;
+
+  /**
+   * Number of results we have found for this request so far.
+   */
+  unsigned int results_found;
+
+  /**
+   * Has this request been started yet (local/p2p operations)?  Or are
+   * we still constructing it?
+   */
+  int has_started;
+};
+
+
+/**
+ * Handle a reply to a pending request.  Also called if a request
+ * expires (then with data == NULL).  The handler may be called
+ * many times (depending on the request type), but will not be
+ * called during or after a call to GSF_pending_request_cancel
+ * and will also not be called anymore after a call signalling
+ * expiration.
+ *
+ * @param cls user-specified closure
+ * @param eval evaluation of the result
+ * @param pr handle to the original pending request
+ * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown"
+ * @param expiration when does @a data expire?
+ * @param last_transmission the last time we've tried to get this block (FOREVER if unknown)
+ * @param type type of the block
+ * @param data response data, NULL on request expiration
+ * @param data_len number of bytes in @a data
+ */
+typedef void
+(*GSF_PendingRequestReplyHandler) (
+  void *cls,
+  enum GNUNET_BLOCK_ReplyEvaluationResult eval,
+  struct GSF_PendingRequest *pr,
+  uint32_t reply_anonymity_level,
+  struct GNUNET_TIME_Absolute expiration,
+  struct GNUNET_TIME_Absolute
+  last_transmission,
+  enum GNUNET_BLOCK_Type type,
+  const void *data,
+  size_t data_len);
+
+
+/**
+ * Create a new pending request.
+ *
+ * @param options request options
+ * @param type type of the block that is being requested
+ * @param query key for the lookup
+ * @param target preferred target for the request, NULL for none
+ * @param bf_data raw data for bloom filter for known replies, can be NULL
+ * @param bf_size number of bytes in bf_data
+ * @param anonymity_level desired anonymity level
+ * @param priority maximum outgoing cumulative request priority to use
+ * @param ttl current time-to-live for the request
+ * @param sender_pid peer ID to use for the sender when forwarding, 0 for none;
+ *                   reference counter is taken over by this function
+ * @param origin_pid peer ID of origin of query (do not loop back)
+ * @param replies_seen hash codes of known local replies
+ * @param replies_seen_count size of the 'replies_seen' array
+ * @param rh handle to call when we get a reply
+ * @param rh_cls closure for rh
+ * @return handle for the new pending request
+ */
+struct GSF_PendingRequest *
+GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
+                             enum GNUNET_BLOCK_Type type,
+                             const struct GNUNET_HashCode *query,
+                             const struct GNUNET_PeerIdentity *target,
+                             const char *bf_data,
+                             size_t bf_size,
+                             uint32_t anonymity_level,
+                             uint32_t priority,
+                             int32_t ttl,
+                             GNUNET_PEER_Id sender_pid,
+                             GNUNET_PEER_Id origin_pid,
+                             const struct GNUNET_HashCode *replies_seen,
+                             unsigned int replies_seen_count,
+                             GSF_PendingRequestReplyHandler rh,
+                             void *rh_cls);
+
+
+/**
+ * Update a given pending request with additional replies
+ * that have been seen.
+ *
+ * @param pr request to update
+ * @param replies_seen hash codes of replies that we've seen
+ * @param replies_seen_count size of the @a replies_seen array
+ */
+void
+GSF_pending_request_update_ (struct GSF_PendingRequest *pr,
+                             const struct GNUNET_HashCode *replies_seen,
+                             unsigned int replies_seen_count);
+
+
+/**
+ * Obtain the public data associated with a pending request
+ *
+ * @param pr pending request
+ * @return associated public data
+ */
+struct GSF_PendingRequestData *
+GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr);
+
+
+/**
+ * Check if the given request is still active.
+ *
+ * @param pr pending request
+ * @return #GNUNET_YES if the request is still active
+ */
+int
+GSF_pending_request_test_active_ (struct GSF_PendingRequest *pr);
+
+
+/**
+ * Test if two pending requests are compatible (would generate
+ * the same query modulo filters and should thus be processed
+ * jointly).
+ *
+ * @param pra a pending request
+ * @param prb another pending request
+ * @return #GNUNET_OK if the requests are compatible
+ */
+int
+GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra,
+                                    struct GSF_PendingRequest *prb);
+
+
+/**
+ * Generate the message corresponding to the given pending request for
+ * transmission to other peers.
+ *
+ * @param pr request to generate the message for
+ * @return envelope with the request message
+ */
+struct GNUNET_MQ_Envelope *
+GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr);
+
+
+/**
+ * Explicitly cancel a pending request.
+ *
+ * @param pr request to cancel
+ * @param full_cleanup fully purge the request
+ */
+void
+GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr,
+                             int full_cleanup);
+
+
+/**
+ * Signature of function called on each request.
+ * (Note: 'subtype' of GNUNET_CONTAINER_HashMapIterator).
+ *
+ * @param cls closure
+ * @param key query for the request
+ * @param pr handle to the pending request
+ * @return #GNUNET_YES to continue to iterate
+ */
+typedef int
+(*GSF_PendingRequestIterator) (void *cls,
+                               const struct GNUNET_HashCode *key,
+                               struct GSF_PendingRequest *pr);
+
+
+/**
+ * Iterate over all pending requests.
+ *
+ * @param it function to call for each request
+ * @param cls closure for it
+ */
+void
+GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it,
+                               void *cls);
+
+
+/**
+ * Handle P2P "CONTENT" message.  Checks that the message is
+ * well-formed and then checks if there are any pending requests for
+ * this content and possibly passes it on (to local clients or other
+ * peers).  Does NOT perform migration (content caching at this peer).
+ *
+ * @param cls the other peer involved (sender)
+ * @param put the actual message
+ */
+void
+handle_p2p_put (void *cls,
+                const struct PutMessage *put);
+
+
+/**
+ * Consider looking up the data in the DHT (anonymity-level permitting).
+ *
+ * @param pr the pending request to process
+ */
+void
+GSF_dht_lookup_ (struct GSF_PendingRequest *pr);
+
+
+/**
+ * Consider downloading via cadet (if possible)
+ *
+ * @param pr the pending request to process
+ */
+void
+GSF_cadet_lookup_ (struct GSF_PendingRequest *pr);
+
+
+/**
+ * Function to be called after we're done processing
+ * replies from the local lookup.
+ *
+ * @param cls closure
+ * @param pr the pending request we were processing
+ * @param result final datastore lookup result
+ */
+typedef void
+(*GSF_LocalLookupContinuation) (
+  void *cls,
+  struct GSF_PendingRequest *pr,
+  enum GNUNET_BLOCK_ReplyEvaluationResult result);
+
+
+/**
+ * Look up the request in the local datastore.
+ *
+ * @param pr the pending request to process
+ * @param cont function to call at the end
+ * @param cont_cls closure for @a cont
+ */
+void
+GSF_local_lookup_ (struct GSF_PendingRequest *pr,
+                   GSF_LocalLookupContinuation cont,
+                   void *cont_cls);
+
+
+/**
+ * Is the given target a legitimate peer for forwarding the given request?
+ *
+ * @param pr request
+ * @param target
+ * @return #GNUNET_YES if this request could be forwarded to the given peer
+ */
+int
+GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr,
+                                  const struct GNUNET_PeerIdentity *target);
+
+
+/**
+ * Setup the subsystem.
+ */
+void
+GSF_pending_request_init_ (void);
+
+
+/**
+ * Shutdown the subsystem.
+ */
+void
+GSF_pending_request_done_ (void);
+
+
+#endif
+/* end of gnunet-service-fs_pr.h */
diff --git a/src/service/fs/gnunet-service-fs_push.c b/src/service/fs/gnunet-service-fs_push.c
new file mode 100644
index 000000000..92dbba8e6
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_push.c
@@ -0,0 +1,672 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011, 2016 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_push.c
+ * @brief API to push content from our datastore to other peers
+ *            ('anonymous'-content P2P migration)
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_cp.h"
+#include "gnunet-service-fs_indexing.h"
+#include "gnunet-service-fs_push.h"
+
+
+/**
+ * Maximum number of blocks we keep in memory for migration.
+ */
+#define MAX_MIGRATION_QUEUE 8
+
+/**
+ * Blocks are at most migrated to this number of peers
+ * plus one, each time they are fetched from the database.
+ */
+#define MIGRATION_LIST_SIZE 2
+
+/**
+ * How long must content remain valid for us to consider it for migration?
+ * If content will expire too soon, there is clearly no point in pushing
+ * it to other peers.  This value gives the threshold for migration.  Note
+ * that if this value is increased, the migration testcase may need to be
+ * adjusted as well (especially the CONTENT_LIFETIME in fs_test_lib.c).
+ */
+#define MIN_MIGRATION_CONTENT_LIFETIME GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_MINUTES, 30)
+
+
+/**
+ * Block that is ready for migration to other peers.  Actual data is at the end of the block.
+ */
+struct MigrationReadyBlock
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct MigrationReadyBlock *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct MigrationReadyBlock *prev;
+
+  /**
+   * Query for the block.
+   */
+  struct GNUNET_HashCode query;
+
+  /**
+   * When does this block expire?
+   */
+  struct GNUNET_TIME_Absolute expiration;
+
+  /**
+   * Peers we already forwarded this
+   * block to.  Zero for empty entries.
+   */
+  GNUNET_PEER_Id target_list[MIGRATION_LIST_SIZE];
+
+  /**
+   * Size of the block.
+   */
+  size_t size;
+
+  /**
+   *  Number of targets already used.
+   */
+  unsigned int used_targets;
+
+  /**
+   * Type of the block.
+   */
+  enum GNUNET_BLOCK_Type type;
+};
+
+
+/**
+ * Information about a peer waiting for migratable data.
+ */
+struct MigrationReadyPeer
+{
+  /**
+   * This is a doubly-linked list.
+   */
+  struct MigrationReadyPeer *next;
+
+  /**
+   * This is a doubly-linked list.
+   */
+  struct MigrationReadyPeer *prev;
+
+  /**
+   * Handle to peer.
+   */
+  struct GSF_ConnectedPeer *peer;
+
+  /**
+   * Envelope of the currently pushed message.
+   */
+  struct GNUNET_MQ_Envelope *env;
+};
+
+
+/**
+ * Head of linked list of blocks that can be migrated.
+ */
+static struct MigrationReadyBlock *mig_head;
+
+/**
+ * Tail of linked list of blocks that can be migrated.
+ */
+static struct MigrationReadyBlock *mig_tail;
+
+/**
+ * Head of linked list of peers.
+ */
+static struct MigrationReadyPeer *peer_head;
+
+/**
+ * Tail of linked list of peers.
+ */
+static struct MigrationReadyPeer *peer_tail;
+
+/**
+ * Request to datastore for migration (or NULL).
+ */
+static struct GNUNET_DATASTORE_QueueEntry *mig_qe;
+
+/**
+ * ID of task that collects blocks for migration.
+ */
+static struct GNUNET_SCHEDULER_Task *mig_task;
+
+/**
+ * What is the maximum frequency at which we are allowed to
+ * poll the datastore for migration content?
+ */
+static struct GNUNET_TIME_Relative min_migration_delay;
+
+/**
+ * Size of the doubly-linked list of migration blocks.
+ */
+static unsigned int mig_size;
+
+/**
+ * Is this module enabled?
+ */
+static int enabled;
+
+/**
+ * Did we find anything in the datastore?
+ */
+static int value_found;
+
+
+/**
+ * Delete the given migration block.
+ *
+ * @param mb block to delete
+ */
+static void
+delete_migration_block (struct MigrationReadyBlock *mb)
+{
+  GNUNET_CONTAINER_DLL_remove (mig_head,
+                               mig_tail,
+                               mb);
+  GNUNET_PEER_decrement_rcs (mb->target_list,
+                             MIGRATION_LIST_SIZE);
+  mig_size--;
+  GNUNET_free (mb);
+}
+
+
+/**
+ * Find content for migration to this peer.
+ *
+ * @param cls A `struct MigrationReadyPeer *` to find content for
+ */
+static void
+find_content (void *cls);
+
+
+/**
+ * Send the given block to the given peer.
+ *
+ * @param mrp target peer
+ * @param block the block
+ * @return #GNUNET_YES if the block was deleted (!)
+ */
+static int
+transmit_content (struct MigrationReadyPeer *mrp,
+                  struct MigrationReadyBlock *block)
+{
+  struct PutMessage *msg;
+  unsigned int i;
+  struct GSF_PeerPerformanceData *ppd;
+  int ret;
+
+  ppd = GSF_get_peer_performance_data_ (mrp->peer);
+  GNUNET_assert (NULL == mrp->env);
+  mrp->env = GNUNET_MQ_msg_extra (msg,
+                                  block->size,
+                                  GNUNET_MESSAGE_TYPE_FS_PUT);
+  msg->type = htonl (block->type);
+  msg->expiration = GNUNET_TIME_absolute_hton (block->expiration);
+  GNUNET_memcpy (&msg[1],
+                 &block[1],
+                 block->size);
+  for (i = 0; i < MIGRATION_LIST_SIZE; i++)
+  {
+    if (block->target_list[i] == 0)
+    {
+      block->target_list[i] = ppd->pid;
+      GNUNET_PEER_change_rc (block->target_list[i],
+                             1);
+      break;
+    }
+  }
+  if (MIGRATION_LIST_SIZE == i)
+  {
+    delete_migration_block (block);
+    ret = GNUNET_YES;
+  }
+  else
+  {
+    ret = GNUNET_NO;
+  }
+  GNUNET_MQ_notify_sent (mrp->env,
+                         &find_content,
+                         mrp);
+  GSF_peer_transmit_ (mrp->peer,
+                      GNUNET_NO,
+                      0 /* priority */,
+                      mrp->env);
+  return ret;
+}
+
+
+/**
+ * Count the number of peers this block has
+ * already been forwarded to.
+ *
+ * @param block the block
+ * @return number of times block was forwarded
+ */
+static unsigned int
+count_targets (struct MigrationReadyBlock *block)
+{
+  unsigned int i;
+
+  for (i = 0; i < MIGRATION_LIST_SIZE; i++)
+    if (block->target_list[i] == 0)
+      return i;
+  return i;
+}
+
+
+/**
+ * Check if sending this block to this peer would
+ * be a good idea.
+ *
+ * @param mrp target peer
+ * @param block the block
+ * @return score (>= 0: feasible, negative: infeasible)
+ */
+static long
+score_content (struct MigrationReadyPeer *mrp,
+               struct MigrationReadyBlock *block)
+{
+  unsigned int i;
+  struct GSF_PeerPerformanceData *ppd;
+  struct GNUNET_PeerIdentity id;
+  struct GNUNET_HashCode hc;
+  uint32_t dist;
+
+  ppd = GSF_get_peer_performance_data_ (mrp->peer);
+  for (i = 0; i < MIGRATION_LIST_SIZE; i++)
+    if (block->target_list[i] == ppd->pid)
+      return -1;
+  GNUNET_assert (0 != ppd->pid);
+  GNUNET_PEER_resolve (ppd->pid,
+                       &id);
+  GNUNET_CRYPTO_hash (&id,
+                      sizeof(struct GNUNET_PeerIdentity),
+                      &hc);
+  dist = GNUNET_CRYPTO_hash_distance_u32 (&block->query,
+                                          &hc);
+  /* closer distance, higher score: */
+  return UINT32_MAX - dist;
+}
+
+
+/**
+ * If the migration task is not currently running, consider
+ * (re)scheduling it with the appropriate delay.
+ */
+static void
+consider_gathering (void);
+
+
+static void
+find_content (void *cls)
+{
+  struct MigrationReadyPeer *mrp = cls;
+  struct MigrationReadyBlock *pos;
+  long score;
+  long best_score;
+  struct MigrationReadyBlock *best;
+
+  mrp->env = NULL;
+  best = NULL;
+  best_score = -1;
+  pos = mig_head;
+  while (NULL != pos)
+  {
+    score = score_content (mrp, pos);
+    if (score > best_score)
+    {
+      best_score = score;
+      best = pos;
+    }
+    pos = pos->next;
+  }
+  if (NULL == best)
+  {
+    if (mig_size < MAX_MIGRATION_QUEUE)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "No content found for pushing, waiting for queue to fill\n");
+      return;                   /* will fill up eventually... */
+    }
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "No suitable content found, purging content from full queue\n");
+    /* failed to find migration target AND
+     * queue is full, purge most-forwarded
+     * block from queue to make room for more */
+    pos = mig_head;
+    while (NULL != pos)
+    {
+      score = count_targets (pos);
+      if (score >= best_score)
+      {
+        best_score = score;
+        best = pos;
+      }
+      pos = pos->next;
+    }
+    GNUNET_assert (NULL != best);
+    delete_migration_block (best);
+    consider_gathering ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Preparing to push best content to peer\n");
+  transmit_content (mrp,
+                    best);
+}
+
+
+/**
+ * Task that is run periodically to obtain blocks for content
+ * migration
+ *
+ * @param cls unused
+ */
+static void
+gather_migration_blocks (void *cls);
+
+
+/**
+ * If the migration task is not currently running, consider
+ * (re)scheduling it with the appropriate delay.
+ */
+static void
+consider_gathering ()
+{
+  struct GNUNET_TIME_Relative delay;
+
+  if (NULL == GSF_dsh)
+    return;
+  if (NULL != mig_qe)
+    return;
+  if (NULL != mig_task)
+    return;
+  if (mig_size >= MAX_MIGRATION_QUEUE)
+    return;
+  delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                         mig_size);
+  delay = GNUNET_TIME_relative_divide (delay,
+                                       MAX_MIGRATION_QUEUE);
+  delay = GNUNET_TIME_relative_max (delay,
+                                    min_migration_delay);
+  if (GNUNET_NO == value_found)
+  {
+    /* wait at least 5s if the datastore is empty */
+    delay = GNUNET_TIME_relative_max (delay,
+                                      GNUNET_TIME_relative_multiply (
+                                        GNUNET_TIME_UNIT_SECONDS,
+                                        5));
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Scheduling gathering task (queue size: %u)\n",
+              mig_size);
+  mig_task = GNUNET_SCHEDULER_add_delayed (delay,
+                                           &gather_migration_blocks,
+                                           NULL);
+}
+
+
+/**
+ * Process content offered for migration.
+ *
+ * @param cls closure
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+process_migration_content (void *cls,
+                           const struct GNUNET_HashCode *key,
+                           size_t size,
+                           const void *data,
+                           enum GNUNET_BLOCK_Type type,
+                           uint32_t priority,
+                           uint32_t anonymity,
+                           uint32_t replication,
+                           struct GNUNET_TIME_Absolute expiration,
+                           uint64_t uid)
+{
+  struct MigrationReadyBlock *mb;
+  struct MigrationReadyPeer *pos;
+
+  mig_qe = NULL;
+  if (NULL == key)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "No content found for migration...\n");
+    consider_gathering ();
+    return;
+  }
+  value_found = GNUNET_YES;
+  if (GNUNET_TIME_absolute_get_remaining (expiration).rel_value_us <
+      MIN_MIGRATION_CONTENT_LIFETIME.rel_value_us)
+  {
+    /* content will expire soon, don't bother */
+    consider_gathering ();
+    return;
+  }
+  if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND)
+  {
+    if (GNUNET_OK !=
+        GNUNET_FS_handle_on_demand_block (key,
+                                          size,
+                                          data,
+                                          type,
+                                          priority,
+                                          anonymity,
+                                          replication,
+                                          expiration,
+                                          uid,
+                                          &process_migration_content,
+                                          NULL))
+      consider_gathering ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Retrieved block `%s' of type %u for migration (queue size: %u/%u)\n",
+              GNUNET_h2s (key),
+              type, mig_size + 1,
+              MAX_MIGRATION_QUEUE);
+  mb = GNUNET_malloc (sizeof(struct MigrationReadyBlock) + size);
+  mb->query = *key;
+  mb->expiration = expiration;
+  mb->size = size;
+  mb->type = type;
+  GNUNET_memcpy (&mb[1], data, size);
+  GNUNET_CONTAINER_DLL_insert_after (mig_head,
+                                     mig_tail,
+                                     mig_tail,
+                                     mb);
+  mig_size++;
+  for (pos = peer_head; NULL != pos; pos = pos->next)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Preparing to push best content to peer %s\n",
+                GNUNET_i2s (GSF_connected_peer_get_identity2_ (pos->peer)));
+    if ((NULL == pos->env) &&
+        (GNUNET_YES == transmit_content (pos,
+                                         mb)))
+    {
+      break;                  /* 'mb' was freed! */
+    }
+  }
+  consider_gathering ();
+}
+
+
+/**
+ * Task that is run periodically to obtain blocks for content
+ * migration
+ *
+ * @param cls unused
+ */
+static void
+gather_migration_blocks (void *cls)
+{
+  mig_task = NULL;
+  if (mig_size >= MAX_MIGRATION_QUEUE)
+    return;
+  if (NULL == GSF_dsh)
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Asking datastore for content for replication (queue size: %u)\n",
+              mig_size);
+  value_found = GNUNET_NO;
+  mig_qe = GNUNET_DATASTORE_get_for_replication (GSF_dsh,
+                                                 0,
+                                                 UINT_MAX,
+                                                 &process_migration_content,
+                                                 NULL);
+  if (NULL == mig_qe)
+    consider_gathering ();
+}
+
+
+/**
+ * A peer connected to us.  Start pushing content
+ * to this peer.
+ *
+ * @param peer handle for the peer that connected
+ */
+void
+GSF_push_start_ (struct GSF_ConnectedPeer *peer)
+{
+  struct MigrationReadyPeer *mrp;
+
+  if (GNUNET_YES != enabled)
+    return;
+  for (mrp = peer_head; NULL != mrp; mrp = mrp->next)
+    if (mrp->peer == peer)
+      break;
+  if (NULL != mrp)
+  {
+    /* same peer added twice, must not happen */
+    GNUNET_break (0);
+    return;
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Adding peer %s to list for pushing\n",
+              GNUNET_i2s (GSF_connected_peer_get_identity2_ (peer)));
+
+  mrp = GNUNET_new (struct MigrationReadyPeer);
+  mrp->peer = peer;
+  find_content (mrp);
+  GNUNET_CONTAINER_DLL_insert (peer_head,
+                               peer_tail,
+                               mrp);
+}
+
+
+/**
+ * A peer disconnected from us.  Stop pushing content
+ * to this peer.
+ *
+ * @param peer handle for the peer that disconnected
+ */
+void
+GSF_push_stop_ (struct GSF_ConnectedPeer *peer)
+{
+  struct MigrationReadyPeer *pos;
+
+  for (pos = peer_head; NULL != pos; pos = pos->next)
+    if (pos->peer == peer)
+      break;
+  if (NULL == pos)
+    return;
+  if (NULL != pos->env)
+    GNUNET_MQ_send_cancel (pos->env);
+  GNUNET_CONTAINER_DLL_remove (peer_head,
+                               peer_tail,
+                               pos);
+  GNUNET_free (pos);
+}
+
+
+/**
+ * Setup the module.
+ */
+void
+GSF_push_init_ ()
+{
+  enabled =
+    GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg,
+                                          "FS",
+                                          "CONTENT_PUSHING");
+  if (GNUNET_YES != enabled)
+    return;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (GSF_cfg,
+                                           "fs",
+                                           "MIN_MIGRATION_DELAY",
+                                           &min_migration_delay))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "fs",
+                               "MIN_MIGRATION_DELAY",
+                               _ ("time required, content pushing disabled"));
+    return;
+  }
+  consider_gathering ();
+}
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GSF_push_done_ ()
+{
+  if (NULL != mig_task)
+  {
+    GNUNET_SCHEDULER_cancel (mig_task);
+    mig_task = NULL;
+  }
+  if (NULL != mig_qe)
+  {
+    GNUNET_DATASTORE_cancel (mig_qe);
+    mig_qe = NULL;
+  }
+  while (NULL != mig_head)
+    delete_migration_block (mig_head);
+  GNUNET_assert (0 == mig_size);
+}
+
+
+/* end of gnunet-service-fs_push.c */
diff --git a/src/service/fs/gnunet-service-fs_push.h b/src/service/fs/gnunet-service-fs_push.h
new file mode 100644
index 000000000..2cd621bbb
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_push.h
@@ -0,0 +1,66 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_push.h
+ * @brief support for pushing out content
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_PUSH_H
+#define GNUNET_SERVICE_FS_PUSH_H
+
+#include "gnunet-service-fs.h"
+
+
+/**
+ * Setup the module.
+ */
+void
+GSF_push_init_ (void);
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GSF_push_done_ (void);
+
+
+/**
+ * A peer connected to us or we are now again allowed to push content.
+ * Start pushing content to this peer.
+ *
+ * @param peer handle for the peer that connected
+ */
+void
+GSF_push_start_ (struct GSF_ConnectedPeer *peer);
+
+
+/**
+ * A peer disconnected from us or asked us to stop pushing content for
+ * a while.  Stop pushing content to this peer.
+ *
+ * @param peer handle for the peer that disconnected
+ */
+void
+GSF_push_stop_ (struct GSF_ConnectedPeer *peer);
+
+
+#endif
diff --git a/src/service/fs/gnunet-service-fs_put.c b/src/service/fs/gnunet-service-fs_put.c
new file mode 100644
index 000000000..ca2c85724
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_put.c
@@ -0,0 +1,290 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_put.c
+ * @brief API to PUT zero-anonymity index data from our datastore into the DHT
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet-service-fs.h"
+#include "gnunet-service-fs_put.h"
+
+
+/**
+ * How often do we at most PUT content into the DHT?
+ */
+#define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply ( \
+    GNUNET_TIME_UNIT_SECONDS, 5)
+
+/**
+ * How many replicas do we try to create per PUT?
+ */
+#define DEFAULT_PUT_REPLICATION 5
+
+
+/**
+ * Context for each zero-anonymity iterator.
+ */
+struct PutOperator
+{
+  /**
+   * Request to datastore for DHT PUTs (or NULL).
+   */
+  struct GNUNET_DATASTORE_QueueEntry *dht_qe;
+
+  /**
+   * Type we request from the datastore.
+   */
+  enum GNUNET_BLOCK_Type dht_put_type;
+
+  /**
+   * Handle to PUT operation.
+   */
+  struct GNUNET_DHT_PutHandle *dht_put;
+
+  /**
+   * ID of task that collects blocks for DHT PUTs.
+   */
+  struct GNUNET_SCHEDULER_Task *dht_task;
+
+  /**
+   * How many entries with zero anonymity of our type do we currently
+   * estimate to have in the database?
+   */
+  uint64_t zero_anonymity_count_estimate;
+
+  /**
+   * Count of results received from the database.
+   */
+  uint64_t result_count;
+
+  /**
+   * Next UID to request when iterating the database.
+   */
+  uint64_t next_uid;
+};
+
+
+/**
+ * ANY-terminated list of our operators (one per type
+ * of block that we're putting into the DHT).
+ */
+static struct PutOperator operators[] = {
+  { NULL, GNUNET_BLOCK_TYPE_FS_UBLOCK, 0, 0, 0 },
+  { NULL, GNUNET_BLOCK_TYPE_ANY, 0, 0, 0 }
+};
+
+
+/**
+ * Task that is run periodically to obtain blocks for DHT PUTs.
+ *
+ * @param cls type of blocks to gather
+ */
+static void
+gather_dht_put_blocks (void *cls);
+
+
+/**
+ * Calculate when to run the next PUT operation and schedule it.
+ *
+ * @param po put operator to schedule
+ */
+static void
+schedule_next_put (struct PutOperator *po)
+{
+  struct GNUNET_TIME_Relative delay;
+
+  if (po->zero_anonymity_count_estimate > 0)
+  {
+    delay =
+      GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
+                                   po->zero_anonymity_count_estimate);
+    delay = GNUNET_TIME_relative_min (delay, MAX_DHT_PUT_FREQ);
+  }
+  else
+  {
+    /* if we have NO zero-anonymity content yet, wait 5 minutes for some to
+     * (hopefully) appear */
+    delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5);
+  }
+  po->dht_task =
+    GNUNET_SCHEDULER_add_delayed (delay, &gather_dht_put_blocks, po);
+}
+
+
+/**
+ * Continuation called after DHT PUT operation has finished.
+ *
+ * @param cls type of blocks to gather
+ */
+static void
+delay_dht_put_blocks (void *cls)
+{
+  struct PutOperator *po = cls;
+
+  po->dht_put = NULL;
+  schedule_next_put (po);
+}
+
+
+/**
+ * Task that is run periodically to obtain blocks for DHT PUTs.
+ *
+ * @param cls type of blocks to gather
+ */
+static void
+delay_dht_put_task (void *cls)
+{
+  struct PutOperator *po = cls;
+
+  po->dht_task = NULL;
+  schedule_next_put (po);
+}
+
+
+/**
+ * Store content in DHT.
+ *
+ * @param cls closure
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param replication replication-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ */
+static void
+process_dht_put_content (void *cls,
+                         const struct GNUNET_HashCode *key,
+                         size_t size,
+                         const void *data,
+                         enum GNUNET_BLOCK_Type type,
+                         uint32_t priority,
+                         uint32_t anonymity,
+                         uint32_t replication,
+                         struct GNUNET_TIME_Absolute expiration,
+                         uint64_t uid)
+{
+  struct PutOperator *po = cls;
+
+  po->dht_qe = NULL;
+  if (key == NULL)
+  {
+    po->zero_anonymity_count_estimate = po->result_count;
+    po->result_count = 0;
+    po->next_uid = 0;
+    po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
+    return;
+  }
+  po->result_count++;
+  po->next_uid = uid + 1;
+  po->zero_anonymity_count_estimate =
+    GNUNET_MAX (po->result_count, po->zero_anonymity_count_estimate);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Retrieved block `%s' of type %u for DHT PUT\n", GNUNET_h2s (key),
+              type);
+  po->dht_put = GNUNET_DHT_put (GSF_dht,
+                                key,
+                                DEFAULT_PUT_REPLICATION,
+                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                type,
+                                size,
+                                data,
+                                expiration,
+                                &delay_dht_put_blocks,
+                                po);
+}
+
+
+static void
+gather_dht_put_blocks (void *cls)
+{
+  struct PutOperator *po = cls;
+
+  po->dht_task = NULL;
+  po->dht_qe =
+    GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh,
+                                         po->next_uid,
+                                         0,
+                                         UINT_MAX,
+                                         po->dht_put_type,
+                                         &process_dht_put_content,
+                                         po);
+  if (NULL == po->dht_qe)
+    po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
+}
+
+
+/**
+ * Setup the module.
+ */
+void
+GSF_put_init_ ()
+{
+  unsigned int i;
+
+  i = 0;
+  while (operators[i].dht_put_type != GNUNET_BLOCK_TYPE_ANY)
+  {
+    operators[i].dht_task =
+      GNUNET_SCHEDULER_add_now (&gather_dht_put_blocks, &operators[i]);
+    i++;
+  }
+}
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GSF_put_done_ ()
+{
+  struct PutOperator *po;
+  unsigned int i;
+
+  i = 0;
+  while ((po = &operators[i])->dht_put_type != GNUNET_BLOCK_TYPE_ANY)
+  {
+    if (NULL != po->dht_task)
+    {
+      GNUNET_SCHEDULER_cancel (po->dht_task);
+      po->dht_task = NULL;
+    }
+    if (NULL != po->dht_put)
+    {
+      GNUNET_DHT_put_cancel (po->dht_put);
+      po->dht_put = NULL;
+    }
+    if (NULL != po->dht_qe)
+    {
+      GNUNET_DATASTORE_cancel (po->dht_qe);
+      po->dht_qe = NULL;
+    }
+    i++;
+  }
+}
+
+
+/* end of gnunet-service-fs_put.c */
diff --git a/src/service/fs/gnunet-service-fs_put.h b/src/service/fs/gnunet-service-fs_put.h
new file mode 100644
index 000000000..b6c9ba86f
--- /dev/null
+++ b/src/service/fs/gnunet-service-fs_put.h
@@ -0,0 +1,46 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2011 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/gnunet-service-fs_put.h
+ * @brief support for putting content into the DHT
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_FS_PUT_H
+#define GNUNET_SERVICE_FS_PUT_H
+
+#include "gnunet-service-fs.h"
+
+
+/**
+ * Setup the module.
+ */
+void
+GSF_put_init_ (void);
+
+
+/**
+ * Shutdown the module.
+ */
+void
+GSF_put_done_ (void);
+
+
+#endif
diff --git a/src/service/fs/meson.build b/src/service/fs/meson.build
new file mode 100644
index 000000000..9e3e05cf7
--- /dev/null
+++ b/src/service/fs/meson.build
@@ -0,0 +1,82 @@
+libgnunetfs_src = ['fs_api.c',
+                   'fs_directory.c',
+                   'fs_dirmetascan.c',
+                   'fs_download.c',
+                   'fs_file_information.c',
+                   'fs_getopt.c',
+                   'fs_list_indexed.c',
+                   'fs_publish.c',
+                   'fs_publish_ksk.c',
+                   'fs_publish_ublock.c',
+                   'fs_misc.c',
+                   'fs_namespace.c',
+                   'fs_search.c',
+                   'fs_sharetree.c',
+                   'fs_tree.c',
+                   'fs_unindex.c',
+                   'fs_uri.c',
+                   'meta_data.c']
+
+gnunetservicefs_src = ['gnunet-service-fs.c',
+                       'gnunet-service-fs_cp.c',
+                       'gnunet-service-fs_indexing.c',
+                       'gnunet-service-fs_pe.c',
+                       'gnunet-service-fs_pr.c',
+                       'gnunet-service-fs_push.c',
+                       'gnunet-service-fs_put.c',
+                       'gnunet-service-fs_cadet_client.c',
+                       'gnunet-service-fs_cadet_server.c']
+
+configure_file(input : 'fs.conf.in',
+               output : 'fs.conf',
+               configuration : cdata,
+               install: true,
+               install_dir: pkgcfgdir)
+
+
+if get_option('monolith')
+  foreach p : libgnunetfs_src + gnunetservicefs_src
+    gnunet_src += 'fs/' + p
+  endforeach
+endif
+
+libgnunetfs = library('gnunetfs',
+        libgnunetfs_src,
+        soversion: '2',
+        version: '2.1.1',
+        dependencies: [libgnunetutil_dep,
+                       libgnunetdatastore_dep,
+                       libgnunetstatistics_dep,
+                       unistr_dep],
+        include_directories: [incdir, configuration_inc],
+        install: true,
+        install_dir: get_option('libdir'))
+libgnunetfs_dep = declare_dependency(link_with : libgnunetfs)
+pkg.generate(libgnunetfs, url: 'https://www.gnunet.org',
+             description : 'Provides API for GNUnet File-Sharing service')
+
+executable ('gnunet-service-fs',
+            gnunetservicefs_src,
+            dependencies: [libgnunetfs_dep,
+                           libgnunetutil_dep,
+                           libgnunetstatistics_dep,
+                           libgnunetcore_dep,
+                           libgnunetdht_dep,
+                           libgnunetidentity_dep,
+                           m_dep,
+                           libgnunetcadet_dep,
+                           libgnunetpeerstore_dep,
+                           libgnunetdatastore_dep,
+                           libgnunetblock_dep],
+            include_directories: [incdir, configuration_inc],
+            install: true,
+            install_dir: get_option('libdir') / 'gnunet' / 'libexec')
+executable ('gnunet-helper-fs-publish',
+            ['gnunet-helper-fs-publish.c'],
+            dependencies: [libgnunetfs_dep,
+                           libgnunetutil_dep,
+                           libgnunetblock_dep],
+            include_directories: [incdir, configuration_inc],
+            install: true,
+            install_dir: get_option('libdir') / 'gnunet' / 'libexec')
+
diff --git a/src/service/fs/meta_data.c b/src/service/fs/meta_data.c
new file mode 100644
index 000000000..b3db0e6c7
--- /dev/null
+++ b/src/service/fs/meta_data.c
@@ -0,0 +1,1238 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010, 2022 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/meta_data.c
+ * @brief Storing of meta data
+ * @author Christian Grothoff
+ * @author Martin Schanzenbach
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+
+/**
+ * Maximum size allowed for meta data written/read from disk.
+ * File-sharing limits to 64k, so this should be rather generous.
+ */
+#define MAX_META_DATA (1024 * 1024)
+
+
+#define LOG(kind, ...) GNUNET_log_from (kind, "fs-meta-data", \
+                                        __VA_ARGS__)
+
+
+/**
+ * Meta data item.
+ */
+struct MetaItem
+{
+  /**
+   * This is a doubly linked list.
+   */
+  struct MetaItem *next;
+
+  /**
+   * This is a doubly linked list.
+   */
+  struct MetaItem *prev;
+
+  /**
+   * Name of the extracting plugin.
+   */
+  char *plugin_name;
+
+  /**
+   * Mime-type of data.
+   */
+  char *mime_type;
+
+  /**
+   * The actual meta data.
+   */
+  char *data;
+
+  /**
+   * Number of bytes in 'data'.
+   */
+  size_t data_size;
+
+  /**
+   * Type of the meta data.
+   */
+  enum EXTRACTOR_MetaType type;
+
+  /**
+   * Format of the meta data.
+   */
+  enum EXTRACTOR_MetaFormat format;
+};
+
+/**
+ * Meta data to associate with a file, directory or namespace.
+ */
+struct GNUNET_FS_MetaData
+{
+  /**
+   * Head of linked list of the meta data items.
+   */
+  struct MetaItem *items_head;
+
+  /**
+   * Tail of linked list of the meta data items.
+   */
+  struct MetaItem *items_tail;
+
+  /**
+   * Complete serialized and compressed buffer of the items.
+   * NULL if we have not computed that buffer yet.
+   */
+  char *sbuf;
+
+  /**
+   * Number of bytes in 'sbuf'. 0 if the buffer is stale.
+   */
+  size_t sbuf_size;
+
+  /**
+   * Number of items in the linked list.
+   */
+  unsigned int item_count;
+};
+
+
+/**
+ * Create a fresh struct FS_MetaData token.
+ *
+ * @return empty meta-data container
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_create ()
+{
+  return GNUNET_new (struct GNUNET_FS_MetaData);
+}
+
+
+/**
+ * Free meta data item.
+ *
+ * @param mi item to free
+ */
+static void
+meta_item_free (struct MetaItem *mi)
+{
+  GNUNET_free (mi->plugin_name);
+  GNUNET_free (mi->mime_type);
+  GNUNET_free (mi->data);
+  GNUNET_free (mi);
+}
+
+
+/**
+ * The meta data has changed, invalidate its serialization
+ * buffer.
+ *
+ * @param md meta data that changed
+ */
+static void
+invalidate_sbuf (struct GNUNET_FS_MetaData *md)
+{
+  if (NULL == md->sbuf)
+    return;
+  GNUNET_free (md->sbuf);
+  md->sbuf = NULL;
+  md->sbuf_size = 0;
+}
+
+
+void
+GNUNET_FS_meta_data_destroy (struct GNUNET_FS_MetaData *md)
+{
+  struct MetaItem *pos;
+
+  if (NULL == md)
+    return;
+  while (NULL != (pos = md->items_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, pos);
+    meta_item_free (pos);
+  }
+  GNUNET_free (md->sbuf);
+  GNUNET_free (md);
+}
+
+
+void
+GNUNET_FS_meta_data_clear (struct GNUNET_FS_MetaData *md)
+{
+  struct MetaItem *mi;
+
+  if (NULL == md)
+    return;
+  while (NULL != (mi = md->items_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, mi);
+    meta_item_free (mi);
+  }
+  GNUNET_free (md->sbuf);
+  memset (md, 0, sizeof(struct GNUNET_FS_MetaData));
+}
+
+
+int
+GNUNET_FS_meta_data_test_equal (const struct GNUNET_FS_MetaData
+                                *md1,
+                                const struct GNUNET_FS_MetaData
+                                *md2)
+{
+  struct MetaItem *i;
+  struct MetaItem *j;
+  int found;
+
+  if (md1 == md2)
+    return GNUNET_YES;
+  if (md1->item_count != md2->item_count)
+    return GNUNET_NO;
+  for (i = md1->items_head; NULL != i; i = i->next)
+  {
+    found = GNUNET_NO;
+    for (j = md2->items_head; NULL != j; j = j->next)
+    {
+      if ((i->type == j->type) && (i->format == j->format) &&
+          (i->data_size == j->data_size) &&
+          (0 == memcmp (i->data, j->data, i->data_size)))
+      {
+        found = GNUNET_YES;
+        break;
+      }
+      if (j->data_size < i->data_size)
+        break;     /* elements are sorted by (decreasing) size... */
+    }
+    if (GNUNET_NO == found)
+      return GNUNET_NO;
+  }
+  return GNUNET_YES;
+}
+
+
+/**
+ * Extend metadata.  Note that the list of meta data items is
+ * sorted by size (largest first).
+ *
+ * @param md metadata to extend
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_size number of bytes in @a data
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists
+ *         data_mime_type and plugin_name are not considered for "exists" checks
+ */
+int
+GNUNET_FS_meta_data_insert (struct GNUNET_FS_MetaData *md,
+                            const char *plugin_name,
+                            enum EXTRACTOR_MetaType type,
+                            enum EXTRACTOR_MetaFormat format,
+                            const char *data_mime_type, const char *data,
+                            size_t data_size)
+{
+  struct MetaItem *pos;
+  struct MetaItem *mi;
+  char *p;
+
+  if ((EXTRACTOR_METAFORMAT_UTF8 == format) ||
+      (EXTRACTOR_METAFORMAT_C_STRING == format))
+    GNUNET_break ('\0' == data[data_size - 1]);
+
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+  {
+    if (pos->data_size < data_size)
+      break;   /* elements are sorted by size in the list */
+    if ((pos->type == type) && (pos->data_size == data_size) &&
+        (0 == memcmp (pos->data, data, data_size)))
+    {
+      if ((NULL == pos->mime_type) && (NULL != data_mime_type))
+      {
+        pos->mime_type = GNUNET_strdup (data_mime_type);
+        invalidate_sbuf (md);
+      }
+      if ((EXTRACTOR_METAFORMAT_C_STRING == pos->format) &&
+          (EXTRACTOR_METAFORMAT_UTF8 == format))
+      {
+        pos->format = EXTRACTOR_METAFORMAT_UTF8;
+        invalidate_sbuf (md);
+      }
+      return GNUNET_SYSERR;
+    }
+  }
+  md->item_count++;
+  mi = GNUNET_new (struct MetaItem);
+  mi->type = type;
+  mi->format = format;
+  mi->data_size = data_size;
+  if (NULL == pos)
+    GNUNET_CONTAINER_DLL_insert_tail (md->items_head,
+                                      md->items_tail,
+                                      mi);
+  else
+    GNUNET_CONTAINER_DLL_insert_after (md->items_head,
+                                       md->items_tail,
+                                       pos->prev,
+                                       mi);
+  mi->mime_type =
+    (NULL == data_mime_type) ? NULL : GNUNET_strdup (data_mime_type);
+  mi->plugin_name = (NULL == plugin_name) ? NULL : GNUNET_strdup (plugin_name);
+  mi->data = GNUNET_malloc (data_size);
+  GNUNET_memcpy (mi->data, data, data_size);
+  /* change all dir separators to POSIX style ('/') */
+  if ((EXTRACTOR_METATYPE_FILENAME == type) ||
+      (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type))
+  {
+    p = mi->data;
+    while (('\0' != *p) && (p < mi->data + data_size))
+    {
+      if ('\\' == *p)
+        *p = '/';
+      p++;
+    }
+  }
+  invalidate_sbuf (md);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Merge given meta data.
+ *
+ * @param cls the `struct GNUNET_FS_MetaData` to merge into
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_size number of bytes in @a data
+ * @return 0 (to continue)
+ */
+static int
+merge_helper (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type,
+              enum EXTRACTOR_MetaFormat format, const char *data_mime_type,
+              const char *data, size_t data_size)
+{
+  struct GNUNET_FS_MetaData *md = cls;
+
+  (void) GNUNET_FS_meta_data_insert (md, plugin_name, type, format,
+                                     data_mime_type, data, data_size);
+  return 0;
+}
+
+
+void
+GNUNET_FS_meta_data_merge (struct GNUNET_FS_MetaData *md,
+                           const struct GNUNET_FS_MetaData *in)
+{
+  GNUNET_FS_meta_data_iterate (in, &merge_helper, md);
+}
+
+
+int
+GNUNET_FS_meta_data_delete (struct GNUNET_FS_MetaData *md,
+                            enum EXTRACTOR_MetaType type,
+                            const char *data, size_t data_size)
+{
+  struct MetaItem *pos;
+
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+  {
+    if (pos->data_size < data_size)
+      break;   /* items are sorted by (decreasing) size */
+    if ((pos->type == type) &&
+        ((NULL == data) ||
+         ((pos->data_size == data_size) &&
+          (0 == memcmp (pos->data, data, data_size)))))
+    {
+      GNUNET_CONTAINER_DLL_remove (md->items_head, md->items_tail, pos);
+      meta_item_free (pos);
+      md->item_count--;
+      invalidate_sbuf (md);
+      return GNUNET_OK;
+    }
+  }
+  return GNUNET_SYSERR;
+}
+
+
+void
+GNUNET_FS_meta_data_add_publication_date (struct
+                                          GNUNET_FS_MetaData *md)
+{
+  const char *dat;
+  struct GNUNET_TIME_Absolute t;
+
+  t = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_meta_data_delete (md,
+                              EXTRACTOR_METATYPE_PUBLICATION_DATE,
+                              NULL, 0);
+  dat = GNUNET_STRINGS_absolute_time_to_string (t);
+  GNUNET_FS_meta_data_insert (md, "<gnunet>",
+                              EXTRACTOR_METATYPE_PUBLICATION_DATE,
+                              EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                              dat, strlen (dat) + 1);
+}
+
+
+/**
+ * Iterate over MD entries.
+ *
+ * @param md metadata to inspect
+ * @param iter function to call on each entry
+ * @param iter_cls closure for iterator
+ * @return number of entries
+ */
+int
+GNUNET_FS_meta_data_iterate (const struct GNUNET_FS_MetaData *md,
+                             EXTRACTOR_MetaDataProcessor iter,
+                             void *iter_cls)
+{
+  struct MetaItem *pos;
+
+  if (NULL == md)
+    return 0;
+  if (NULL == iter)
+    return md->item_count;
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+    if (0 !=
+        iter (iter_cls, pos->plugin_name, pos->type, pos->format,
+              pos->mime_type, pos->data, pos->data_size))
+      return md->item_count;
+  return md->item_count;
+}
+
+
+char *
+GNUNET_FS_meta_data_get_by_type (const struct
+                                 GNUNET_FS_MetaData *md,
+                                 enum EXTRACTOR_MetaType type)
+{
+  struct MetaItem *pos;
+
+  if (NULL == md)
+    return NULL;
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+    if ((type == pos->type) &&
+        ((pos->format == EXTRACTOR_METAFORMAT_UTF8) ||
+         (pos->format == EXTRACTOR_METAFORMAT_C_STRING)))
+      return GNUNET_strdup (pos->data);
+  return NULL;
+}
+
+
+char *
+GNUNET_FS_meta_data_get_first_by_types (const struct
+                                        GNUNET_FS_MetaData *md,
+                                        ...)
+{
+  char *ret;
+  va_list args;
+  int type;
+
+  if (NULL == md)
+    return NULL;
+  ret = NULL;
+  va_start (args, md);
+  while (1)
+  {
+    type = va_arg (args, int);
+    if (-1 == type)
+      break;
+    if (NULL != (ret = GNUNET_FS_meta_data_get_by_type (md, type)))
+      break;
+  }
+  va_end (args);
+  return ret;
+}
+
+
+/**
+ * Get a thumbnail from the meta-data (if present).
+ *
+ * @param md metadata to get the thumbnail from
+ * @param thumb will be set to the thumbnail data.  Must be
+ *        freed by the caller!
+ * @return number of bytes in thumbnail, 0 if not available
+ */
+size_t
+GNUNET_FS_meta_data_get_thumbnail (const struct GNUNET_FS_MetaData
+                                   *md, unsigned char **thumb)
+{
+  struct MetaItem *pos;
+  struct MetaItem *match;
+
+  if (NULL == md)
+    return 0;
+  match = NULL;
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+  {
+    if ((NULL != pos->mime_type) &&
+        (0 == strncasecmp ("image/", pos->mime_type, strlen ("image/"))) &&
+        (EXTRACTOR_METAFORMAT_BINARY == pos->format))
+    {
+      if (NULL == match)
+        match = pos;
+      else if ((match->type != EXTRACTOR_METATYPE_THUMBNAIL) &&
+               (pos->type == EXTRACTOR_METATYPE_THUMBNAIL))
+        match = pos;
+    }
+  }
+  if ((NULL == match) || (0 == match->data_size))
+    return 0;
+  *thumb = GNUNET_malloc (match->data_size);
+  GNUNET_memcpy (*thumb, match->data, match->data_size);
+  return match->data_size;
+}
+
+
+/**
+ * Duplicate a `struct GNUNET_FS_MetaData`.
+ *
+ * @param md what to duplicate
+ * @return duplicate meta-data container
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_duplicate (const struct GNUNET_FS_MetaData
+                               *md)
+{
+  struct GNUNET_FS_MetaData *ret;
+  struct MetaItem *pos;
+
+  if (NULL == md)
+    return NULL;
+  ret = GNUNET_FS_meta_data_create ();
+  for (pos = md->items_tail; NULL != pos; pos = pos->prev)
+    GNUNET_FS_meta_data_insert (ret, pos->plugin_name, pos->type,
+                                pos->format, pos->mime_type, pos->data,
+                                pos->data_size);
+  return ret;
+}
+
+
+/**
+ * Flag in 'version' that indicates compressed meta-data.
+ */
+#define HEADER_COMPRESSED 0x80000000
+
+
+/**
+ * Bits in 'version' that give the version number.
+ */
+#define HEADER_VERSION_MASK 0x7FFFFFFF
+
+
+/**
+ * Header for serialized meta data.
+ */
+struct MetaDataHeader
+{
+  /**
+   * The version of the MD serialization.  The highest bit is used to
+   * indicate compression.
+   *
+   * Version 0 is traditional (pre-0.9) meta data (unsupported)
+   * Version is 1 for a NULL pointer
+   * Version 2 is for 0.9.x (and possibly higher)
+   * Other version numbers are not yet defined.
+   */
+  uint32_t version;
+
+  /**
+   * How many MD entries are there?
+   */
+  uint32_t entries;
+
+  /**
+   * Size of the decompressed meta data.
+   */
+  uint32_t size;
+
+  /**
+   * This is followed by 'entries' values of type 'struct MetaDataEntry'
+   * and then by 'entry' plugin names, mime-types and data blocks
+   * as specified in those meta data entries.
+   */
+};
+
+
+/**
+ * Entry of serialized meta data.
+ */
+struct MetaDataEntry
+{
+  /**
+   * Meta data type.  Corresponds to an 'enum EXTRACTOR_MetaType'
+   */
+  uint32_t type;
+
+  /**
+   * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat'
+   */
+  uint32_t format;
+
+  /**
+   * Number of bytes of meta data.
+   */
+  uint32_t data_size;
+
+  /**
+   * Number of bytes in the plugin name including 0-terminator.  0 for NULL.
+   */
+  uint32_t plugin_name_len;
+
+  /**
+   * Number of bytes in the mime type including 0-terminator.  0 for NULL.
+   */
+  uint32_t mime_type_len;
+};
+
+
+/**
+ * Serialize meta-data to target.
+ *
+ * @param md metadata to serialize
+ * @param target where to write the serialized metadata;
+ *         *target can be NULL, in which case memory is allocated
+ * @param max maximum number of bytes available in target
+ * @param opt is it ok to just write SOME of the
+ *        meta-data to match the size constraint,
+ *        possibly discarding some data?
+ * @return number of bytes written on success,
+ *         #GNUNET_SYSERR on error (typically: not enough
+ *         space)
+ */
+ssize_t
+GNUNET_FS_meta_data_serialize (const struct GNUNET_FS_MetaData
+                               *md, char **target, size_t max,
+                               enum
+                               GNUNET_FS_MetaDataSerializationOptions
+                               opt)
+{
+  struct GNUNET_FS_MetaData *vmd;
+  struct MetaItem *pos;
+  struct MetaDataHeader ihdr;
+  struct MetaDataHeader *hdr;
+  struct MetaDataEntry *ent;
+  char *dst;
+  unsigned int i;
+  uint64_t msize;
+  size_t off;
+  char *mdata;
+  char *cdata;
+  size_t mlen;
+  size_t plen;
+  size_t size;
+  size_t left;
+  size_t clen;
+  size_t rlen;
+  int comp;
+
+  if (max < sizeof(struct MetaDataHeader))
+    return GNUNET_SYSERR;       /* far too small */
+  if (NULL == md)
+    return 0;
+
+  if (NULL != md->sbuf)
+  {
+    /* try to use serialization cache */
+    if (md->sbuf_size <= max)
+    {
+      if (NULL == *target)
+        *target = GNUNET_malloc (md->sbuf_size);
+      GNUNET_memcpy (*target, md->sbuf, md->sbuf_size);
+      return md->sbuf_size;
+    }
+    if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_PART))
+      return GNUNET_SYSERR;     /* can say that this will fail */
+    /* need to compute a partial serialization, sbuf useless ... */
+  }
+  dst = NULL;
+  msize = 0;
+  for (pos = md->items_tail; NULL != pos; pos = pos->prev)
+  {
+    msize += sizeof(struct MetaDataEntry);
+    msize += pos->data_size;
+    if (NULL != pos->plugin_name)
+      msize += strlen (pos->plugin_name) + 1;
+    if (NULL != pos->mime_type)
+      msize += strlen (pos->mime_type) + 1;
+  }
+  size = (size_t) msize;
+  if (size != msize)
+  {
+    GNUNET_break (0);           /* integer overflow */
+    return GNUNET_SYSERR;
+  }
+  if (size >= GNUNET_MAX_MALLOC_CHECKED)
+  {
+    /* too large to be processed */
+    return GNUNET_SYSERR;
+  }
+  ent = GNUNET_malloc (size);
+  mdata = (char *) &ent[md->item_count];
+  off = size - (md->item_count * sizeof(struct MetaDataEntry));
+  i = 0;
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+  {
+    ent[i].type = htonl ((uint32_t) pos->type);
+    ent[i].format = htonl ((uint32_t) pos->format);
+    ent[i].data_size = htonl ((uint32_t) pos->data_size);
+    if (NULL == pos->plugin_name)
+      plen = 0;
+    else
+      plen = strlen (pos->plugin_name) + 1;
+    ent[i].plugin_name_len = htonl ((uint32_t) plen);
+    if (NULL == pos->mime_type)
+      mlen = 0;
+    else
+      mlen = strlen (pos->mime_type) + 1;
+    ent[i].mime_type_len = htonl ((uint32_t) mlen);
+    off -= pos->data_size;
+    if ((EXTRACTOR_METAFORMAT_UTF8 == pos->format) ||
+        (EXTRACTOR_METAFORMAT_C_STRING == pos->format))
+      GNUNET_break ('\0' == pos->data[pos->data_size - 1]);
+    GNUNET_memcpy (&mdata[off], pos->data, pos->data_size);
+    off -= plen;
+    if (NULL != pos->plugin_name)
+      GNUNET_memcpy (&mdata[off], pos->plugin_name, plen);
+    off -= mlen;
+    if (NULL != pos->mime_type)
+      GNUNET_memcpy (&mdata[off], pos->mime_type, mlen);
+    i++;
+  }
+  GNUNET_assert (0 == off);
+
+  clen = 0;
+  cdata = NULL;
+  left = size;
+  i = 0;
+  for (pos = md->items_head; NULL != pos; pos = pos->next)
+  {
+    comp = GNUNET_NO;
+    if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_NO_COMPRESS))
+      comp = GNUNET_try_compression ((const char *) &ent[i],
+                                     left,
+                                     &cdata,
+                                     &clen);
+
+    if ((NULL == md->sbuf) && (0 == i))
+    {
+      /* fill 'sbuf'; this "modifies" md, but since this is only
+       * an internal cache we will cast away the 'const' instead
+       * of making the API look strange. */
+      vmd = (struct GNUNET_FS_MetaData *) md;
+      hdr = GNUNET_malloc (left + sizeof(struct MetaDataHeader));
+      hdr->size = htonl (left);
+      hdr->entries = htonl (md->item_count);
+      if (GNUNET_YES == comp)
+      {
+        GNUNET_assert (clen < left);
+        hdr->version = htonl (2 | HEADER_COMPRESSED);
+        GNUNET_memcpy (&hdr[1], cdata, clen);
+        vmd->sbuf_size = clen + sizeof(struct MetaDataHeader);
+      }
+      else
+      {
+        hdr->version = htonl (2);
+        GNUNET_memcpy (&hdr[1], &ent[0], left);
+        vmd->sbuf_size = left + sizeof(struct MetaDataHeader);
+      }
+      vmd->sbuf = (char *) hdr;
+    }
+
+    if (((left + sizeof(struct MetaDataHeader)) <= max) ||
+        ((GNUNET_YES == comp) && (clen <= max)))
+    {
+      /* success, this now fits! */
+      if (GNUNET_YES == comp)
+      {
+        if (NULL == dst)
+          dst = GNUNET_malloc (clen + sizeof(struct MetaDataHeader));
+        hdr = (struct MetaDataHeader *) dst;
+        hdr->version = htonl (2 | HEADER_COMPRESSED);
+        hdr->size = htonl (left);
+        hdr->entries = htonl (md->item_count - i);
+        GNUNET_memcpy (&dst[sizeof(struct MetaDataHeader)], cdata, clen);
+        GNUNET_free (cdata);
+        cdata = NULL;
+        GNUNET_free (ent);
+        rlen = clen + sizeof(struct MetaDataHeader);
+      }
+      else
+      {
+        if (NULL == dst)
+          dst = GNUNET_malloc (left + sizeof(struct MetaDataHeader));
+        hdr = (struct MetaDataHeader *) dst;
+        hdr->version = htonl (2);
+        hdr->entries = htonl (md->item_count - i);
+        hdr->size = htonl (left);
+        GNUNET_memcpy (&dst[sizeof(struct MetaDataHeader)], &ent[i], left);
+        GNUNET_free (ent);
+        rlen = left + sizeof(struct MetaDataHeader);
+      }
+      if (NULL != *target)
+      {
+        if (GNUNET_YES == comp)
+          GNUNET_memcpy (*target, dst, clen + sizeof(struct MetaDataHeader));
+        else
+          GNUNET_memcpy (*target, dst, left + sizeof(struct MetaDataHeader));
+        GNUNET_free (dst);
+      }
+      else
+      {
+        *target = dst;
+      }
+      return rlen;
+    }
+
+    if (0 == (opt & GNUNET_FS_META_DATA_SERIALIZE_PART))
+    {
+      /* does not fit! */
+      GNUNET_free (ent);
+      if (NULL != cdata)
+        GNUNET_free (cdata);
+      cdata = NULL;
+      return GNUNET_SYSERR;
+    }
+
+    /* next iteration: ignore the corresponding meta data at the
+     * end and try again without it */
+    left -= sizeof(struct MetaDataEntry);
+    left -= pos->data_size;
+    if (NULL != pos->plugin_name)
+      left -= strlen (pos->plugin_name) + 1;
+    if (NULL != pos->mime_type)
+      left -= strlen (pos->mime_type) + 1;
+
+    if (NULL != cdata)
+      GNUNET_free (cdata);
+    cdata = NULL;
+    i++;
+  }
+  GNUNET_free (ent);
+
+  /* nothing fit, only write header! */
+  ihdr.version = htonl (2);
+  ihdr.entries = htonl (0);
+  ihdr.size = htonl (0);
+  if (NULL == *target)
+    *target = (char *) GNUNET_new (struct MetaDataHeader);
+  GNUNET_memcpy (*target, &ihdr, sizeof(struct MetaDataHeader));
+  return sizeof(struct MetaDataHeader);
+}
+
+
+ssize_t
+GNUNET_FS_meta_data_get_serialized_size (const struct
+                                         GNUNET_FS_MetaData *md)
+{
+  ssize_t ret;
+  char *ptr;
+
+  if (NULL != md->sbuf)
+    return md->sbuf_size;
+  ptr = NULL;
+  ret =
+    GNUNET_FS_meta_data_serialize (md, &ptr, GNUNET_MAX_MALLOC_CHECKED,
+                                   GNUNET_FS_META_DATA_SERIALIZE_FULL);
+  if (-1 != ret)
+    GNUNET_free (ptr);
+  return ret;
+}
+
+
+/**
+ * Deserialize meta-data.  Initializes md.
+ *
+ * @param input buffer with the serialized metadata
+ * @param size number of bytes available in input
+ * @return MD on success, NULL on error (i.e.
+ *         bad format)
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_deserialize (const char *input, size_t size)
+{
+  struct GNUNET_FS_MetaData *md;
+  struct MetaDataHeader hdr;
+  struct MetaDataEntry ent;
+  uint32_t ic;
+  uint32_t i;
+  char *data;
+  const char *cdata;
+  uint32_t version;
+  uint32_t dataSize;
+  int compressed;
+  size_t left;
+  uint32_t mlen;
+  uint32_t plen;
+  uint32_t dlen;
+  const char *mdata;
+  const char *meta_data;
+  const char *plugin_name;
+  const char *mime_type;
+  enum EXTRACTOR_MetaFormat format;
+
+  if (size < sizeof(struct MetaDataHeader))
+    return NULL;
+  GNUNET_memcpy (&hdr, input, sizeof(struct MetaDataHeader));
+  version = ntohl (hdr.version) & HEADER_VERSION_MASK;
+  compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0;
+
+  if (1 == version)
+    return NULL;                /* null pointer */
+  if (2 != version)
+  {
+    GNUNET_break_op (0);        /* unsupported version */
+    return NULL;
+  }
+
+  ic = ntohl (hdr.entries);
+  dataSize = ntohl (hdr.size);
+  if (((sizeof(struct MetaDataEntry) * ic) > dataSize) ||
+      ((0 != ic) &&
+       (dataSize / ic < sizeof(struct MetaDataEntry))))
+  {
+    GNUNET_break_op (0);
+    return NULL;
+  }
+
+  if (compressed)
+  {
+    if (dataSize >= GNUNET_MAX_MALLOC_CHECKED)
+    {
+      /* make sure we don't blow our memory limit because of a mal-formed
+       * message... */
+      GNUNET_break_op (0);
+      return NULL;
+    }
+    data =
+      GNUNET_decompress ((const char *) &input[sizeof(struct MetaDataHeader)],
+                         size - sizeof(struct MetaDataHeader),
+                         dataSize);
+    if (NULL == data)
+    {
+      GNUNET_break_op (0);
+      return NULL;
+    }
+    cdata = data;
+  }
+  else
+  {
+    data = NULL;
+    cdata = (const char *) &input[sizeof(struct MetaDataHeader)];
+    if (dataSize != size - sizeof(struct MetaDataHeader))
+    {
+      GNUNET_break_op (0);
+      return NULL;
+    }
+  }
+
+  md = GNUNET_FS_meta_data_create ();
+  left = dataSize - ic * sizeof(struct MetaDataEntry);
+  mdata = &cdata[ic * sizeof(struct MetaDataEntry)];
+  for (i = 0; i < ic; i++)
+  {
+    GNUNET_memcpy (&ent, &cdata[i * sizeof(struct MetaDataEntry)],
+                   sizeof(struct MetaDataEntry));
+    format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format);
+    if ((EXTRACTOR_METAFORMAT_UTF8 != format) &&
+        (EXTRACTOR_METAFORMAT_C_STRING != format) &&
+        (EXTRACTOR_METAFORMAT_BINARY != format))
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    dlen = ntohl (ent.data_size);
+    plen = ntohl (ent.plugin_name_len);
+    mlen = ntohl (ent.mime_type_len);
+    if (dlen > left)
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    left -= dlen;
+    meta_data = &mdata[left];
+    if ((EXTRACTOR_METAFORMAT_UTF8 == format) ||
+        (EXTRACTOR_METAFORMAT_C_STRING == format))
+    {
+      if (0 == dlen)
+      {
+        GNUNET_break_op (0);
+        break;
+      }
+      if ('\0' != meta_data[dlen - 1])
+      {
+        GNUNET_break_op (0);
+        break;
+      }
+    }
+    if (plen > left)
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    left -= plen;
+    if ((plen > 0) && ('\0' != mdata[left + plen - 1]))
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    if (0 == plen)
+      plugin_name = NULL;
+    else
+      plugin_name = &mdata[left];
+
+    if (mlen > left)
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    left -= mlen;
+    if ((mlen > 0) && ('\0' != mdata[left + mlen - 1]))
+    {
+      GNUNET_break_op (0);
+      break;
+    }
+    if (0 == mlen)
+      mime_type = NULL;
+    else
+      mime_type = &mdata[left];
+    GNUNET_FS_meta_data_insert (md, plugin_name,
+                                (enum EXTRACTOR_MetaType)
+                                ntohl (ent.type), format, mime_type,
+                                meta_data, dlen);
+  }
+  GNUNET_free (data);
+  return md;
+}
+
+
+/**
+ * Read a metadata container.
+ *
+ * @param h handle to an open file
+ * @param what describes what is being read (for error message creation)
+ * @param result the buffer to store a pointer to the (allocated) metadata
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+enum GNUNET_GenericReturnValue
+GNUNET_FS_read_meta_data (struct GNUNET_BIO_ReadHandle *h,
+                          const char *what,
+                          struct GNUNET_FS_MetaData **result)
+{
+  uint32_t size;
+  char *buf;
+  char *emsg;
+  struct GNUNET_FS_MetaData *meta;
+
+  if (GNUNET_OK != GNUNET_BIO_read_int32 (h,
+                                          _ ("metadata length"),
+                                          (int32_t *) &size))
+    return GNUNET_SYSERR;
+  if (0 == size)
+  {
+    *result = NULL;
+    return GNUNET_OK;
+  }
+  if (MAX_META_DATA < size)
+  {
+    GNUNET_asprintf (&emsg,
+                     _ (
+                       "Serialized metadata `%s' larger than allowed (%u > %u)\n"),
+                     what,
+                     size,
+                     MAX_META_DATA);
+    GNUNET_BIO_read_set_error (h, emsg);
+    GNUNET_free (emsg);
+    return GNUNET_SYSERR;
+  }
+  buf = GNUNET_malloc (size);
+  if (GNUNET_OK != GNUNET_BIO_read (h, what, buf, size))
+  {
+    GNUNET_free (buf);
+    return GNUNET_SYSERR;
+  }
+  meta = GNUNET_FS_meta_data_deserialize (buf, size);
+  if (NULL == meta)
+  {
+    GNUNET_free (buf);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _ ("Failed to deserialize metadata `%s'"), what);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (buf);
+  *result = meta;
+  return GNUNET_OK;
+}
+
+
+/**
+ * Write a metadata container.
+ *
+ * @param h the IO handle to write to
+ * @param what what is being written (for error message creation)
+ * @param m metadata to write
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+enum GNUNET_GenericReturnValue
+GNUNET_FS_write_meta_data (struct GNUNET_BIO_WriteHandle *h,
+                           const char *what,
+                           const struct GNUNET_FS_MetaData *m)
+{
+  ssize_t size;
+  char *buf;
+
+  if (m == NULL)
+    return GNUNET_BIO_write_int32 (h, _ ("metadata length"), 0);
+  buf = NULL;
+  size = GNUNET_FS_meta_data_serialize (m,
+                                        &buf,
+                                        MAX_META_DATA,
+                                        GNUNET_FS_META_DATA_SERIALIZE_PART);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Serialized %lld bytes of metadata",
+              (long long) size);
+
+  if (-1 == size)
+  {
+    GNUNET_free (buf);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to serialize metadata `%s'",
+                what);
+    return GNUNET_SYSERR;
+  }
+  if ( (GNUNET_OK !=
+        GNUNET_BIO_write_int32 (h,
+                                "metadata length",
+                                (uint32_t) size)) ||
+       (GNUNET_OK !=
+        GNUNET_BIO_write (h,
+                          what,
+                          buf,
+                          size)) )
+  {
+    GNUNET_free (buf);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (buf);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function used internally to read a metadata container from within a read
+ * spec.
+ *
+ * @param cls ignored, always NULL
+ * @param h the IO handle to read from
+ * @param what what is being read (for error message creation)
+ * @param target where to store the data
+ * @param target_size ignored
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+read_spec_handler_meta_data (void *cls,
+                             struct GNUNET_BIO_ReadHandle *h,
+                             const char *what,
+                             void *target,
+                             size_t target_size)
+{
+  struct GNUNET_FS_MetaData **result = target;
+  return GNUNET_FS_read_meta_data (h, what, result);
+}
+
+
+/**
+ * Create the specification to read a metadata container.
+ *
+ * @param what describes what is being read (for error message creation)
+ * @param result the buffer to store a pointer to the (allocated) metadata
+ * @return the read spec
+ */
+struct GNUNET_BIO_ReadSpec
+GNUNET_FS_read_spec_meta_data (const char *what,
+                               struct GNUNET_FS_MetaData **result)
+{
+  struct GNUNET_BIO_ReadSpec rs = {
+    .rh = &read_spec_handler_meta_data,
+    .cls = NULL,
+    .target = result,
+    .size = 0,
+  };
+
+  return rs;
+}
+
+
+/**
+ * Function used internally to write a metadata container from within a write
+ * spec.
+ *
+ * @param cls ignored, always NULL
+ * @param h the IO handle to write to
+ * @param what what is being written (for error message creation)
+ * @param source the data to write
+ * @param source_size ignored
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise
+ */
+static int
+write_spec_handler_meta_data (void *cls,
+                              struct GNUNET_BIO_WriteHandle *h,
+                              const char *what,
+                              void *source,
+                              size_t source_size)
+{
+  const struct GNUNET_FS_MetaData *m = source;
+  return GNUNET_FS_write_meta_data (h, what, m);
+}
+
+
+struct GNUNET_BIO_WriteSpec
+GNUNET_FS_write_spec_meta_data (const char *what,
+                                const struct GNUNET_FS_MetaData *m)
+{
+  struct GNUNET_BIO_WriteSpec ws = {
+    .wh = &write_spec_handler_meta_data,
+    .cls = NULL,
+    .what = what,
+    .source = (void *) m,
+    .source_size = 0,
+  };
+
+  return ws;
+}
+
+
+/* end of meta_data.c */
diff --git a/src/service/fs/perf_gnunet_service_fs_p2p.c b/src/service/fs/perf_gnunet_service_fs_p2p.c
new file mode 100644
index 000000000..2c7830f5f
--- /dev/null
+++ b/src/service/fs/perf_gnunet_service_fs_p2p.c
@@ -0,0 +1,370 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/perf_gnunet_service_fs_p2p.c
+ * @brief profile P2P routing using simple publish + download operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fs_test_lib.h"
+#include "gnunet_testbed_service.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 10)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
+
+#define NUM_DAEMONS 2
+
+#define SEED 42
+
+static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS];
+
+static int ok;
+
+static struct GNUNET_TIME_Absolute start_time;
+
+static const char *progname;
+
+static struct GNUNET_TIME_Absolute start_time;
+
+
+/**
+ * Master context for 'stat_run'.
+ */
+struct StatMaster
+{
+  struct GNUNET_STATISTICS_Handle *stat;
+  struct GNUNET_TESTBED_Operation *op;
+  unsigned int daemon;
+  unsigned int value;
+};
+
+struct StatValues
+{
+  const char *subsystem;
+  const char *name;
+};
+
+/**
+ * Statistics we print out.
+ */
+static struct StatValues stats[] = {
+  { "fs", "# queries forwarded" },
+  { "fs", "# replies received and matched" },
+  { "fs", "# results found locally" },
+  { "fs", "# requests forwarded due to high load" },
+  { "fs", "# requests done for free (low load)" },
+  { "fs", "# requests dropped, priority insufficient" },
+  { "fs", "# requests done for a price (normal load)" },
+  { "fs", "# requests dropped by datastore (queue length limit)" },
+  { "fs", "# P2P searches received" },
+  { "fs", "# P2P searches discarded (queue length bound)" },
+  { "fs", "# replies received for local clients" },
+  { "fs", "# queries retransmitted to same target" },
+  { "core", "# bytes decrypted" },
+  { "core", "# bytes encrypted" },
+  { "core", "# discarded CORE_SEND requests" },
+  { "core", "# discarded CORE_SEND request bytes" },
+  { "core", "# discarded lower priority CORE_SEND requests" },
+  { "core", "# discarded lower priority CORE_SEND request bytes" },
+  { "transport", "# bytes received via TCP" },
+  { "transport", "# bytes transmitted via TCP" },
+  { "datacache", "# bytes stored" },
+  { NULL, NULL }
+};
+
+
+/**
+ * Callback function to process statistic values.
+ *
+ * @param cls closure
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
+ * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
+ */
+static int
+print_stat (void *cls, const char *subsystem, const char *name, uint64_t value,
+            int is_persistent)
+{
+  struct StatMaster *sm = cls;
+
+  fprintf (stderr,
+           "Peer %2u: %12s/%50s = %12llu\n",
+           sm->daemon,
+           subsystem,
+           name,
+           (unsigned long long) value);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function that gathers stats from all daemons.
+ */
+static void
+stat_run (void *cls,
+          struct GNUNET_TESTBED_Operation *op,
+          void *ca_result,
+          const char *emsg);
+
+
+/**
+ * Function called when GET operation on stats is done.
+ */
+static void
+get_done (void *cls, int success)
+{
+  struct StatMaster *sm = cls;
+
+  GNUNET_break (GNUNET_OK == success);
+  sm->value++;
+  stat_run (sm, sm->op, sm->stat, NULL);
+}
+
+
+/**
+ * Adapter function called to establish a connection to
+ * statistics service.
+ *
+ * @param cls closure
+ * @param cfg configuration of the peer to connect to; will be available until
+ *          GNUNET_TESTBED_operation_done() is called on the operation returned
+ *          from GNUNET_TESTBED_service_connect()
+ * @return service handle to return in 'op_result', NULL on error
+ */
+static void *
+statistics_connect_adapter (void *cls,
+                            const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  return GNUNET_STATISTICS_create ("<driver>",
+                                   cfg);
+}
+
+
+/**
+ * Adapter function called to destroy a connection to
+ * statistics service.
+ *
+ * @param cls closure
+ * @param op_result service handle returned from the connect adapter
+ */
+static void
+statistics_disconnect_adapter (void *cls,
+                               void *op_result)
+{
+  GNUNET_STATISTICS_destroy (op_result, GNUNET_NO);
+}
+
+
+/**
+ * Function that gathers stats from all daemons.
+ */
+static void
+stat_run (void *cls,
+          struct GNUNET_TESTBED_Operation *op,
+          void *ca_result,
+          const char *emsg)
+{
+  struct StatMaster *sm = cls;
+
+  if (NULL != emsg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to statistics service: %s\n",
+                emsg);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  sm->stat = ca_result;
+
+  if (stats[sm->value].name != NULL)
+  {
+    GNUNET_STATISTICS_get (sm->stat,
+#if 0
+                           NULL, NULL,
+#else
+                           stats[sm->value].subsystem, stats[sm->value].name,
+#endif
+                           &get_done, &print_stat,
+                           sm);
+    return;
+  }
+  GNUNET_TESTBED_operation_done (sm->op);
+  sm->value = 0;
+  sm->daemon++;
+  if (NUM_DAEMONS == sm->daemon)
+  {
+    GNUNET_free (sm);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  sm->op =
+    GNUNET_TESTBED_service_connect (NULL,
+                                    daemons[sm->daemon],
+                                    "statistics",
+                                    &stat_run, sm,
+                                    &statistics_connect_adapter,
+                                    &statistics_disconnect_adapter,
+                                    NULL);
+}
+
+
+static void
+do_report (void *cls)
+{
+  char *fn = cls;
+  struct GNUNET_TIME_Relative del;
+  char *fancy;
+  struct StatMaster *sm;
+
+  if (NULL != fn)
+  {
+    GNUNET_DISK_directory_remove (fn);
+    GNUNET_free (fn);
+  }
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time,
+                                                                    TIMEOUT)).
+      rel_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Timeout during download, shutting down with error\n");
+    ok = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+
+  del = GNUNET_TIME_absolute_get_duration (start_time);
+  if (del.rel_value_us == 0)
+    del.rel_value_us = 1;
+  fancy =
+    GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE)
+                                    * 1000000LL / del.rel_value_us);
+  fprintf (stdout,
+           "Download speed was %s/s\n",
+           fancy);
+  GNUNET_free (fancy);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished download, shutting down\n");
+  sm = GNUNET_new (struct StatMaster);
+  sm->op =
+    GNUNET_TESTBED_service_connect (NULL,
+                                    daemons[sm->daemon],
+                                    "statistics",
+                                    &stat_run, sm,
+                                    &statistics_connect_adapter,
+                                    &statistics_disconnect_adapter,
+                                    NULL);
+}
+
+
+static void
+do_download (void *cls,
+             const struct GNUNET_FS_Uri *uri,
+             const char *fn)
+{
+  int anonymity;
+
+  if (NULL == uri)
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Timeout during upload attempt, shutting down with error\n");
+    ok = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  start_time = GNUNET_TIME_absolute_get ();
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+  start_time = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_TEST_download (daemons[0],
+                           TIMEOUT,
+                           anonymity,
+                           SEED,
+                           uri,
+                           VERBOSE,
+                           &do_report,
+                           (NULL == fn) ? NULL : GNUNET_strdup (fn));
+}
+
+
+static void
+do_publish (void *cls,
+            struct GNUNET_TESTBED_RunHandle *h,
+            unsigned int num_peers,
+            struct GNUNET_TESTBED_Peer **peers,
+            unsigned int links_succeeded,
+            unsigned int links_failed)
+{
+  unsigned int i;
+  int do_index;
+  int anonymity;
+
+  GNUNET_assert (NUM_DAEMONS == num_peers);
+  for (i = 0; i < num_peers; i++)
+    daemons[i] = peers[i];
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  if (NULL != strstr (progname, "index"))
+    do_index = GNUNET_YES;
+  else
+    do_index = GNUNET_NO;
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+  GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity,
+                          do_index, FILESIZE, SEED, VERBOSE, &do_download,
+                          NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  progname = argv[0];
+  (void) GNUNET_TESTBED_test_run ("perf-gnunet-service-fs-p2p",
+                                  "perf_gnunet_service_fs_p2p.conf",
+                                  NUM_DAEMONS,
+                                  0, NULL, NULL,
+                                  &do_publish, NULL);
+  GNUNET_DISK_purge_cfg_dir 
+    ("perf_gnunet_service_fs_p2p.conf",
+     "GNUNET_TEST_HOME");
+  return ok;
+}
+
+
+/* end of perf_gnunet_service_fs_p2p.c */
diff --git a/src/service/fs/perf_gnunet_service_fs_p2p.conf b/src/service/fs/perf_gnunet_service_fs_p2p.conf
new file mode 100644
index 000000000..00f0f512e
--- /dev/null
+++ b/src/service/fs/perf_gnunet_service_fs_p2p.conf
@@ -0,0 +1,7 @@
+@INLINE@ fs_test_lib_data.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-fs-test-lib/
+
+[fs]
+GAUGER_HEAP = "2-peer 10 MB P2P download"
+# PREFIX = valgrind
diff --git a/src/service/fs/perf_gnunet_service_fs_p2p_respect.c b/src/service/fs/perf_gnunet_service_fs_p2p_respect.c
new file mode 100644
index 000000000..6b71b1f93
--- /dev/null
+++ b/src/service/fs/perf_gnunet_service_fs_p2p_respect.c
@@ -0,0 +1,480 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2011, 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/perf_gnunet_service_fs_p2p_respect.c
+ * @brief profile P2P routing respect mechanism. Creates
+ *        a clique of NUM_DAEMONS (at least 3) where two
+ *        peers share (seed) different files and download
+ *        them from each other while all the other peers
+ *        just "leach" those files.  Ideally, the seeders
+ *        "learn" that they contribute (to each other),
+ *        and give the other seeder higher priority;
+ *        naturally, this only happens nicely for larger
+ *        files; finally, once the seeders are done, the
+ *        leachers should see fast download rates as well.
+ * @author Christian Grothoff
+ *
+ * Sample output:
+ * - 10 MB, 3 peers, with delays:
+ * Download speed of type `seeder 1' was 757 KiB/s
+ * Download speed of type `seeder 2' was 613 KiB/s
+ * Download speed of type `leach` was 539 KiB/s
+ *
+ * - 10 MB, 3 peers, without delays:
+ * Download speed of type `seeder 1' was 1784 KiB/s
+ * Download speed of type `seeder 2' was 1604 KiB/s
+ * Download speed of type `leach` was 1384 KiB/s
+ */
+#include "platform.h"
+#include "fs_test_lib.h"
+#include "gnunet_testbed_service.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 1)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
+
+/**
+ * Number of daemons in clique, must be at least 3 (!).
+ */
+#define NUM_DAEMONS 3
+
+/**
+ * Seed for first file on offer.
+ */
+#define SEED1 42
+
+/**
+ * Seed for second file on offer.
+ */
+#define SEED2 43
+
+static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS];
+
+static int ok;
+
+static struct GNUNET_TIME_Absolute start_time;
+
+static const char *progname;
+
+static struct GNUNET_FS_Uri *uri1;
+
+static struct GNUNET_FS_Uri *uri2;
+
+static char *fn1;
+
+static char *fn2;
+
+/**
+ * Master context for 'stat_run'.
+ */
+struct StatMaster
+{
+  struct GNUNET_STATISTICS_Handle *stat;
+  struct GNUNET_TESTBED_Operation *op;
+  unsigned int daemon;
+  unsigned int value;
+};
+
+struct StatValues
+{
+  const char *subsystem;
+  const char *name;
+};
+
+/**
+ * Statistics we print out.
+ */
+static struct StatValues stats[] = {
+  { "fs", "# artificial delays introduced (ms)" },
+  { "fs", "# queries forwarded" },
+  { "fs", "# replies received and matched" },
+  { "fs", "# results found locally" },
+  { "fs", "# requests forwarded due to high load" },
+  { "fs", "# requests done for free (low load)" },
+  { "fs", "# requests dropped, priority insufficient" },
+  { "fs", "# requests done for a price (normal load)" },
+  { "fs", "# requests dropped by datastore (queue length limit)" },
+  { "fs", "# P2P searches received" },
+  { "fs", "# P2P searches discarded (queue length bound)" },
+  { "fs", "# replies received for local clients" },
+  { "fs", "# queries retransmitted to same target" },
+  { "core", "# bytes decrypted" },
+  { "core", "# bytes encrypted" },
+  { "core", "# discarded CORE_SEND requests" },
+  { "core", "# discarded lower priority CORE_SEND requests" },
+  { "transport", "# bytes received via TCP" },
+  { "transport", "# bytes transmitted via TCP" },
+  { "datacache", "# bytes stored" },
+  { NULL, NULL }
+};
+
+
+static void
+cleanup ()
+{
+  GNUNET_SCHEDULER_shutdown ();
+  if (NULL != fn1)
+  {
+    GNUNET_DISK_directory_remove (fn1);
+    GNUNET_free (fn1);
+  }
+  if (NULL != fn2)
+  {
+    GNUNET_DISK_directory_remove (fn2);
+    GNUNET_free (fn2);
+  }
+}
+
+
+/**
+ * Callback function to process statistic values.
+ *
+ * @param cls closure
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+print_stat (void *cls, const char *subsystem, const char *name, uint64_t value,
+            int is_persistent)
+{
+  struct StatMaster *sm = cls;
+
+  fprintf (stderr, "Peer %2u: %12s/%50s = %12llu\n", sm->daemon, subsystem,
+           name, (unsigned long long) value);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Function that gathers stats from all daemons.
+ */
+static void
+stat_run (void *cls,
+          struct GNUNET_TESTBED_Operation *op,
+          void *ca_result,
+          const char *emsg);
+
+
+/**
+ * Function called when GET operation on stats is done.
+ */
+static void
+get_done (void *cls, int success)
+{
+  struct StatMaster *sm = cls;
+
+  GNUNET_break (GNUNET_OK == success);
+  sm->value++;
+  stat_run (sm, sm->op, sm->stat, NULL);
+}
+
+
+/**
+ * Adapter function called to establish a connection to
+ * statistics service.
+ *
+ * @param cls closure
+ * @param cfg configuration of the peer to connect to; will be available until
+ *          GNUNET_TESTBED_operation_done() is called on the operation returned
+ *          from GNUNET_TESTBED_service_connect()
+ * @return service handle to return in 'op_result', NULL on error
+ */
+static void *
+statistics_connect_adapter (void *cls,
+                            const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  return GNUNET_STATISTICS_create ("<driver>",
+                                   cfg);
+}
+
+
+/**
+ * Adapter function called to destroy a connection to
+ * statistics service.
+ *
+ * @param cls closure
+ * @param op_result service handle returned from the connect adapter
+ */
+static void
+statistics_disconnect_adapter (void *cls,
+                               void *op_result)
+{
+  GNUNET_STATISTICS_destroy (op_result, GNUNET_NO);
+}
+
+
+/**
+ * Function that gathers stats from all daemons.
+ */
+static void
+stat_run (void *cls,
+          struct GNUNET_TESTBED_Operation *op,
+          void *ca_result,
+          const char *emsg)
+{
+  struct StatMaster *sm = cls;
+
+  sm->stat = ca_result;
+  GNUNET_assert (NULL != sm->stat);
+  if (NULL != stats[sm->value].name)
+  {
+    GNUNET_STATISTICS_get (sm->stat,
+#if 0
+                           NULL, NULL,
+#else
+                           stats[sm->value].subsystem, stats[sm->value].name,
+#endif
+                           &get_done, &print_stat,
+                           sm);
+    return;
+  }
+  GNUNET_TESTBED_operation_done (sm->op);
+  sm->value = 0;
+  sm->daemon++;
+  if (NUM_DAEMONS == sm->daemon)
+  {
+    GNUNET_free (sm);
+    cleanup ();
+    return;
+  }
+  sm->op =
+    GNUNET_TESTBED_service_connect (NULL,
+                                    daemons[sm->daemon],
+                                    "statistics",
+                                    &stat_run, sm,
+                                    &statistics_connect_adapter,
+                                    &statistics_disconnect_adapter,
+                                    NULL);
+}
+
+
+static void
+do_report (void *cls)
+{
+  static int download_counter;
+  const char *type = cls;
+  struct GNUNET_TIME_Relative del;
+  char *fancy;
+  struct StatMaster *sm;
+
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time,
+                                                                    TIMEOUT)).
+      rel_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Timeout during download for type `%s', shutting down with error\n",
+                type);
+    ok = 1;
+    cleanup ();
+    return;
+  }
+  del = GNUNET_TIME_absolute_get_duration (start_time);
+  if (del.rel_value_us == 0)
+    del.rel_value_us = 1;
+  fancy =
+    GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE)
+                                    * 1000000LL / del.rel_value_us);
+  fprintf (stderr, "Download speed of type `%s' was %s/s\n", type, fancy);
+  GNUNET_free (fancy);
+  if (NUM_DAEMONS != ++download_counter)
+    return;                   /* more downloads to come */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Finished all downloads, getting statistics\n");
+  sm = GNUNET_new (struct StatMaster);
+  sm->op =
+    GNUNET_TESTBED_service_connect (NULL,
+                                    daemons[sm->daemon],
+                                    "statistics",
+                                    &stat_run, sm,
+                                    &statistics_connect_adapter,
+                                    &statistics_disconnect_adapter,
+                                    NULL);
+}
+
+
+static void
+do_downloads (void *cls, const struct GNUNET_FS_Uri *u2,
+              const char *fn)
+{
+  int anonymity;
+  unsigned int i;
+
+  if (NULL == u2)
+  {
+    cleanup ();
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Timeout during upload attempt, shutting down with error\n");
+    ok = 1;
+    return;
+  }
+  if (NULL != fn)
+    fn2 = GNUNET_strdup (fn);
+  uri2 = GNUNET_FS_uri_dup (u2);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  start_time = GNUNET_TIME_absolute_get ();
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+  /* (semi) leach-download(s); not true leaches since
+   * these peers do participate in sharing, they just
+   * don't have to offer anything *initially*.  */
+  for (i = 0; i < NUM_DAEMONS - 2; i++)
+    GNUNET_FS_TEST_download (daemons[i], TIMEOUT, anonymity,
+                             0 == (i % 2) ? SEED1 : SEED2,
+                             0 == (i % 2) ? uri1 : uri2, VERBOSE, &do_report,
+                             "leach");
+  /* mutual downloads of (primary) sharing peers */
+  GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity, SEED1,
+                           uri1, VERBOSE, &do_report, "seeder 2");
+  GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, SEED2,
+                           uri2, VERBOSE, &do_report, "seeder 1");
+}
+
+
+static void
+do_publish2 (void *cls,
+             const struct GNUNET_FS_Uri *u1,
+             const char *fn)
+{
+  int do_index;
+  int anonymity;
+
+  if (NULL == u1)
+  {
+    cleanup ();
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Timeout during upload attempt, shutting down with error\n");
+    ok = 1;
+    return;
+  }
+  if (NULL != fn)
+    fn1 = GNUNET_strdup (fn);
+  uri1 = GNUNET_FS_uri_dup (u1);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  if (NULL != strstr (progname, "index"))
+    do_index = GNUNET_YES;
+  else
+    do_index = GNUNET_NO;
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+
+  GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity,
+                          do_index, FILESIZE, SEED2, VERBOSE, &do_downloads,
+                          NULL);
+}
+
+
+static void
+do_publish1 (void *cls,
+             struct GNUNET_TESTBED_Operation *op,
+             const char *emsg)
+{
+  unsigned int *coco = cls;
+  int do_index;
+  int anonymity;
+
+  GNUNET_TESTBED_operation_done (op);
+  if (NULL != emsg)
+  {
+    cleanup ();
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error trying to connect: %s\n", emsg);
+    ok = 1;
+    return;
+  }
+  if (0 != (--(*coco)))
+    return; /* more connections to be created */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  if (NULL != strstr (progname, "index"))
+    do_index = GNUNET_YES;
+  else
+    do_index = GNUNET_NO;
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+  GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity,
+                          do_index, FILESIZE, SEED1, VERBOSE, &do_publish2,
+                          NULL);
+}
+
+
+static void
+do_connect (void *cls,
+            struct GNUNET_TESTBED_RunHandle *h,
+            unsigned int num_peers,
+            struct GNUNET_TESTBED_Peer **peers,
+            unsigned int links_succeeded,
+            unsigned int links_failed)
+{
+  static unsigned int coco;
+  unsigned int i;
+  unsigned int j;
+
+  GNUNET_assert (NUM_DAEMONS == num_peers);
+  for (i = 0; i < num_peers; i++)
+    daemons[i] = peers[i];
+  for (i = 0; i < NUM_DAEMONS; i++)
+    for (j = i + 1; j < NUM_DAEMONS; j++)
+    {
+      coco++;
+      GNUNET_TESTBED_overlay_connect (NULL,
+                                      &do_publish1,
+                                      &coco,
+                                      peers[i],
+                                      peers[j]);
+    }
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  progname = argv[0];
+  (void) GNUNET_TESTBED_test_run ("perf-gnunet-service-fs-p2p-respect",
+                                  "perf_gnunet_service_fs_p2p.conf",
+                                  NUM_DAEMONS,
+                                  0, NULL, NULL,
+                                  &do_connect, NULL);
+  GNUNET_DISK_purge_cfg_dir ("perf_gnunet_service_fs_p2p.conf",
+                             "GNUNET_TEST_HOME");
+  return ok;
+}
+
+
+/* end of perf_gnunet_service_fs_p2p_respect.c */
diff --git a/src/service/fs/test_fs.c b/src/service/fs/test_fs.c
new file mode 100644
index 000000000..7a57e98b0
--- /dev/null
+++ b/src/service/fs/test_fs.c
@@ -0,0 +1,262 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs.c
+ * @brief testcase for FS (upload-search-download-unindex)
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util.h"
+#include "gnunet_fsui_lib.h"
+
+#define DEBUG_VERBOSE GNUNET_NO
+
+#define CHECK(a) if (! (a)) { ok = GNUNET_NO; GNUNET_GE_BREAK (NULL, 0); \
+                              goto FAILURE; }
+
+static char *
+makeName (unsigned int i)
+{
+  char *fn;
+
+  fn = GNUNET_malloc (strlen ("/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST")
+                      + 14);
+  GNUNET_snprintf (fn,
+                   strlen ("/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST") + 14,
+                   "/tmp/gnunet-basic_fsui_test/BASIC_FSUI_TEST%u", i);
+  GNUNET_disk_directory_create_for_file (NULL, fn);
+  return fn;
+}
+
+
+static enum GNUNET_FSUI_EventType lastEvent;
+
+static struct GNUNET_MetaData *search_meta;
+
+static struct GNUNET_ECRS_URI *search_uri;
+
+static struct GNUNET_FSUI_Context *ctx;
+
+static void *
+eventCallback (void *cls, const GNUNET_FSUI_Event *event)
+{
+  static char unused;
+
+  switch (event->type)
+  {
+  case GNUNET_FSUI_search_resumed:
+  case GNUNET_FSUI_download_resumed:
+  case GNUNET_FSUI_upload_resumed:
+  case GNUNET_FSUI_unindex_resumed:
+    return &unused;
+
+  case GNUNET_FSUI_search_result:
+#if DEBUG_VERBOSE
+    printf ("Received search result\n");
+#endif
+    search_uri = GNUNET_ECRS_uri_duplicate (event->data.SearchResult.fi.uri);
+    search_meta = GNUNET_meta_data_duplicate (event->data.SearchResult.fi.meta);
+    break;
+
+  case GNUNET_FSUI_upload_completed:
+#if DEBUG_VERBOSE
+    printf ("Upload complete.\n");
+#endif
+    break;
+
+  case GNUNET_FSUI_download_completed:
+#if DEBUG_VERBOSE
+    printf ("Download complete.\n");
+#endif
+    break;
+
+  case GNUNET_FSUI_unindex_completed:
+#if DEBUG_VERBOSE
+    printf ("Unindex complete.\n");
+#endif
+    break;
+
+  default:
+    break;
+  }
+  lastEvent = event->type;
+  return NULL;
+}
+
+
+#define START_DAEMON 1
+
+int
+main (int argc, char *argv[])
+{
+#if START_DAEMON
+  struct GNUNET_OS_Process *daemon;
+#endif
+  int ok;
+  struct GNUNET_ECRS_URI *uri;
+  char *filename = NULL;
+
+  char *keywords[] = {
+    "fsui_foo",
+    "fsui_bar",
+  };
+  char keyword[40];
+  char *fn;
+  int prog;
+  struct GNUNET_MetaData *meta;
+  struct GNUNET_ECRS_URI *kuri;
+  struct GNUNET_GC_Configuration *cfg;
+  struct GNUNET_FSUI_UploadList *upload = NULL;
+  struct GNUNET_FSUI_SearchList *search = NULL;
+  struct GNUNET_FSUI_UnindexList *unindex = NULL;
+  struct GNUNET_FSUI_DownloadList *download = NULL;
+
+  cfg = GNUNET_GC_create ();
+  if (-1 == GNUNET_GC_parse_configuration (cfg, "check.conf"))
+  {
+    GNUNET_GC_free (cfg);
+    return -1;
+  }
+#if START_DAEMON
+  daemon = GNUNET_daemon_start (NULL, cfg, "peer.conf", GNUNET_NO);
+  GNUNET_GE_ASSERT (NULL, daemon != NULL);
+  CHECK (GNUNET_OK ==
+         GNUNET_wait_for_daemon_running (NULL, cfg, 60 * GNUNET_CRON_SECONDS));
+#endif
+  GNUNET_thread_sleep (5 * GNUNET_CRON_SECONDS);         /* give apps time to start */
+  ok = GNUNET_YES;
+
+  /* ACTUAL TEST CODE */
+  ctx = GNUNET_FSUI_start (NULL, cfg, "basic_fsui_test", 32,     /* thread pool size */
+                           GNUNET_NO,   /* no resume */
+                           &eventCallback, NULL);
+  CHECK (ctx != NULL);
+  filename = makeName (42);
+  GNUNET_disk_file_write (NULL, filename, "foo bar test!",
+                          strlen ("foo bar test!"), "600");
+  meta = GNUNET_meta_data_create ();
+  kuri =
+    GNUNET_ECRS_keyword_command_line_to_uri (NULL, 2,
+                                             (const char **) keywords);
+  /* upload */
+  upload = GNUNET_FSUI_upload_start (ctx, filename,
+                                     (GNUNET_FSUI_DirectoryScanCallback)
+                                     & GNUNET_disk_directory_scan, NULL, 0,                                                     /* anonymity */
+                                     0, /* priority */
+                                     GNUNET_YES, GNUNET_NO, GNUNET_NO,
+                                     GNUNET_get_time () + 5 * GNUNET_CRON_HOURS,
+                                     meta, kuri, kuri);
+  CHECK (upload != NULL);
+  GNUNET_ECRS_uri_destroy (kuri);
+  GNUNET_meta_data_destroy (meta);
+  prog = 0;
+  while (lastEvent != GNUNET_FSUI_upload_completed)
+  {
+    prog++;
+    CHECK (prog < 10000) GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS);
+    if (GNUNET_shutdown_test () == GNUNET_YES)
+      break;
+  }
+
+  /* search */
+  GNUNET_snprintf (keyword, 40, "+%s +%s", keywords[0], keywords[1]);
+  uri = GNUNET_ECRS_keyword_string_to_uri (NULL, keyword);
+  search = GNUNET_FSUI_search_start (ctx, 0, uri);
+  GNUNET_ECRS_uri_destroy (uri);
+  CHECK (search != NULL);
+  prog = 0;
+  while (lastEvent != GNUNET_FSUI_search_result)
+  {
+    prog++;
+    CHECK (prog < 10000);
+    GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS);
+    if (GNUNET_shutdown_test () == GNUNET_YES)
+      break;
+  }
+  GNUNET_FSUI_search_abort (search);
+  GNUNET_FSUI_search_stop (search);
+
+  /* download */
+  fn = makeName (43);
+  download =
+    GNUNET_FSUI_download_start (ctx, 0, GNUNET_NO, search_uri, search_meta,
+                                fn, NULL, NULL);
+  GNUNET_free (fn);
+  prog = 0;
+  while (lastEvent != GNUNET_FSUI_download_completed)
+  {
+    prog++;
+    CHECK (prog < 10000);
+    GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS);
+    if (GNUNET_shutdown_test () == GNUNET_YES)
+      break;
+  }
+  GNUNET_FSUI_download_stop (download);
+  download = NULL;
+  GNUNET_ECRS_uri_destroy (search_uri);
+  GNUNET_meta_data_destroy (search_meta);
+  /* unindex */
+  unindex = GNUNET_FSUI_unindex_start (ctx, filename);
+  prog = 0;
+  while (lastEvent != GNUNET_FSUI_unindex_completed)
+  {
+    prog++;
+    CHECK (prog < 10000);
+    GNUNET_thread_sleep (50 * GNUNET_CRON_MILLISECONDS);
+    if (GNUNET_shutdown_test () == GNUNET_YES)
+      break;
+  }
+  if (lastEvent != GNUNET_FSUI_unindex_completed)
+    GNUNET_FSUI_unindex_abort (unindex);
+  GNUNET_FSUI_unindex_stop (unindex);
+
+
+  /* END OF TEST CODE */
+FAILURE:
+  if (ctx != NULL)
+    GNUNET_FSUI_stop (ctx);
+  if (filename != NULL)
+  {
+    unlink (filename);
+    GNUNET_free (filename);
+  }
+  if (download != NULL)
+  {
+    GNUNET_FSUI_download_abort (download);
+    GNUNET_FSUI_download_stop (download);
+  }
+  filename = makeName (43);
+  /* TODO: verify file 'filename(42)' == file 'filename(43)' */
+  unlink (filename);
+  GNUNET_free (filename);
+
+#if START_DAEMON
+  GNUNET_GE_ASSERT (NULL, GNUNET_OK == GNUNET_daemon_stop (NULL, daemon));
+  GNUNET_OS_process_destroy (daemon);
+#endif
+  GNUNET_GC_free (cfg);
+
+  return (ok == GNUNET_YES) ? 0 : 1;
+}
+
+
+/* end of test_fs.c */
diff --git a/src/service/fs/test_fs_data.conf b/src/service/fs/test_fs_data.conf
new file mode 100644
index 000000000..993141064
--- /dev/null
+++ b/src/service/fs/test_fs_data.conf
@@ -0,0 +1,7 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-data/
+
+[fs]
+ACTIVEMIGRATION = NO
+
diff --git a/src/service/fs/test_fs_defaults.conf b/src/service/fs/test_fs_defaults.conf
new file mode 100644
index 000000000..74456c4a2
--- /dev/null
+++ b/src/service/fs/test_fs_defaults.conf
@@ -0,0 +1,47 @@
+@INLINE@ ../../../contrib/conf/gnunet/no_forcestart.conf
+
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-lib/
+
+[transport]
+PLUGINS = tcp
+
+[nat]
+DISABLEV6 = YES
+ENABLE_UPNP = NO
+BEHIND_NAT = NO
+ALLOW_NAT = NO
+INTERNAL_ADDRESS = 127.0.0.1
+EXTERNAL_ADDRESS = 127.0.0.1
+USE_LOCALADDR = YES
+RETURN_LOCAL_ADDRESSES = YES
+
+[datastore]
+QUOTA = 100 MB
+
+[transport-tcp]
+BINDTO = 127.0.0.1
+PORT = 54368
+
+[peerinfo]
+NO_IO = YES
+
+[ats]
+WAN_QUOTA_IN = 65536
+WAN_QUOTA_OUT = 65536
+
+[fs]
+CONTENT_CACHING = YES
+CONTENT_PUSHING = YES
+DELAY = YES
+# PREFIX = valgrind --leak-check=full
+
+[dhtcache]
+QUOTA=65536
+DATABASE=heap
+
+[cadet]
+REFRESH_PATH_TIME = 30 min
+
+[nse]
+WORKBITS = 0
diff --git a/src/service/fs/test_fs_directory.c b/src/service/fs/test_fs_directory.c
new file mode 100644
index 000000000..f5121a3e7
--- /dev/null
+++ b/src/service/fs/test_fs_directory.c
@@ -0,0 +1,186 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2005, 2006, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_directory.c
+ * @brief Test for fs_directory.c
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#if HAVE_EXTRACTOR_H
+#include <extractor.h>
+#endif
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+
+#define ABORT() { fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__); \
+                  return 1; }
+
+struct PCLS
+{
+  struct GNUNET_FS_Uri **uri;
+  struct GNUNET_FS_MetaData **md;
+  unsigned int pos;
+  unsigned int max;
+};
+
+static void
+processor (void *cls, const char *filename, const struct GNUNET_FS_Uri *uri,
+           const struct GNUNET_FS_MetaData *md, size_t length,
+           const void *data)
+{
+  struct PCLS *p = cls;
+  int i;
+
+  if (NULL == uri)
+    return;                     /* ignore directory's meta data */
+  for (i = 0; i < p->max; i++)
+  {
+    if (GNUNET_FS_meta_data_test_equal (p->md[i], md) &&
+        GNUNET_FS_uri_test_equal (p->uri[i], uri))
+    {
+      p->pos++;
+      return;
+    }
+  }
+  fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__);
+}
+
+
+static int
+testDirectory (unsigned int i)
+{
+  struct GNUNET_FS_DirectoryBuilder *db;
+  char *data;
+  size_t dlen;
+  struct GNUNET_FS_Uri **uris;
+  struct GNUNET_FS_MetaData **mds;
+  struct GNUNET_FS_MetaData *meta;
+  struct PCLS cls;
+  char *emsg;
+  int p;
+  int q;
+  char uri[512];
+  char txt[128];
+  int ret = 0;
+  struct GNUNET_TIME_Absolute start;
+  const char *s;
+
+  cls.max = i;
+  uris = GNUNET_malloc (sizeof(struct GNUNET_FS_Uri *) * i);
+  mds = GNUNET_malloc (sizeof(struct GNUNET_FS_MetaData *) * i);
+  meta = GNUNET_FS_meta_data_create ();
+  GNUNET_FS_meta_data_insert (meta, "<test>", EXTRACTOR_METATYPE_TITLE,
+                              EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                              "A title", strlen ("A title") + 1);
+  GNUNET_FS_meta_data_insert (meta, "<test>",
+                              EXTRACTOR_METATYPE_AUTHOR_NAME,
+                              EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                              "An author", strlen ("An author") + 1);
+  for (p = 0; p < i; p++)
+  {
+    mds[p] = GNUNET_FS_meta_data_create ();
+    for (q = 0; q <= p; q++)
+    {
+      GNUNET_snprintf (txt, sizeof(txt), "%u -- %u\n", p, q);
+      GNUNET_FS_meta_data_insert (mds[p], "<test>",
+#if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR
+                                  q % EXTRACTOR_metatype_get_max (),
+#else
+                                  q % 128,
+#endif
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", txt, strlen (txt) + 1);
+    }
+    GNUNET_snprintf (uri, sizeof(uri),
+                     "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.%u",
+                     p);
+    emsg = NULL;
+    uris[p] = GNUNET_FS_uri_parse (uri, &emsg);
+    if (uris[p] == NULL)
+    {
+      GNUNET_FS_meta_data_destroy (mds[p]);
+      while (--p > 0)
+      {
+        GNUNET_FS_meta_data_destroy (mds[p]);
+        GNUNET_FS_uri_destroy (uris[p]);
+      }
+      GNUNET_free (mds);
+      GNUNET_free (uris);
+      GNUNET_free (emsg);
+      GNUNET_FS_meta_data_destroy (meta);
+      ABORT ();                 /* error in testcase */
+    }
+    GNUNET_assert (emsg == NULL);
+  }
+  start = GNUNET_TIME_absolute_get ();
+  db = GNUNET_FS_directory_builder_create (meta);
+  for (p = 0; p < i; p++)
+    GNUNET_FS_directory_builder_add (db, uris[p], mds[p], NULL);
+  GNUNET_FS_directory_builder_finish (db, &dlen, (void **) &data);
+  s = GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration
+                                                (start),
+                                              GNUNET_YES);
+  fprintf (stdout,
+           "Creating directory with %u entries and total size %llu took %s\n",
+           i, (unsigned long long) dlen, s);
+  if (i < 100)
+  {
+    cls.pos = 0;
+    cls.uri = uris;
+    cls.md = mds;
+    GNUNET_FS_directory_list_contents (dlen, data, 0, &processor, &cls);
+    GNUNET_assert (cls.pos == i);
+  }
+  GNUNET_free (data);
+  GNUNET_FS_meta_data_destroy (meta);
+  for (p = 0; p < i; p++)
+  {
+    GNUNET_FS_meta_data_destroy (mds[p]);
+    GNUNET_FS_uri_destroy (uris[p]);
+  }
+  GNUNET_free (uris);
+  GNUNET_free (mds);
+  return ret;
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int failureCount = 0;
+  int i;
+
+  GNUNET_log_setup ("test_fs_directory",
+#if VERBOSE
+                    "DEBUG",
+#else
+                    "WARNING",
+#endif
+                    NULL);
+  for (i = 17; i < 1000; i *= 2)
+    failureCount += testDirectory (i);
+  if (failureCount != 0)
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_directory.c */
diff --git a/src/service/fs/test_fs_download.c b/src/service/fs/test_fs_download.c
new file mode 100644
index 000000000..fc6b32c0f
--- /dev/null
+++ b/src/service/fs/test_fs_download.c
@@ -0,0 +1,368 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009, 2011, 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_download.c
+ * @brief simple testcase for simple publish + download operation
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+#include "gnunet_testing_lib.h"
+#include <gauger.h>
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+static unsigned int anonymity_level;
+
+static int indexed;
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_DownloadContext *download;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_SCHEDULER_Task *timeout_kill;
+
+static char *fn;
+
+static char *fn1;
+
+static int err;
+
+
+static void
+timeout_kill_task (void *cls)
+{
+  if (NULL != download)
+  {
+    GNUNET_FS_download_stop (download, GNUNET_YES);
+    download = NULL;
+  }
+  else if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n");
+  timeout_kill = NULL;
+  err = 1;
+}
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+}
+
+
+static void
+stop_fs_task (void *cls)
+{
+  GNUNET_FS_stop (fs);
+  fs = NULL;
+}
+
+
+static void
+abort_download_task (void *cls)
+{
+  uint64_t size;
+
+  if (NULL != download)
+  {
+    GNUNET_FS_download_stop (download, GNUNET_YES);
+    download = NULL;
+  }
+  GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES,
+                                                     GNUNET_NO));
+  GNUNET_assert (size == FILESIZE);
+  GNUNET_DISK_directory_remove (fn);
+  GNUNET_free (fn);
+  fn = NULL;
+  GNUNET_SCHEDULER_cancel (timeout_kill);
+  timeout_kill = NULL;
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    fprintf (stdout,
+             "Publishing complete, %llu kb/s.\n",
+             (unsigned long long) (FILESIZE * 1000000LL
+                                   / (1
+                                      + GNUNET_TIME_absolute_get_duration
+                                        (start).rel_value_us) / 1024LL));
+    GAUGER ("FS",
+            (GNUNET_YES == indexed)
+            ? "Publishing speed (indexing)"
+            : "Publishing speed (insertion)",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024LL), "kb/s");
+    fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst");
+    start = GNUNET_TIME_absolute_get ();
+    download =
+      GNUNET_FS_download_start (fs,
+                                event->value.publish.specifics.
+                                completed.chk_uri, NULL, fn, NULL, 0,
+                                FILESIZE, anonymity_level,
+                                GNUNET_FS_DOWNLOAD_OPTION_NONE,
+                                "download", NULL);
+    GNUNET_assert (download != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED:
+    fprintf (stdout,
+             "Download complete,  %llu kb/s.\n",
+             (unsigned long long) (FILESIZE * 1000000LL
+                                   / (1
+                                      + GNUNET_TIME_absolute_get_duration
+                                        (start).rel_value_us) / 1024LL));
+    GAUGER ("FS",
+            (GNUNET_YES == indexed)
+            ? "Local download speed (indexed)"
+            : "Local download speed (inserted)",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024LL), "kb/s");
+    GNUNET_SCHEDULER_add_now (&abort_download_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS:
+    GNUNET_assert (download == event->value.download.dc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Download is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.download.completed,
+                (unsigned long long) event->value.download.size,
+                event->value.download.specifics.progress.depth,
+                (unsigned long long) event->value.download.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    GNUNET_SCHEDULER_shutdown ();
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_ERROR:
+    fprintf (stderr, "Error downloading file: %s\n",
+             event->value.download.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_download_task, NULL);
+    GNUNET_SCHEDULER_shutdown ();
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE:
+  case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_SCHEDULER_add_now (&stop_fs_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_START:
+    GNUNET_assert (0 == strcmp ("download", event->value.download.cctx));
+    GNUNET_assert (NULL == event->value.download.pctx);
+    GNUNET_assert (NULL != event->value.download.uri);
+    GNUNET_assert (0 == strcmp (fn, event->value.download.filename));
+    GNUNET_assert (FILESIZE == event->value.download.size);
+    GNUNET_assert (0 == event->value.download.completed);
+    GNUNET_assert (1 == event->value.download.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_STOPPED:
+    GNUNET_assert (download == event->value.download.dc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *binary_name = cls;
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  if (GNUNET_YES ==
+      GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                            "download-test",
+                                            "USE_STREAM"))
+    anonymity_level = 0;
+  else
+    anonymity_level = 1;
+  fs = GNUNET_FS_start (cfg, binary_name, &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = anonymity_level;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+
+  if (GNUNET_YES ==
+      GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                            "download-test",
+                                            "USE_INDEX"))
+  {
+    fn1 = GNUNET_DISK_mktemp ("gnunet-download-indexed-test");
+    (void) GNUNET_DISK_directory_remove (fn1);
+    GNUNET_assert (GNUNET_OK ==
+                   GNUNET_DISK_fn_write (fn1,
+                                         buf,
+                                         FILESIZE,
+                                         GNUNET_DISK_PERM_USER_READ
+                                         | GNUNET_DISK_PERM_USER_WRITE));
+    GNUNET_free (buf);
+    fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context",
+                                                      fn1,
+                                                      kuri, meta, GNUNET_YES,
+                                                      &bo);
+    indexed = GNUNET_YES;
+  }
+  else
+  {
+    fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                      FILESIZE, buf, kuri, meta,
+                                                      GNUNET_NO, &bo);
+    /* note: buf will be free'd as part of 'fi' now */
+    indexed = GNUNET_NO;
+  }
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  timeout_kill =
+    GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  const char *binary_name;
+  const char *config_name;
+
+  binary_name = "test-fs-download";
+  config_name = "test_fs_download_data.conf";
+  if (NULL != strstr (argv[0], "indexed"))
+  {
+    binary_name = "test-fs-download-indexed";
+    config_name = "test_fs_download_indexed.conf";
+  }
+  if (NULL != strstr (argv[0], "cadet"))
+  {
+    binary_name = "test-fs-download-cadet";
+    config_name = "test_fs_download_cadet.conf";
+  }
+  if (0 != GNUNET_TESTING_peer_run (binary_name,
+                                    config_name,
+                                    &run, (void *) binary_name))
+    return 1;
+  if (NULL != fn1)
+  {
+    unlink (fn1);
+    GNUNET_free (fn1);
+  }
+  return err;
+}
+
+
+/* end of test_fs_download.c */
diff --git a/src/service/fs/test_fs_download_data.conf b/src/service/fs/test_fs_download_data.conf
new file mode 100644
index 000000000..160dec3ae
--- /dev/null
+++ b/src/service/fs/test_fs_download_data.conf
@@ -0,0 +1,10 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-download/
+
+[download-test]
+# set to 'YES' to test non-anonymous download
+USE_STREAM = NO
+
+# set to 'YES' to use indexing
+USE_INDEX = NO
\ No newline at end of file
diff --git a/src/service/fs/test_fs_download_indexed.conf b/src/service/fs/test_fs_download_indexed.conf
new file mode 100644
index 000000000..7f1e36935
--- /dev/null
+++ b/src/service/fs/test_fs_download_indexed.conf
@@ -0,0 +1,10 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-download/
+
+[download-test]
+# set to 'YES' to test non-anonymous download
+USE_STREAM = NO
+
+# set to 'YES' to use indexing
+USE_INDEX = YES
diff --git a/src/service/fs/test_fs_download_persistence.c b/src/service/fs/test_fs_download_persistence.c
new file mode 100644
index 000000000..b66fefd6b
--- /dev/null
+++ b/src/service/fs/test_fs_download_persistence.c
@@ -0,0 +1,351 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_download_persistence.c
+ * @brief simple testcase for persistence of simple download operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_DownloadContext *download;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_SCHEDULER_Task *timeout_kill;
+
+static char *fn;
+
+static int err;
+
+
+static void
+timeout_kill_task (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n");
+  if (download != NULL)
+  {
+    GNUNET_FS_download_stop (download, GNUNET_YES);
+    download = NULL;
+  }
+  else if (publish != NULL)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  timeout_kill = NULL;
+  err = 1;
+}
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (publish != NULL)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+}
+
+
+static void
+abort_download_task (void *cls)
+{
+  uint64_t size;
+
+  if (download != NULL)
+  {
+    GNUNET_FS_download_stop (download, GNUNET_YES);
+    download = NULL;
+  }
+  GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES,
+                                                     GNUNET_NO));
+  GNUNET_assert (size == FILESIZE);
+  GNUNET_DISK_directory_remove (fn);
+  GNUNET_free (fn);
+  fn = NULL;
+  GNUNET_SCHEDULER_cancel (timeout_kill);
+  timeout_kill = NULL;
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event);
+
+
+static void
+restart_fs_task (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Restarting FS.\n");
+  GNUNET_FS_stop (fs);
+  fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+}
+
+
+/**
+ * Consider scheduling the restart-task.
+ * Only runs the restart task once per event
+ * category.
+ *
+ * @param ev type of the event to consider
+ */
+static void
+consider_restart (int ev)
+{
+  static int prev[32];
+  static int off;
+  int i;
+
+  for (i = 0; i < off; i++)
+    if (prev[i] == ev)
+      return;
+  prev[off++] = ev;
+  GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT,
+                                      &restart_fs_task, NULL);
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    printf ("Publishing complete, %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024LL));
+    fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst");
+    start = GNUNET_TIME_absolute_get ();
+    GNUNET_assert (download == NULL);
+    GNUNET_FS_download_start (fs,
+                              event->value.publish.specifics.completed.chk_uri,
+                              NULL, fn, NULL, 0, FILESIZE, 1,
+                              GNUNET_FS_DOWNLOAD_OPTION_NONE, "download", NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED:
+    printf ("Download complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024LL));
+    GNUNET_SCHEDULER_add_now (&abort_download_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS:
+    consider_restart (event->status);
+    GNUNET_assert (download == event->value.download.dc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Download is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.download.completed,
+                (unsigned long long) event->value.download.size,
+                event->value.download.specifics.progress.depth,
+                (unsigned long long) event->value.download.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_ERROR:
+    fprintf (stderr, "Error downloading file: %s\n",
+             event->value.download.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_download_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_SUSPEND:
+    GNUNET_assert (event->value.publish.pc == publish);
+    publish = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_RESUME:
+    GNUNET_assert (NULL == publish);
+    publish = event->value.publish.pc;
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download suspended.\n");
+    GNUNET_assert (event->value.download.dc == download);
+    download = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_RESUME:
+    GNUNET_assert (NULL == download);
+    download = event->value.download.dc;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download resumed.\n");
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE:
+    consider_restart (event->status);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download active.\n");
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE:
+    consider_restart (event->status);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download inactive.\n");
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_START:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download started.\n");
+    consider_restart (event->status);
+    GNUNET_assert (download == NULL);
+    download = event->value.download.dc;
+    GNUNET_assert (0 == strcmp ("download", event->value.download.cctx));
+    GNUNET_assert (NULL == event->value.download.pctx);
+    GNUNET_assert (NULL != event->value.download.uri);
+    GNUNET_assert (0 == strcmp (fn, event->value.download.filename));
+    GNUNET_assert (FILESIZE == event->value.download.size);
+    GNUNET_assert (0 == event->value.download.completed);
+    GNUNET_assert (1 == event->value.download.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_DOWNLOAD_STOPPED:
+    GNUNET_assert (download == event->value.download.dc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    download = NULL;
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  cfg = c;
+  fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                    FILESIZE, buf, kuri, meta,
+                                                    GNUNET_NO, &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  timeout_kill =
+    GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-download-persistence",
+                                    "test_fs_download_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_download_persistence.c */
diff --git a/src/service/fs/test_fs_file_information.c b/src/service/fs/test_fs_file_information.c
new file mode 100644
index 000000000..15380bfc4
--- /dev/null
+++ b/src/service/fs/test_fs_file_information.c
@@ -0,0 +1,163 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_file_information.c
+ * @brief simple testcase for file_information operations
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - test that metadata, etc. are all correct (for example,
+ *   there is a known bug with dirname never being set that is
+ *   not detected!)
+ * - need to iterate over file-information structure
+ * - other API functions may not yet be tested (such as
+ *   filedata-from-callback)
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static int
+mycleaner (void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length,
+           struct GNUNET_FS_MetaData *meta, struct GNUNET_FS_Uri **uri,
+           struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info)
+{
+  return GNUNET_OK;
+}
+
+
+static void
+run (void *cls, char *const *args, const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *fn1;
+  char *fn2;
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi1;
+  struct GNUNET_FS_FileInformation *fi2;
+  struct GNUNET_FS_FileInformation *fidir;
+  struct GNUNET_FS_Handle *fs;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-file-information", NULL, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  fn1 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn1);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn1, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  fn2 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn2, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi1 =
+    GNUNET_FS_file_information_create_from_file (fs,
+                                                 "file_information-context1",
+                                                 fn1, kuri, meta, GNUNET_YES,
+                                                 &bo);
+  GNUNET_assert (fi1 != NULL);
+  fi2 =
+    GNUNET_FS_file_information_create_from_file (fs,
+                                                 "file_information-context2",
+                                                 fn2, kuri, meta, GNUNET_YES,
+                                                 &bo);
+  GNUNET_assert (fi2 != NULL);
+  fidir =
+    GNUNET_FS_file_information_create_empty_directory (fs,
+                                                       "file_information-context-dir",
+                                                       kuri, meta, &bo, NULL);
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1));
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2));
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fidir);
+  /* FIXME: test more of API! */
+  GNUNET_FS_file_information_destroy (fidir, &mycleaner, NULL);
+  GNUNET_DISK_directory_remove (fn1);
+  GNUNET_DISK_directory_remove (fn2);
+  GNUNET_free (fn1);
+  GNUNET_free (fn2);
+  GNUNET_FS_stop (fs);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  char *const argvx[] = {
+    "test-fs-file_information",
+    "-c",
+    "test_fs_file_information_data.conf",
+    NULL
+  };
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_END
+  };
+
+  GNUNET_log_setup ("test_fs_file_information",
+                    "WARNING",
+                    NULL);
+  GNUNET_PROGRAM_run ((sizeof(argvx) / sizeof(char *)) - 1, argvx,
+                      "test-fs-file_information", "nohelp", options, &run,
+                      NULL);
+  return 0;
+}
+
+
+/* end of test_fs_file_information.c */
diff --git a/src/service/fs/test_fs_file_information_data.conf b/src/service/fs/test_fs_file_information_data.conf
new file mode 100644
index 000000000..c8fc0938c
--- /dev/null
+++ b/src/service/fs/test_fs_file_information_data.conf
@@ -0,0 +1,7 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-file-information/
+
+[transport]
+PLUGINS = 
+
diff --git a/src/service/fs/test_fs_getopt.c b/src/service/fs/test_fs_getopt.c
new file mode 100644
index 000000000..3d0da752b
--- /dev/null
+++ b/src/service/fs/test_fs_getopt.c
@@ -0,0 +1,37 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_getopt.c
+ * @brief test for fs_getopt.c
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_fs_service.h"
+
+
+int
+main (int argc, char *argv[])
+{
+  GNUNET_log_setup ("test_fs_getopt",
+                    "WARNING",
+                    NULL);
+  fprintf (stderr, "%s", "WARNING: testcase not yet written.\n");
+  return 0;                     /* testcase passed */
+}
diff --git a/src/service/fs/test_fs_list_indexed.c b/src/service/fs/test_fs_list_indexed.c
new file mode 100644
index 000000000..7e06c47f5
--- /dev/null
+++ b/src/service/fs/test_fs_list_indexed.c
@@ -0,0 +1,265 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_list_indexed.c
+ * @brief simple testcase for list_indexed operation (indexing, listing
+ *        indexed)
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - actually call list_indexed API!
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static char *fn1;
+
+static char *fn2;
+
+static int err;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  GNUNET_FS_publish_stop (publish);
+  publish = NULL;
+  GNUNET_DISK_directory_remove (fn1);
+  GNUNET_free (fn1);
+  fn1 = NULL;
+  GNUNET_DISK_directory_remove (fn2);
+  GNUNET_free (fn2);
+  fn2 = NULL;
+}
+
+
+static void
+list_indexed_task (void *cls)
+{
+  GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  void *ret;
+
+  ret = NULL;
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    ret = event->value.publish.cctx;
+    printf ("Publish complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx))
+      GNUNET_SCHEDULER_add_now (&list_indexed_task, NULL);
+
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    ret = event->value.publish.cctx;
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    ret = event->value.publish.cctx;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    ret = event->value.publish.cctx;
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    err = 1;
+    if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx))
+      GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    ret = event->value.publish.cctx;
+    if (0 == strcmp ("list_indexed-context1", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("list_indexed-context-dir",
+                             event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (1 == event->value.publish.anonymity);
+    }
+    else if (0 == strcmp ("list_indexed-context2", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("list_indexed-context-dir",
+                             event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (2 == event->value.publish.anonymity);
+    }
+    else if (0 ==
+             strcmp ("list_indexed-context-dir", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (3 == event->value.publish.anonymity);
+    }
+    else
+      GNUNET_assert (0);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx))
+    {
+      GNUNET_assert (publish == event->value.publish.pc);
+      publish = NULL;
+    }
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return ret;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi1;
+  struct GNUNET_FS_FileInformation *fi2;
+  struct GNUNET_FS_FileInformation *fidir;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-list_indexed", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  fn1 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn1);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn1, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  fn2 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn2, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi1 =
+    GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context1",
+                                                 fn1, kuri, meta, GNUNET_YES,
+                                                 &bo);
+  GNUNET_assert (NULL != fi1);
+  bo.anonymity_level = 2;
+  fi2 =
+    GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context2",
+                                                 fn2, kuri, meta, GNUNET_YES,
+                                                 &bo);
+  GNUNET_assert (NULL != fi2);
+  bo.anonymity_level = 3;
+  fidir =
+    GNUNET_FS_file_information_create_empty_directory (fs,
+                                                       "list_indexed-context-dir",
+                                                       kuri, meta, &bo, NULL);
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1));
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2));
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fidir);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-list-indexed",
+                                    "test_fs_list_indexed_data.conf",
+                                    &run, NULL))
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_list_indexed.c */
diff --git a/src/service/fs/test_fs_list_indexed_data.conf b/src/service/fs/test_fs_list_indexed_data.conf
new file mode 100644
index 000000000..941809322
--- /dev/null
+++ b/src/service/fs/test_fs_list_indexed_data.conf
@@ -0,0 +1,10 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-list-indexed/
+
+[transport]
+PLUGINS = 
+
+[fs]
+ACTIVEMIGRATION = NO
+
diff --git a/src/service/fs/test_fs_meta_data.c b/src/service/fs/test_fs_meta_data.c
new file mode 100644
index 000000000..4e7439d7b
--- /dev/null
+++ b/src/service/fs/test_fs_meta_data.c
@@ -0,0 +1,492 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003, 2004, 2006, 2009, 2010, 2022 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file util/test_fs_meta_data.c
+ * @brief Test for fs_meta_data.c
+ * @author Christian Grothoff
+ * @author Martin Schanzenbach
+ */
+
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+
+#include "gnunet_fs_service.h"
+
+#define ABORT(m) { fprintf (stderr, "Error at %s:%d\n", __FILE__, __LINE__); \
+                   if (m != NULL) GNUNET_FS_meta_data_destroy (m); \
+                   return 1; }
+
+
+static int
+testMeta (int i)
+{
+  struct GNUNET_FS_MetaData *m;
+  char val[256];
+  char *sval;
+  int j;
+  unsigned int size;
+
+  m = GNUNET_FS_meta_data_create ();
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_insert (m, "<test>", EXTRACTOR_METATYPE_TITLE,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", "TestTitle",
+                                  strlen ("TestTitle") + 1))
+    ABORT (m);
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_insert (m, "<test>",
+                                  EXTRACTOR_METATYPE_AUTHOR_NAME,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", "TestTitle",
+                                  strlen ("TestTitle") + 1))
+    ABORT (m);
+  if (GNUNET_OK == GNUNET_FS_meta_data_insert (m, "<test>",
+                                               EXTRACTOR_METATYPE_TITLE,
+                                               EXTRACTOR_METAFORMAT_UTF8,
+                                               "text/plain",
+                                               "TestTitle", strlen (
+                                                 "TestTitle") + 1))                                                                                                             /* dup! */
+    ABORT (m);
+  if (GNUNET_OK == GNUNET_FS_meta_data_insert (m, "<test>",
+                                               EXTRACTOR_METATYPE_AUTHOR_NAME,
+                                               EXTRACTOR_METAFORMAT_UTF8,
+                                               "text/plain",
+                                               "TestTitle", strlen (
+                                                 "TestTitle") + 1))                                                                                                                     /* dup! */
+    ABORT (m);
+  if (2 != GNUNET_FS_meta_data_iterate (m, NULL, NULL))
+    ABORT (m);
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_delete (m, EXTRACTOR_METATYPE_AUTHOR_NAME,
+                                  "TestTitle", strlen ("TestTitle") + 1))
+    ABORT (m);
+  if (GNUNET_OK == GNUNET_FS_meta_data_delete (m,
+                                               EXTRACTOR_METATYPE_AUTHOR_NAME,
+                                               "TestTitle", strlen (
+                                                 "TestTitle") + 1))                                                                     /* already gone */
+    ABORT (m);
+  if (1 != GNUNET_FS_meta_data_iterate (m, NULL, NULL))
+    ABORT (m);
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_delete (m, EXTRACTOR_METATYPE_TITLE,
+                                  "TestTitle", strlen ("TestTitle") + 1))
+    ABORT (m);
+  if (GNUNET_OK == GNUNET_FS_meta_data_delete (m,
+                                               EXTRACTOR_METATYPE_TITLE,
+                                               "TestTitle", strlen (
+                                                 "TestTitle") + 1))                                                             /* already gone */
+    ABORT (m);
+  if (0 != GNUNET_FS_meta_data_iterate (m, NULL, NULL))
+    ABORT (m);
+  for (j = 0; j < i; j++)
+  {
+    GNUNET_snprintf (val, sizeof(val), "%s.%d",
+                     "A teststring that should compress well.", j);
+    if (GNUNET_OK !=
+        GNUNET_FS_meta_data_insert (m, "<test>",
+                                    EXTRACTOR_METATYPE_UNKNOWN,
+                                    EXTRACTOR_METAFORMAT_UTF8,
+                                    "text/plain", val, strlen (val) + 1))
+      ABORT (m);
+  }
+  if (i != GNUNET_FS_meta_data_iterate (m, NULL, NULL))
+    ABORT (m);
+
+  size = GNUNET_FS_meta_data_get_serialized_size (m);
+  sval = NULL;
+  if (size !=
+      GNUNET_FS_meta_data_serialize (m, &sval, size,
+                                     GNUNET_FS_META_DATA_SERIALIZE_FULL))
+  {
+    GNUNET_free (sval);
+    ABORT (m);
+  }
+  GNUNET_FS_meta_data_destroy (m);
+  m = GNUNET_FS_meta_data_deserialize (sval, size);
+  GNUNET_free (sval);
+  if (m == NULL)
+    ABORT (m);
+  for (j = 0; j < i; j++)
+  {
+    GNUNET_snprintf (val,
+                     sizeof(val),
+                     "%s.%d",
+                     "A teststring that should compress well.",
+                     j);
+    if (GNUNET_OK !=
+        GNUNET_FS_meta_data_delete (m,
+                                    EXTRACTOR_METATYPE_UNKNOWN,
+                                    val,
+                                    strlen (val) + 1))
+    {
+      ABORT (m);
+    }
+  }
+  if (0 != GNUNET_FS_meta_data_iterate (m, NULL, NULL))
+    ABORT (m);
+  GNUNET_FS_meta_data_destroy (m);
+  return 0;
+}
+
+
+static int
+testMetaMore (int i)
+{
+  struct GNUNET_FS_MetaData *meta;
+  int q;
+  char txt[128];
+  char *data;
+  unsigned long long size;
+
+  meta = GNUNET_FS_meta_data_create ();
+  for (q = 0; q <= i; q++)
+  {
+    GNUNET_snprintf (txt, 128, "%u -- %u\n", i, q);
+    GNUNET_FS_meta_data_insert (meta, "<test>",
+                                q
+                                % 42 /* EXTRACTOR_metatype_get_max () */,
+                                EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                txt, strlen (txt) + 1);
+  }
+  size = GNUNET_FS_meta_data_get_serialized_size (meta);
+  data = GNUNET_malloc (size * 4);
+  if (size !=
+      GNUNET_FS_meta_data_serialize (meta, &data, size * 4,
+                                     GNUNET_FS_META_DATA_SERIALIZE_FULL))
+  {
+    GNUNET_free (data);
+    ABORT (meta);
+  }
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_free (data);
+  return 0;
+}
+
+
+static int
+testMetaLink ()
+{
+  struct GNUNET_FS_MetaData *m;
+  char *val;
+  unsigned int size;
+
+  m = GNUNET_FS_meta_data_create ();
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_insert (m, "<test>",
+                                  EXTRACTOR_METATYPE_UNKNOWN,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", "link",
+                                  strlen ("link") + 1))
+    ABORT (m);
+  if (GNUNET_OK !=
+      GNUNET_FS_meta_data_insert (m, "<test>",
+                                  EXTRACTOR_METATYPE_FILENAME,
+                                  EXTRACTOR_METAFORMAT_UTF8,
+                                  "text/plain", "lib-link.m4",
+                                  strlen ("lib-link.m4") + 1))
+    ABORT (m);
+  val = NULL;
+  size =
+    GNUNET_FS_meta_data_serialize (m, &val, (size_t) -1,
+                                   GNUNET_FS_META_DATA_SERIALIZE_FULL);
+  GNUNET_FS_meta_data_destroy (m);
+  m = GNUNET_FS_meta_data_deserialize (val, size);
+  GNUNET_free (val);
+  if (m == NULL)
+    ABORT (m);
+  GNUNET_FS_meta_data_destroy (m);
+  return 0;
+}
+
+
+static int
+check ()
+{
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_MetaData *meta2;
+  int q;
+  int i = 100;
+  char txt[128];
+  char *str;
+  unsigned char *thumb;
+
+  meta = GNUNET_FS_meta_data_create ();
+  meta2 = GNUNET_FS_meta_data_create ();
+  for (q = 0; q <= i; q++)
+  {
+    GNUNET_snprintf (txt, 128, "%u -- %u\n", i, q);
+    GNUNET_FS_meta_data_insert (meta, "<test>",
+                                EXTRACTOR_METATYPE_UNKNOWN,
+                                EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                "TestTitle", strlen ("TestTitle") + 1);
+    GNUNET_FS_meta_data_insert (meta2, "<test>",
+                                EXTRACTOR_METATYPE_UNKNOWN,
+                                EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+                                "TestTitle", strlen ("TestTitle") + 1);
+  }
+
+  // check meta_data_test_equal
+  if (GNUNET_YES != GNUNET_FS_meta_data_test_equal (meta, meta2))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+
+  // check meta_data_clear
+  GNUNET_FS_meta_data_clear (meta2);
+  if (0 != GNUNET_FS_meta_data_iterate (meta2, NULL, NULL))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+  // check equal branch in meta_data_test_equal
+  if (GNUNET_YES != GNUNET_FS_meta_data_test_equal (meta, meta))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+  // check "count" branch in meta_data_test_equal
+  if (GNUNET_NO != GNUNET_FS_meta_data_test_equal (meta, meta2))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+
+  // check meta_data_add_publication_date
+  GNUNET_FS_meta_data_add_publication_date (meta2);
+
+  // check meta_data_merge
+  GNUNET_FS_meta_data_clear (meta2);
+  GNUNET_FS_meta_data_merge (meta2, meta);
+  if (100 == GNUNET_FS_meta_data_iterate (meta2, NULL, NULL))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+
+  // check meta_data_get_by_type
+  GNUNET_FS_meta_data_clear (meta2);
+  if (NULL !=
+      (str =
+         GNUNET_FS_meta_data_get_by_type (meta2,
+                                          EXTRACTOR_METATYPE_UNKNOWN)))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    GNUNET_free (str);
+    ABORT (meta);
+  }
+
+  str =
+    GNUNET_FS_meta_data_get_by_type (meta, EXTRACTOR_METATYPE_UNKNOWN);
+  GNUNET_assert (NULL != str);
+  if (str[0] != 'T')
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    GNUNET_free (str);
+    ABORT (meta);
+  }
+  GNUNET_free (str);
+
+  // check branch
+  if (NULL !=
+      (str =
+         GNUNET_FS_meta_data_get_by_type (meta,
+                                          EXTRACTOR_METATYPE_PUBLICATION_DATE)))
+  {
+    GNUNET_free (str);
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+
+  // check meta_data_get_first_by_types
+  str =
+    GNUNET_FS_meta_data_get_first_by_types (meta,
+                                            EXTRACTOR_METATYPE_UNKNOWN,
+                                            -1);
+  GNUNET_assert (NULL != str);
+  if (str[0] != 'T')
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    GNUNET_free (str);
+    ABORT (meta);
+  }
+  GNUNET_free (str);
+
+  // check meta_data_get_thumbnail
+  if (GNUNET_FS_meta_data_get_thumbnail (meta, &thumb) != 0)
+  {
+    GNUNET_free (thumb);
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+  GNUNET_FS_meta_data_destroy (meta2);
+  // check meta_data_duplicate
+  meta2 = GNUNET_FS_meta_data_duplicate (meta);
+  if (200 == GNUNET_FS_meta_data_iterate (meta2, NULL, NULL))
+  {
+    GNUNET_FS_meta_data_destroy (meta2);
+    ABORT (meta);
+  }
+  GNUNET_FS_meta_data_destroy (meta2);
+  GNUNET_FS_meta_data_destroy (meta);
+  return 0;
+}
+
+
+static int
+test_bigmeta_rw (void)
+{
+  static char meta[1024 * 1024 * 10];
+  struct GNUNET_BIO_WriteHandle *wh;
+  struct GNUNET_BIO_ReadHandle *rh;
+  char *filename = GNUNET_DISK_mktemp ("gnunet_bio");
+  struct GNUNET_FS_MetaData *mdR = NULL;
+
+  memset (meta, 'b', sizeof (meta));
+  meta[sizeof (meta) - 1] = '\0';
+
+  wh = GNUNET_BIO_write_open_file (filename);
+  GNUNET_assert (NULL != wh);
+  if (GNUNET_OK != GNUNET_BIO_write_int32 (wh,
+                                           "test-bigmeta-rw-int32",
+                                           sizeof (meta)))
+  {
+    GNUNET_BIO_write_close (wh, NULL);
+    return 1;
+  }
+  if (GNUNET_OK != GNUNET_BIO_write (wh,
+                                     "test-bigmeta-rw-bytes",
+                                     meta,
+                                     sizeof (meta)))
+  {
+    GNUNET_BIO_write_close (wh, NULL);
+    return 1;
+  }
+  GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL));
+
+  rh = GNUNET_BIO_read_open_file (filename);
+  GNUNET_assert (NULL != rh);
+  GNUNET_assert (GNUNET_SYSERR ==
+                 GNUNET_FS_read_meta_data (rh,
+                                           "test-bigmeta-rw-metadata",
+                                           &mdR));
+  GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL));
+
+  GNUNET_assert (NULL == mdR);
+
+  GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename));
+  GNUNET_free (filename);
+  return 0;
+}
+
+static int
+test_fakemeta_rw (void)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  struct GNUNET_BIO_ReadHandle *rh;
+  char *filename = GNUNET_DISK_mktemp ("gnunet_bio");
+  struct GNUNET_FS_MetaData *mdR = NULL;
+
+  wh = GNUNET_BIO_write_open_file (filename);
+  GNUNET_assert (NULL != wh);
+  if (GNUNET_OK != GNUNET_BIO_write_int32 (wh,
+                                           "test-fakestring-rw-int32",
+                                           2))
+  {
+    GNUNET_BIO_write_close (wh, NULL);
+    return 1;
+  }
+  GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL));
+
+  rh = GNUNET_BIO_read_open_file (filename);
+  GNUNET_assert (NULL != rh);
+  GNUNET_assert (GNUNET_SYSERR ==
+                 GNUNET_FS_read_meta_data (rh,
+                                           "test-fakestring-rw-metadata",
+                                           &mdR));
+  GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL));
+
+  GNUNET_assert (NULL == mdR);
+
+  GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename));
+  GNUNET_free (filename);
+  return 0;
+}
+
+static int
+test_fakebigmeta_rw (void)
+{
+  struct GNUNET_BIO_WriteHandle *wh;
+  struct GNUNET_BIO_ReadHandle *rh;
+  char *filename = GNUNET_DISK_mktemp ("gnunet_bio");
+  struct GNUNET_FS_MetaData *mdR = NULL;
+  int32_t wNum = 1024 * 1024 * 10;
+
+  wh = GNUNET_BIO_write_open_file (filename);
+  GNUNET_assert (NULL != wh);
+  GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_int32 (wh,
+                                                      "test-fakebigmeta-rw-int32",
+                                                      wNum));
+  GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_close (wh, NULL));
+
+  rh = GNUNET_BIO_read_open_file (filename);
+  GNUNET_assert (NULL != rh);
+  GNUNET_assert (GNUNET_SYSERR ==
+                 GNUNET_FS_read_meta_data (rh,
+                                           "test-fakebigmeta-rw-metadata",
+                                           &mdR));
+  GNUNET_assert (GNUNET_SYSERR == GNUNET_BIO_read_close (rh, NULL));
+
+  GNUNET_assert (NULL == mdR);
+
+  GNUNET_assert (GNUNET_OK == GNUNET_DISK_directory_remove (filename));
+  GNUNET_free (filename);
+  return 0;
+}
+
+int
+main (int argc, char *argv[])
+{
+  int failureCount = 0;
+  int i;
+
+  GNUNET_log_setup ("test-fs-meta-data", "WARNING", NULL);
+  for (i = 0; i < 255; i++)
+    failureCount += testMeta (i);
+  for (i = 1; i < 255; i++)
+    failureCount += testMetaMore (i);
+  failureCount += testMetaLink ();
+  failureCount += test_fakebigmeta_rw ();
+  failureCount += test_fakemeta_rw ();
+  failureCount +=  test_bigmeta_rw ();
+  int ret = check ();
+
+  if (ret == 1)
+    return 1;
+
+  if (failureCount != 0)
+    return 1;
+  return 0;
+}
+
+
+/* end of test_container_meta_data.c */
diff --git a/src/service/fs/test_fs_namespace.c b/src/service/fs/test_fs_namespace.c
new file mode 100644
index 000000000..85d489598
--- /dev/null
+++ b/src/service/fs/test_fs_namespace.c
@@ -0,0 +1,320 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2005-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_namespace.c
+ * @brief Test for fs_namespace.c
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+static struct GNUNET_CRYPTO_EcdsaPublicKey nsid;
+
+static struct GNUNET_FS_Uri *sks_expect_uri;
+
+static struct GNUNET_FS_Uri *ksk_expect_uri;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_SearchContext *sks_search;
+
+static struct GNUNET_FS_SearchContext *ksk_search;
+
+static struct GNUNET_SCHEDULER_Task *kill_task;
+
+static int update_started;
+
+static int err;
+
+
+static void
+abort_ksk_search_task (void *cls)
+{
+  if (ksk_search != NULL)
+  {
+    GNUNET_FS_search_stop (ksk_search);
+    ksk_search = NULL;
+    if (sks_search == NULL)
+    {
+      GNUNET_FS_stop (fs);
+      if (NULL != kill_task)
+        GNUNET_SCHEDULER_cancel (kill_task);
+    }
+  }
+}
+
+
+static void
+abort_sks_search_task (void *cls)
+{
+  if (sks_search == NULL)
+    return;
+  GNUNET_FS_search_stop (sks_search);
+  sks_search = NULL;
+  if (ksk_search == NULL)
+  {
+    GNUNET_FS_stop (fs);
+    if (NULL != kill_task)
+      GNUNET_SCHEDULER_cancel (kill_task);
+  }
+}
+
+
+static void
+do_timeout (void *cls)
+{
+  err = 1;
+  fprintf (stderr, "%s", "Operation timed out\n");
+  kill_task = NULL;
+  abort_sks_search_task (NULL);
+  abort_ksk_search_task (NULL);
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    if (sks_search == event->value.search.sc)
+    {
+      if (! GNUNET_FS_uri_test_equal
+            (sks_expect_uri, event->value.search.specifics.result.uri))
+      {
+        fprintf (stderr, "%s", "Wrong result for sks search!\n");
+        err = 1;
+      }
+      /* give system 1ms to initiate update search! */
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
+                                    &abort_sks_search_task, NULL);
+    }
+    else if (ksk_search == event->value.search.sc)
+    {
+      if (! GNUNET_FS_uri_test_equal
+            (ksk_expect_uri, event->value.search.specifics.result.uri))
+      {
+        fprintf (stderr, "%s", "Wrong result for ksk search!\n");
+        err = 1;
+      }
+      GNUNET_SCHEDULER_add_now (&abort_ksk_search_task, NULL);
+    }
+    else
+    {
+      fprintf (stderr, "%s", "Unexpected search result received!\n");
+      GNUNET_break (0);
+    }
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    fprintf (stderr, "Error searching file: %s\n",
+             event->value.search.specifics.error.message);
+    if (sks_search == event->value.search.sc)
+      GNUNET_SCHEDULER_add_now (&abort_sks_search_task, NULL);
+    else if (ksk_search == event->value.search.sc)
+      GNUNET_SCHEDULER_add_now (&abort_ksk_search_task, NULL);
+    else
+      GNUNET_break (0);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+    GNUNET_assert ((NULL == event->value.search.cctx) ||
+                   (0 == strcmp ("sks_search", event->value.search.cctx)) ||
+                   (0 == strcmp ("ksk_search", event->value.search.cctx)));
+    if (NULL == event->value.search.cctx)
+    {
+      GNUNET_assert (0 == strcmp ("sks_search", event->value.search.pctx));
+      update_started = GNUNET_YES;
+    }
+    GNUNET_assert (1 == event->value.search.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    return NULL;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    return NULL;
+
+  default:
+    fprintf (stderr, "Unexpected event: %d\n", event->status);
+    break;
+  }
+  return event->value.search.cctx;
+}
+
+
+static void
+publish_cont (void *cls, const struct GNUNET_FS_Uri *ksk_uri, const char *emsg)
+{
+  char *msg;
+  struct GNUNET_FS_Uri *sks_uri;
+  char sbuf[1024];
+  char buf[1024];
+  char *ret;
+
+  if (NULL != emsg)
+  {
+    fprintf (stderr, "Error publishing: %s\n", emsg);
+    err = 1;
+    GNUNET_FS_stop (fs);
+    return;
+  }
+  ret = GNUNET_STRINGS_data_to_string (&nsid, sizeof(nsid), buf, sizeof(buf));
+  GNUNET_assert (NULL != ret);
+  ret[0] = '\0';
+  GNUNET_snprintf (sbuf, sizeof(sbuf), "gnunet://fs/sks/%s/this", buf);
+  sks_uri = GNUNET_FS_uri_parse (sbuf, &msg);
+  if (NULL == sks_uri)
+  {
+    fprintf (stderr, "failed to parse URI `%s': %s\n", sbuf, msg);
+    err = 1;
+    GNUNET_FS_stop (fs);
+    GNUNET_free (msg);
+    return;
+  }
+  ksk_search =
+    GNUNET_FS_search_start (fs, ksk_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                            "ksk_search");
+  sks_search =
+    GNUNET_FS_search_start (fs, sks_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                            "sks_search");
+  GNUNET_FS_uri_destroy (sks_uri);
+}
+
+
+static void
+sks_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
+{
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *ksk_uri;
+  char *msg;
+  struct GNUNET_FS_BlockOptions bo;
+
+  if (NULL == uri)
+  {
+    fprintf (stderr, "Error publishing: %s\n", emsg);
+    err = 1;
+    GNUNET_FS_stop (fs);
+    return;
+  }
+  meta = GNUNET_FS_meta_data_create ();
+  msg = NULL;
+  ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/ns-search", &msg);
+  GNUNET_assert (NULL == msg);
+  ksk_expect_uri = GNUNET_FS_uri_dup (uri);
+  bo.content_priority = 1;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time =
+    GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
+  GNUNET_FS_publish_ksk (fs, ksk_uri, meta, uri, &bo,
+                         GNUNET_FS_PUBLISH_OPTION_NONE, &publish_cont, NULL);
+  GNUNET_FS_uri_destroy (ksk_uri);
+  GNUNET_FS_meta_data_destroy (meta);
+}
+
+
+static void
+adv_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
+{
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey ns;
+  struct GNUNET_FS_BlockOptions bo;
+
+  if (NULL != emsg)
+  {
+    fprintf (stderr, "Error publishing: %s\n", emsg);
+    err = 1;
+    GNUNET_FS_stop (fs);
+    return;
+  }
+  GNUNET_CRYPTO_ecdsa_key_create (&ns);
+  meta = GNUNET_FS_meta_data_create ();
+  sks_expect_uri = GNUNET_FS_uri_dup (uri);
+  bo.content_priority = 1;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time =
+    GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
+  GNUNET_CRYPTO_ecdsa_key_get_public (&ns,
+                                      &nsid);
+  GNUNET_FS_publish_sks (fs,
+                         &ns, "this", "next", meta, uri,
+                         &bo, GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont, NULL);
+  GNUNET_FS_meta_data_destroy (meta);
+}
+
+
+static void
+testNamespace (void)
+{
+  struct GNUNET_FS_BlockOptions bo;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *ksk_uri;
+  struct GNUNET_FS_Uri *sks_uri;
+
+  meta = GNUNET_FS_meta_data_create ();
+  ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/testnsa", NULL);
+  bo.content_priority = 1;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time =
+    GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
+  sks_uri = GNUNET_FS_uri_sks_create (&nsid, "root");
+  GNUNET_FS_publish_ksk (fs,
+                         ksk_uri, meta, sks_uri,
+                         &bo, GNUNET_FS_PUBLISH_OPTION_NONE,
+                         &adv_cont, NULL);
+  GNUNET_FS_uri_destroy (sks_uri);
+  kill_task =
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &do_timeout,
+                                  NULL);
+  GNUNET_FS_uri_destroy (ksk_uri);
+  GNUNET_FS_meta_data_destroy (meta);
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  testNamespace ();
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-namespace",
+                                    "test_fs_namespace_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_namespace.c */
diff --git a/src/service/fs/test_fs_namespace_data.conf b/src/service/fs/test_fs_namespace_data.conf
new file mode 100644
index 000000000..70b954f7d
--- /dev/null
+++ b/src/service/fs/test_fs_namespace_data.conf
@@ -0,0 +1,7 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-namespace/
+
+[transport]
+PLUGINS = 
+
diff --git a/src/service/fs/test_fs_namespace_list_updateable.c b/src/service/fs/test_fs_namespace_list_updateable.c
new file mode 100644
index 000000000..d883b7bea
--- /dev/null
+++ b/src/service/fs/test_fs_namespace_list_updateable.c
@@ -0,0 +1,175 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2005-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_namespace_list_updateable.c
+ * @brief Test for fs_namespace_list_updateable.c
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+static struct GNUNET_FS_Handle *fs;
+
+static int err;
+
+static struct GNUNET_CRYPTO_EcdsaPrivateKey ns;
+
+static struct GNUNET_FS_MetaData *meta;
+
+static struct GNUNET_FS_Uri *uri_this;
+
+static struct GNUNET_FS_Uri *uri_next;
+
+static struct GNUNET_FS_BlockOptions bo;
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  return NULL;
+}
+
+
+static void
+do_shutdown ()
+{
+  if (uri_this != NULL)
+    GNUNET_FS_uri_destroy (uri_this);
+  if (uri_next != NULL)
+    GNUNET_FS_uri_destroy (uri_next);
+  if (meta != NULL)
+    GNUNET_FS_meta_data_destroy (meta);
+}
+
+
+static void
+check_next (void *cls, const char *last_id,
+            const struct GNUNET_FS_Uri *last_uri,
+            const struct GNUNET_FS_MetaData *last_meta,
+            const char *next_id)
+{
+  GNUNET_break (0 == strcmp (last_id, "next"));
+  GNUNET_break (0 == strcmp (next_id, "future"));
+  err -= 4;
+}
+
+
+static void
+check_this_next (void *cls, const char *last_id,
+                 const struct GNUNET_FS_Uri *last_uri,
+                 const struct GNUNET_FS_MetaData *last_meta,
+                 const char *next_id)
+{
+  GNUNET_break (0 == strcmp (last_id, "this"));
+  GNUNET_break (0 == strcmp (next_id, "next"));
+  err -= 2;
+  err += 4;
+  GNUNET_FS_namespace_list_updateable (fs, &ns, next_id, &check_next, NULL);
+}
+
+
+static void
+sks_cont_next (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
+{
+  GNUNET_assert (NULL == emsg);
+  err += 2;
+  GNUNET_FS_namespace_list_updateable (fs, &ns, NULL, &check_this_next, NULL);
+}
+
+
+static void
+check_this (void *cls, const char *last_id,
+            const struct GNUNET_FS_Uri *last_uri,
+            const struct GNUNET_FS_MetaData *last_meta,
+            const char *next_id)
+{
+  GNUNET_break (0 == strcmp (last_id, "this"));
+  GNUNET_break (0 == strcmp (next_id, "next"));
+  err -= 1;
+}
+
+
+static void
+sks_cont_this (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
+{
+  GNUNET_assert (NULL == emsg);
+  err = 1;
+  GNUNET_FS_namespace_list_updateable (fs, &ns, NULL, &check_this, NULL);
+  GNUNET_FS_publish_sks (fs,
+                         &ns, "next", "future", meta, uri_next, &bo,
+                         GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_next, NULL);
+}
+
+
+static void
+testNamespace (void)
+{
+  GNUNET_CRYPTO_ecdsa_key_create (&ns);
+  bo.content_priority = 1;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time =
+    GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
+  meta = GNUNET_FS_meta_data_create ();
+
+  uri_this =
+    GNUNET_FS_uri_parse
+    (
+      "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42",
+      NULL);
+  uri_next =
+    GNUNET_FS_uri_parse
+    (
+      "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.43",
+      NULL);
+  GNUNET_FS_publish_sks (fs,
+                         &ns, "this", "next", meta, uri_this, &bo,
+                         GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_this, NULL);
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  testNamespace ();
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-namespace-list-updateable",
+                                    "test_fs_namespace_data.conf",
+                                    &run, NULL))
+    return 1;
+  do_shutdown ();
+  return err;
+}
+
+
+/* end of test_fs_namespace_list_updateable.c */
diff --git a/src/service/fs/test_fs_publish.c b/src/service/fs/test_fs_publish.c
new file mode 100644
index 000000000..0e379bc29
--- /dev/null
+++ b/src/service/fs/test_fs_publish.c
@@ -0,0 +1,251 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_publish.c
+ * @brief simple testcase for publish operation (indexing, listing
+ *        indexed, directory structure)
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static char *fn1;
+
+static char *fn2;
+
+static int err;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  GNUNET_FS_publish_stop (publish);
+  publish = NULL;
+  GNUNET_DISK_directory_remove (fn1);
+  GNUNET_free (fn1);
+  fn1 = NULL;
+  GNUNET_DISK_directory_remove (fn2);
+  GNUNET_free (fn2);
+  fn2 = NULL;
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  void *ret;
+
+  ret = NULL;
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    ret = event->value.publish.cctx;
+    printf ("Publish complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    if (0 == strcmp ("publish-context-dir", event->value.publish.cctx))
+      GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    ret = event->value.publish.cctx;
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    ret = event->value.publish.cctx;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    ret = event->value.publish.cctx;
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    err = 1;
+    if (0 == strcmp ("publish-context-dir", event->value.publish.cctx))
+    {
+      fprintf (stderr, "Scheduling abort task for error on `%s'\n",
+               (const char *) event->value.publish.cctx);
+      GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    }
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    ret = event->value.publish.cctx;
+    if (0 == strcmp ("publish-context1", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("publish-context-dir", event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (1 == event->value.publish.anonymity);
+    }
+    else if (0 == strcmp ("publish-context2", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("publish-context-dir", event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (2 == event->value.publish.anonymity);
+    }
+    else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (3 == event->value.publish.anonymity);
+    }
+    else
+      GNUNET_assert (0);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    if (0 == strcmp ("publish-context-dir", event->value.publish.cctx))
+      GNUNET_assert (publish == event->value.publish.pc);
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return ret;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi1;
+  struct GNUNET_FS_FileInformation *fi2;
+  struct GNUNET_FS_FileInformation *fidir;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-publish", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn1);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn1, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn2, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+
+  fi1 =
+    GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1,
+                                                 kuri, meta, GNUNET_YES, &bo);
+
+  GNUNET_assert (NULL != fi1);
+  bo.anonymity_level = 2;
+  fi2 =
+    GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2,
+                                                 kuri, meta, GNUNET_YES, &bo);
+  GNUNET_assert (NULL != fi2);
+  bo.anonymity_level = 3;
+  fidir =
+    GNUNET_FS_file_information_create_empty_directory (fs,
+                                                       "publish-context-dir",
+                                                       kuri, meta, &bo, NULL);
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1));
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2));
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fidir);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-publish",
+                                    "test_fs_publish_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_publish.c */
diff --git a/src/service/fs/test_fs_publish_data.conf b/src/service/fs/test_fs_publish_data.conf
new file mode 100644
index 000000000..0930cdfed
--- /dev/null
+++ b/src/service/fs/test_fs_publish_data.conf
@@ -0,0 +1,10 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-publish/
+
+[transport]
+PLUGINS = 
+
+[fs]
+ACTIVEMIGRATION = NO
+
diff --git a/src/service/fs/test_fs_publish_persistence.c b/src/service/fs/test_fs_publish_persistence.c
new file mode 100644
index 000000000..e1563f448
--- /dev/null
+++ b/src/service/fs/test_fs_publish_persistence.c
@@ -0,0 +1,322 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_publish_persistence.c
+ * @brief simple testcase for persistence of simple publish operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static char *fn1;
+
+static char *fn2;
+
+static int err;
+
+static struct GNUNET_SCHEDULER_Task *rtask;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  GNUNET_FS_publish_stop (publish);
+  publish = NULL;
+  GNUNET_DISK_directory_remove (fn1);
+  GNUNET_free (fn1);
+  fn1 = NULL;
+  GNUNET_DISK_directory_remove (fn2);
+  GNUNET_free (fn2);
+  fn2 = NULL;
+  GNUNET_FS_stop (fs);
+  fs = NULL;
+  if (NULL != rtask)
+  {
+    GNUNET_SCHEDULER_cancel (rtask);
+    rtask = NULL;
+  }
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event);
+
+
+static void
+restart_fs_task (void *cls)
+{
+  rtask = NULL;
+  GNUNET_FS_stop (fs);
+  fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence",
+                        &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE,
+                        GNUNET_FS_OPTIONS_END);
+}
+
+
+/**
+ * Consider scheduling the restart-task.
+ * Only runs the restart task once per event
+ * category.
+ *
+ * @param ev type of the event to consider
+ */
+static void
+consider_restart (int ev)
+{
+  static int prev[32];
+  static int off;
+  int i;
+
+  for (i = 0; i < off; i++)
+    if (prev[i] == ev)
+      return;
+  prev[off++] = ev;
+  rtask =
+    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT,
+                                        &restart_fs_task, NULL);
+}
+
+
+static void *
+progress_cb (void *cls,
+             const struct GNUNET_FS_ProgressInfo *event)
+{
+  void *ret;
+
+  ret = NULL;
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    ret = event->value.publish.cctx;
+    printf ("Publish complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    if ((NULL != event->value.publish.cctx) &&
+        (0 == strcmp ("publish-context-dir", event->value.publish.cctx)))
+      GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    ret = event->value.publish.cctx;
+    return ret;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    consider_restart (event->status);
+    ret = event->value.publish.cctx;
+    GNUNET_assert (publish == event->value.publish.pc);
+#if VERBOSE
+    printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+            (unsigned long long) event->value.publish.completed,
+            (unsigned long long) event->value.publish.size,
+            event->value.publish.specifics.progress.depth,
+            (unsigned long long) event->value.publish.specifics.
+            progress.offset);
+#endif
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_SUSPEND:
+    if (event->value.publish.pc == publish)
+      publish = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_RESUME:
+    if (NULL == publish)
+    {
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_FS_file_information_is_directory (event->
+                                                              value.publish.
+                                                              fi));
+      publish = event->value.publish.pc;
+      return "publish-context-dir";
+    }
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    ret = event->value.publish.cctx;
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    err = 1;
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    consider_restart (event->status);
+    publish = event->value.publish.pc;
+    ret = event->value.publish.cctx;
+    if (0 == strcmp ("publish-context1", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("publish-context-dir", event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (1 == event->value.publish.anonymity);
+    }
+    else if (0 == strcmp ("publish-context2", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 ==
+                     strcmp ("publish-context-dir", event->value.publish.pctx));
+      GNUNET_assert (FILESIZE == event->value.publish.size);
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (2 == event->value.publish.anonymity);
+    }
+    else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx))
+    {
+      GNUNET_assert (0 == event->value.publish.completed);
+      GNUNET_assert (3 == event->value.publish.anonymity);
+    }
+    else
+      GNUNET_assert (0);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    consider_restart (event->status);
+    if ((NULL != event->value.publish.cctx) &&
+        (0 == strcmp ("publish-context-dir", event->value.publish.cctx)))
+      GNUNET_assert (publish == event->value.publish.pc);
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return ret;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi1;
+  struct GNUNET_FS_FileInformation *fi2;
+  struct GNUNET_FS_FileInformation *fidir;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  cfg = c;
+  fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn1);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn1, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst");
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn2);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn2, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi1 =
+    GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1,
+                                                 kuri, meta, GNUNET_YES, &bo);
+  GNUNET_assert (NULL != fi1);
+  bo.anonymity_level = 2;
+  fi2 =
+    GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2,
+                                                 kuri, meta, GNUNET_YES, &bo);
+  GNUNET_assert (NULL != fi2);
+  bo.anonymity_level = 3;
+  fidir =
+    GNUNET_FS_file_information_create_empty_directory (fs,
+                                                       "publish-context-dir",
+                                                       kuri, meta, &bo, NULL);
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1));
+  GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2));
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fidir);
+  start = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL,
+                           GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-publish-persistence",
+                                    "test_fs_publish_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_publish_persistence.c */
diff --git a/src/service/fs/test_fs_search.c b/src/service/fs/test_fs_search.c
new file mode 100644
index 000000000..f9266582e
--- /dev/null
+++ b/src/service/fs/test_fs_search.c
@@ -0,0 +1,252 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_search.c
+ * @brief simple testcase for simple publish + search operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE 1024
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_SearchContext *search;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+static int err;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+static void
+abort_error (void *cls)
+{
+  fprintf (stderr,
+           "Timeout\n");
+  timeout_task = NULL;
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  err = 1;
+}
+
+
+static void
+abort_search_task (void *cls)
+{
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  const char *keywords[] = {
+    "down_foo"
+  };
+  struct GNUNET_FS_Uri *kuri;
+
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords);
+    start = GNUNET_TIME_absolute_get ();
+    search =
+      GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                              "search");
+    GNUNET_FS_uri_destroy (kuri);
+    GNUNET_assert (search != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Search complete.\n");
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    fprintf (stderr, "Error searching file: %s\n",
+             event->value.search.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+    GNUNET_assert (search == NULL);
+    GNUNET_assert (0 == strcmp ("search", event->value.search.cctx));
+    GNUNET_assert (1 == event->value.search.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    GNUNET_assert (search == event->value.search.sc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    fprintf (stderr, "Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar"
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_BlockOptions bo;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                    FILESIZE, buf, kuri, meta,
+                                                    GNUNET_NO, &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+  timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME,
+                                               &abort_error, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-search",
+                                    "test_fs_search_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_search.c */
diff --git a/src/service/fs/test_fs_search_data.conf b/src/service/fs/test_fs_search_data.conf
new file mode 100644
index 000000000..8b24e328d
--- /dev/null
+++ b/src/service/fs/test_fs_search_data.conf
@@ -0,0 +1,7 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-search/
+
+[transport]
+PLUGINS = 
+
diff --git a/src/service/fs/test_fs_search_persistence.c b/src/service/fs/test_fs_search_persistence.c
new file mode 100644
index 000000000..4ddd40e73
--- /dev/null
+++ b/src/service/fs/test_fs_search_persistence.c
@@ -0,0 +1,318 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_search_persistence.c
+ * @brief simple testcase for persistence of search operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE 1024
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_SearchContext *search;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+static int err;
+
+
+static void
+abort_error (void *cls)
+{
+  timeout_task = NULL;
+  fprintf (stderr,
+           "Timeout\n");
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  err = 1;
+}
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+static void
+abort_search_task (void *cls)
+{
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event);
+
+
+static void
+restart_fs_task (void *cls)
+{
+  GNUNET_FS_stop (fs);
+  fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+}
+
+
+/**
+ * Consider scheduling the restart-task.
+ * Only runs the restart task once per event
+ * category.
+ *
+ * @param ev type of the event to consider
+ */
+static void
+consider_restart (int ev)
+{
+  static int prev[32];
+  static int off;
+  int i;
+
+  for (i = 0; i < off; i++)
+    if (prev[i] == ev)
+      return;
+  prev[off++] = ev;
+  GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT,
+                                      &restart_fs_task, NULL);
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  const char *keywords[] = {
+    "down_foo"
+  };
+  struct GNUNET_FS_Uri *kuri;
+
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords);
+    start = GNUNET_TIME_absolute_get ();
+    GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                            "search");
+    GNUNET_FS_uri_destroy (kuri);
+    GNUNET_assert (search != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_SUSPEND:
+    if (event->value.publish.pc == publish)
+      publish = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_RESUME:
+    if (NULL == publish)
+      publish = event->value.publish.pc;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    /* FIXME: consider_restart (event->status); cannot be tested with
+     * search result since we exit here after the first one... */
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Search complete.\n");
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    fprintf (stderr, "Error searching file: %s\n",
+             event->value.search.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_SUSPEND:
+    if (event->value.search.sc == search)
+      search = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESUME:
+    if (NULL == search)
+    {
+      search = event->value.search.sc;
+      return "search";
+    }
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+    consider_restart (event->status);
+    GNUNET_assert (search == NULL);
+    search = event->value.search.sc;
+    GNUNET_assert (0 == strcmp ("search", event->value.search.cctx));
+    GNUNET_assert (1 == event->value.search.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    GNUNET_assert (search == event->value.search.sc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    search = NULL;
+    break;
+
+  default:
+    fprintf (stderr, "Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar"
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  cfg = c;
+  fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                    FILESIZE, buf, kuri, meta,
+                                                    GNUNET_NO, &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+  timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME,
+                                               &abort_error, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-search-persistence",
+                                    "test_fs_search_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_search_persistence.c */
diff --git a/src/service/fs/test_fs_search_probes.c b/src/service/fs/test_fs_search_probes.c
new file mode 100644
index 000000000..776babaee
--- /dev/null
+++ b/src/service/fs/test_fs_search_probes.c
@@ -0,0 +1,258 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_search_probes.c
+ * @brief simple testcase for publish + search operation with probes
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE 1024
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_SearchContext *search;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+static int err;
+
+
+static void
+abort_error (void *cls)
+{
+  timeout_task = NULL;
+  fprintf (stderr,
+           "Timeout\n");
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  err = 1;
+}
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+static void
+abort_search_task (void *cls)
+{
+  if (search != NULL)
+    GNUNET_FS_search_stop (search);
+  search = NULL;
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  const char *keywords[] = {
+    "down_foo"
+  };
+  struct GNUNET_FS_Uri *kuri;
+
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords);
+    start = GNUNET_TIME_absolute_get ();
+    search =
+      GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                              "search");
+    GNUNET_FS_uri_destroy (kuri);
+    GNUNET_assert (search != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Search complete.\n");
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    fprintf (stderr, "Error searching file: %s\n",
+             event->value.search.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_UPDATE:
+    if ((0 < event->value.search.specifics.update.availability_rank) &&
+        (0 < event->value.search.specifics.update.availability_certainty))
+      GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+    GNUNET_assert (search == NULL);
+    GNUNET_assert (0 == strcmp ("search", event->value.search.cctx));
+    GNUNET_assert (1 == event->value.search.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    GNUNET_assert (search == event->value.search.sc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    fprintf (stderr,
+             "Unexpected event: %d\n",
+             event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar"
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_BlockOptions bo;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_DO_PROBES,
+                        GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                    FILESIZE, buf, kuri, meta,
+                                                    GNUNET_NO, &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+  timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME,
+                                               &abort_error, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-search-probes",
+                                    "test_fs_search_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_search_probes.c */
diff --git a/src/service/fs/test_fs_search_with_and.c b/src/service/fs/test_fs_search_with_and.c
new file mode 100644
index 000000000..9c20936b6
--- /dev/null
+++ b/src/service/fs/test_fs_search_with_and.c
@@ -0,0 +1,272 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004-2013 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_search_with_and.c
+ * @brief testcase for publishing multiple files and search with a and operator
+ * @author Bruno Cabral - 99% based on Christian Grothoff code
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE 1024
+
+/**
+ * Number of files for testing.
+ */
+#define NUM_FILES 10
+
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_SearchContext *search;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static struct GNUNET_SCHEDULER_Task *timeout_task;
+
+static int err;
+
+static int processed_files;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  if (NULL != timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (timeout_task);
+    timeout_task = NULL;
+  }
+}
+
+
+static void
+abort_error (void *cls)
+{
+  fprintf (stderr,
+           "Timeout\n");
+  timeout_task = NULL;
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+  if (NULL != publish)
+  {
+    GNUNET_FS_publish_stop (publish);
+    publish = NULL;
+  }
+  err = 1;
+}
+
+
+static void
+abort_search_task (void *cls)
+{
+  if (NULL != search)
+  {
+    GNUNET_FS_search_stop (search);
+    search = NULL;
+  }
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  struct GNUNET_FS_Uri *kuri;
+
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    processed_files++;
+    if (processed_files == NUM_FILES)
+    {
+      char *emsg = NULL;
+      kuri = GNUNET_FS_uri_ksk_create ("+down_foo +down_bar", &emsg);
+      GNUNET_assert (kuri != NULL);
+
+      start = GNUNET_TIME_absolute_get ();
+      search =
+        GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE,
+                                "search");
+      GNUNET_FS_uri_destroy (kuri);
+      GNUNET_assert (search != NULL);
+    }
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Search complete.\n");
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_ERROR:
+    fprintf (stderr, "Error searching file: %s\n",
+             event->value.search.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_search_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_START:
+    GNUNET_assert (search == NULL);
+    GNUNET_assert (0 == strcmp ("search", event->value.search.cctx));
+    GNUNET_assert (1 == event->value.search.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED:
+    break;
+
+  case GNUNET_FS_STATUS_SEARCH_STOPPED:
+    GNUNET_assert (search == event->value.search.sc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    fprintf (stderr, "Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar"
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_BlockOptions bo;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  size_t j;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+
+  processed_files = 0;
+  for (j = 0; j < NUM_FILES; j++)
+  {
+    buf = GNUNET_malloc (FILESIZE);
+    for (i = 0; i < FILESIZE; i++)
+      buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+    meta = GNUNET_FS_meta_data_create ();
+    kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+    bo.content_priority = 42;
+    bo.anonymity_level = 1;
+    bo.replication_level = 0;
+    bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+    fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context",
+                                                      FILESIZE, buf, kuri, meta,
+                                                      GNUNET_NO, &bo);
+    GNUNET_FS_uri_destroy (kuri);
+    GNUNET_FS_meta_data_destroy (meta);
+    GNUNET_assert (NULL != fi);
+    start = GNUNET_TIME_absolute_get ();
+    publish =
+      GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                               GNUNET_FS_PUBLISH_OPTION_NONE);
+    GNUNET_assert (publish != NULL);
+  }
+
+
+  timeout_task = GNUNET_SCHEDULER_add_delayed (LIFETIME,
+                                               &abort_error, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-search-with-and",
+                                    "test_fs_search_data.conf",
+                                    &run, NULL))
+    return 1;
+  return err;
+}
+
+
+/* end of test_fs_search.c */
diff --git a/src/service/fs/test_fs_start_stop.c b/src/service/fs/test_fs_start_stop.c
new file mode 100644
index 000000000..509fbe76a
--- /dev/null
+++ b/src/service/fs/test_fs_start_stop.c
@@ -0,0 +1,64 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_start_stop.c
+ * @brief testcase for fs.c (start-stop only)
+ * @author Christian Grothoff
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  struct GNUNET_FS_Handle *fs;
+
+  fs = GNUNET_FS_start (cfg, "test-fs-start-stop", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  GNUNET_FS_stop (fs);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-start-stop",
+                                    "test_fs_data.conf",
+                                    &run, NULL))
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_start_stop.c */
diff --git a/src/service/fs/test_fs_test_lib.c b/src/service/fs/test_fs_test_lib.c
new file mode 100644
index 000000000..714dd452e
--- /dev/null
+++ b/src/service/fs/test_fs_test_lib.c
@@ -0,0 +1,181 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_test_lib.c
+ * @brief test fs test library
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fs_test_lib.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
+
+#define NUM_DAEMONS 2
+
+#define SEED 42
+
+static struct GNUNET_TESTBED_Peer *the_peers[NUM_DAEMONS];
+
+static struct GNUNET_TIME_Absolute start_time;
+
+static int ret;
+
+
+static void
+do_stop (void *cls)
+{
+  char *fn = cls;
+
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time,
+                                                                    TIMEOUT)).
+      rel_value_us)
+  {
+    GNUNET_break (0);
+    ret = 1;
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Finished download, shutting down\n");
+  }
+  if (NULL != fn)
+  {
+    GNUNET_DISK_directory_remove (fn);
+    GNUNET_free (fn);
+  }
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+do_download (void *cls,
+             const struct GNUNET_FS_Uri *uri,
+             const char *fn)
+{
+  if (NULL == uri)
+  {
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    ret = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Downloading %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  start_time = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_TEST_download (the_peers[0],
+                           TIMEOUT, 1, SEED,
+                           uri,
+                           VERBOSE,
+                           &do_stop,
+                           (NULL == fn) ? NULL : GNUNET_strdup (fn));
+}
+
+
+static void
+do_publish (void *cls,
+            struct GNUNET_TESTBED_Operation *op,
+            const char *emsg)
+{
+  GNUNET_TESTBED_operation_done (op);
+  if (NULL != emsg)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to connect peers: %s\n", emsg);
+    GNUNET_break (0);
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  GNUNET_FS_TEST_publish (the_peers[0], TIMEOUT, 1, GNUNET_NO, FILESIZE, SEED,
+                          VERBOSE, &do_download, NULL);
+}
+
+
+/**
+ * Actual main function for the test.
+ *
+ * @param cls closure
+ * @param h the run handle
+ * @param num_peers number of peers in 'peers'
+ * @param peers handle to peers run in the testbed
+ * @param links_succeeded the number of overlay link connection attempts that
+ *          succeeded
+ * @param links_failed the number of overlay link connection attempts that
+ *          failed
+ */
+static void
+run (void *cls,
+     struct GNUNET_TESTBED_RunHandle *h,
+     unsigned int num_peers,
+     struct GNUNET_TESTBED_Peer **peers,
+     unsigned int links_succeeded,
+     unsigned int links_failed)
+{
+  unsigned int i;
+
+  GNUNET_assert (NUM_DAEMONS == num_peers);
+  for (i = 0; i < num_peers; i++)
+    the_peers[i] = peers[i];
+  GNUNET_TESTBED_overlay_connect (NULL,
+                                  &do_publish,
+                                  NULL,
+                                  peers[0],
+                                  peers[1]);
+}
+
+
+/**
+ * Main function that initializes the testbed.
+ *
+ * @param argc ignored
+ * @param argv ignored
+ * @return 0 on success
+ */
+int
+main (int argc, char *argv[])
+{
+  GNUNET_DISK_purge_cfg_dir ("fs_test_lib_data.conf", 
+                             "GNUNET_TEST_HOME");
+  (void) GNUNET_TESTBED_test_run ("test_fs_test_lib",
+                                  "fs_test_lib_data.conf",
+                                  NUM_DAEMONS,
+                                  0, NULL, NULL,
+                                  &run, NULL);
+  GNUNET_DISK_purge_cfg_dir ("fs_test_lib_data.conf", 
+                             "GNUNET_TEST_HOME");
+  return ret;
+}
+
+
+/* end of test_fs_test_lib.c */
diff --git a/src/service/fs/test_fs_unindex.c b/src/service/fs/test_fs_unindex.c
new file mode 100644
index 000000000..dbc33090d
--- /dev/null
+++ b/src/service/fs/test_fs_unindex.c
@@ -0,0 +1,237 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_unindex.c
+ * @brief simple testcase for simple publish + unindex operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+#include "gnunet_testing_lib.h"
+
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_UnindexContext *unindex;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static char *fn;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  GNUNET_FS_publish_stop (publish);
+  publish = NULL;
+}
+
+
+static void
+abort_unindex_task (void *cls)
+{
+  GNUNET_FS_unindex_stop (unindex);
+  unindex = NULL;
+  GNUNET_DISK_directory_remove (fn);
+  GNUNET_free (fn);
+  fn = NULL;
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    printf ("Publishing complete, %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    start = GNUNET_TIME_absolute_get ();
+    unindex = GNUNET_FS_unindex_start (fs, fn, "unindex");
+    GNUNET_assert (unindex != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
+    printf ("Unindex complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
+    GNUNET_assert (unindex == event->value.unindex.uc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Unindex is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.unindex.completed,
+                (unsigned long long) event->value.unindex.size,
+                event->value.unindex.specifics.progress.depth,
+                (unsigned long long) event->value.unindex.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_ERROR:
+    fprintf (stderr, "Error unindexing file: %s\n",
+             event->value.unindex.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_START:
+    GNUNET_assert (unindex == NULL);
+    GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx));
+    GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename));
+    GNUNET_assert (FILESIZE == event->value.unindex.size);
+    GNUNET_assert (0 == event->value.unindex.completed);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_STOPPED:
+    GNUNET_assert (unindex == event->value.unindex.uc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst");
+  fs = GNUNET_FS_start (cfg, "test-fs-unindex", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn,
+                                                    kuri, meta, GNUNET_YES,
+                                                    &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-unindex",
+                                    "test_fs_unindex_data.conf",
+                                    &run, NULL))
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_unindex.c */
diff --git a/src/service/fs/test_fs_unindex_data.conf b/src/service/fs/test_fs_unindex_data.conf
new file mode 100644
index 000000000..dde401857
--- /dev/null
+++ b/src/service/fs/test_fs_unindex_data.conf
@@ -0,0 +1,7 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/gnunet-test-fs-unindex/
+
+[transport]
+PLUGINS = 
+
diff --git a/src/service/fs/test_fs_unindex_persistence.c b/src/service/fs/test_fs_unindex_persistence.c
new file mode 100644
index 000000000..b81ce64ab
--- /dev/null
+++ b/src/service/fs/test_fs_unindex_persistence.c
@@ -0,0 +1,307 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2004, 2005, 2006, 2008, 2009, 2010 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_fs_unindex_persistence.c
+ * @brief simple testcase for simple publish + unindex operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_fs_service.h"
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 2)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
+
+/**
+ * How long should our test-content live?
+ */
+#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
+
+
+static struct GNUNET_TIME_Absolute start;
+
+static struct GNUNET_FS_Handle *fs;
+
+static struct GNUNET_FS_UnindexContext *unindex;
+
+static struct GNUNET_FS_PublishContext *publish;
+
+static char *fn;
+
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+
+static void
+abort_publish_task (void *cls)
+{
+  GNUNET_FS_publish_stop (publish);
+  publish = NULL;
+}
+
+
+static void
+abort_unindex_task (void *cls)
+{
+  if (unindex != NULL)
+  {
+    GNUNET_FS_unindex_stop (unindex);
+    unindex = NULL;
+  }
+  if (fn != NULL)
+  {
+    GNUNET_DISK_directory_remove (fn);
+    GNUNET_free (fn);
+    fn = NULL;
+  }
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event);
+
+
+static void
+restart_fs_task (void *cls)
+{
+  GNUNET_FS_stop (fs);
+  fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+}
+
+
+/**
+ * Consider scheduling the restart-task.
+ * Only runs the restart task once per event
+ * category.
+ *
+ * @param ev type of the event to consider
+ */
+static void
+consider_restart (int ev)
+{
+  static int prev[32];
+  static int off;
+  int i;
+
+  for (i = 0; i < off; i++)
+    if (prev[i] == ev)
+      return;
+  prev[off++] = ev;
+  GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT,
+                                      &restart_fs_task, NULL);
+}
+
+
+static void *
+progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event)
+{
+  switch (event->status)
+  {
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Publish is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.publish.completed,
+                (unsigned long long) event->value.publish.size,
+                event->value.publish.specifics.progress.depth,
+                (unsigned long long) event->value.publish.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
+    printf ("Publishing complete, %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    start = GNUNET_TIME_absolute_get ();
+    unindex = GNUNET_FS_unindex_start (fs, fn, "unindex");
+    GNUNET_assert (unindex != NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
+    printf ("Unindex complete,  %llu kbps.\n",
+            (unsigned long long) (FILESIZE * 1000000LL
+                                  / (1
+                                     + GNUNET_TIME_absolute_get_duration
+                                       (start).rel_value_us) / 1024));
+    GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
+    consider_restart (event->status);
+    GNUNET_assert (unindex == event->value.unindex.uc);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Unindex is progressing (%llu/%llu at level %u off %llu)...\n",
+                (unsigned long long) event->value.unindex.completed,
+                (unsigned long long) event->value.unindex.size,
+                event->value.unindex.specifics.progress.depth,
+                (unsigned long long) event->value.unindex.specifics.
+                progress.offset);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_SUSPEND:
+    if (event->value.publish.pc == publish)
+      publish = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_RESUME:
+    if (NULL == publish)
+    {
+      publish = event->value.publish.pc;
+      return "publish-context";
+    }
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_SUSPEND:
+    GNUNET_assert (event->value.unindex.uc == unindex);
+    unindex = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_RESUME:
+    GNUNET_assert (NULL == unindex);
+    unindex = event->value.unindex.uc;
+    return "unindex";
+
+  case GNUNET_FS_STATUS_PUBLISH_ERROR:
+    fprintf (stderr, "Error publishing file: %s\n",
+             event->value.publish.specifics.error.message);
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_ERROR:
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Error unindexing file: %s\n",
+                event->value.unindex.specifics.error.message);
+    GNUNET_SCHEDULER_add_now (&abort_unindex_task, NULL);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_START:
+    GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx));
+    GNUNET_assert (NULL == event->value.publish.pctx);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (0 == event->value.publish.completed);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    break;
+
+  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
+    GNUNET_assert (publish == event->value.publish.pc);
+    GNUNET_assert (FILESIZE == event->value.publish.size);
+    GNUNET_assert (1 == event->value.publish.anonymity);
+    GNUNET_FS_stop (fs);
+    fs = NULL;
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_START:
+    consider_restart (event->status);
+    GNUNET_assert (unindex == NULL);
+    GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx));
+    GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename));
+    GNUNET_assert (FILESIZE == event->value.unindex.size);
+    GNUNET_assert (0 == event->value.unindex.completed);
+    break;
+
+  case GNUNET_FS_STATUS_UNINDEX_STOPPED:
+    GNUNET_assert (unindex == event->value.unindex.uc);
+    GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL);
+    break;
+
+  default:
+    printf ("Unexpected event: %d\n", event->status);
+    break;
+  }
+  return NULL;
+}
+
+
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_TESTING_Peer *peer)
+{
+  const char *keywords[] = {
+    "down_foo",
+    "down_bar",
+  };
+  char *buf;
+  struct GNUNET_FS_MetaData *meta;
+  struct GNUNET_FS_Uri *kuri;
+  struct GNUNET_FS_FileInformation *fi;
+  size_t i;
+  struct GNUNET_FS_BlockOptions bo;
+
+  cfg = c;
+  fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst");
+  fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL,
+                        GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END);
+  GNUNET_assert (NULL != fs);
+  buf = GNUNET_malloc (FILESIZE);
+  for (i = 0; i < FILESIZE; i++)
+    buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256);
+  (void) GNUNET_DISK_directory_remove (fn);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_DISK_fn_write (fn, buf, FILESIZE,
+                                       GNUNET_DISK_PERM_USER_READ
+                                       | GNUNET_DISK_PERM_USER_WRITE));
+  GNUNET_free (buf);
+  meta = GNUNET_FS_meta_data_create ();
+  kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords);
+  bo.content_priority = 42;
+  bo.anonymity_level = 1;
+  bo.replication_level = 0;
+  bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME);
+  fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn,
+                                                    kuri, meta, GNUNET_YES,
+                                                    &bo);
+  GNUNET_FS_uri_destroy (kuri);
+  GNUNET_FS_meta_data_destroy (meta);
+  GNUNET_assert (NULL != fi);
+  start = GNUNET_TIME_absolute_get ();
+  publish =
+    GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL,
+                             GNUNET_FS_PUBLISH_OPTION_NONE);
+  GNUNET_assert (publish != NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  if (0 != GNUNET_TESTING_peer_run ("test-fs-unindex-persistence",
+                                    "test_fs_unindex_data.conf",
+                                    &run, NULL))
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_unindex_persistence.c */
diff --git a/src/service/fs/test_fs_uri.c b/src/service/fs/test_fs_uri.c
new file mode 100644
index 000000000..e0f23097b
--- /dev/null
+++ b/src/service/fs/test_fs_uri.c
@@ -0,0 +1,340 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2003-2014 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @file fs/test_fs_uri.c
+ * @brief Test for fs_uri.c
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_fs_service.h"
+#include "fs_api.h"
+
+
+static int
+testKeyword ()
+{
+  char *uri;
+  struct GNUNET_FS_Uri *ret;
+  char *emsg;
+
+  if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/++", &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/foo+bar", &emsg);
+  if (NULL == ret)
+  {
+    GNUNET_free (emsg);
+    GNUNET_assert (0);
+  }
+  if (! GNUNET_FS_uri_test_ksk (ret))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  if ((2 != ret->data.ksk.keywordCount) ||
+      (0 != strcmp (" foo", ret->data.ksk.keywords[0])) ||
+      (0 != strcmp (" bar", ret->data.ksk.keywords[1])))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+
+  uri = GNUNET_FS_uri_to_string (ret);
+  if (0 != strcmp (uri, "gnunet://fs/ksk/foo+bar"))
+  {
+    GNUNET_free (uri);
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (uri);
+  GNUNET_FS_uri_destroy (ret);
+  return 0;
+}
+
+
+static int
+testLocation ()
+{
+  struct GNUNET_FS_Uri *uri;
+  char *uric;
+  struct GNUNET_FS_Uri *uri2;
+  struct GNUNET_FS_Uri *baseURI;
+  char *emsg = NULL;
+  struct GNUNET_CRYPTO_EddsaPrivateKey pk;
+
+  baseURI =
+    GNUNET_FS_uri_parse
+    (
+      "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.15999",
+      &emsg);
+  GNUNET_assert (baseURI != NULL);
+  GNUNET_assert (emsg == NULL);
+  GNUNET_CRYPTO_eddsa_key_create (&pk);
+  uri = GNUNET_FS_uri_loc_create (baseURI,
+                                  &pk,
+                                  GNUNET_TIME_absolute_get ());
+  if (NULL == uri)
+  {
+    GNUNET_break (0);
+    GNUNET_FS_uri_destroy (baseURI);
+    return 1;
+  }
+  if (! GNUNET_FS_uri_test_loc (uri))
+  {
+    GNUNET_break (0);
+    GNUNET_FS_uri_destroy (uri);
+    GNUNET_FS_uri_destroy (baseURI);
+    return 1;
+  }
+  uri2 = GNUNET_FS_uri_loc_get_uri (uri);
+  if (! GNUNET_FS_uri_test_equal (baseURI, uri2))
+  {
+    GNUNET_break (0);
+    GNUNET_FS_uri_destroy (uri);
+    GNUNET_FS_uri_destroy (uri2);
+    GNUNET_FS_uri_destroy (baseURI);
+    return 1;
+  }
+  GNUNET_FS_uri_destroy (uri2);
+  GNUNET_FS_uri_destroy (baseURI);
+  uric = GNUNET_FS_uri_to_string (uri);
+#if 0
+  /* not for the faint of heart: */
+  printf ("URI: `%s'\n", uric);
+#endif
+  uri2 = GNUNET_FS_uri_parse (uric, &emsg);
+  GNUNET_free (uric);
+  if (uri2 == NULL)
+  {
+    fprintf (stderr, "URI parsing failed: %s\n", emsg);
+    GNUNET_break (0);
+    GNUNET_FS_uri_destroy (uri);
+    GNUNET_free (emsg);
+    return 1;
+  }
+  GNUNET_assert (NULL == emsg);
+  if (GNUNET_YES != GNUNET_FS_uri_test_equal (uri, uri2))
+  {
+    GNUNET_break (0);
+    GNUNET_FS_uri_destroy (uri);
+    GNUNET_FS_uri_destroy (uri2);
+    return 1;
+  }
+  GNUNET_FS_uri_destroy (uri2);
+  GNUNET_FS_uri_destroy (uri);
+  return 0;
+}
+
+
+static int
+testNamespace (int i)
+{
+  char *uri;
+  struct GNUNET_FS_Uri *ret;
+  char *emsg;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey ph;
+  struct GNUNET_CRYPTO_EcdsaPublicKey id;
+  char buf[1024];
+  char ubuf[1024];
+  char *sret;
+
+  if (NULL !=
+      (ret =
+         GNUNET_FS_uri_parse (
+           "gnunet://fs/sks/D1KJS9H2A82Q65VKQ0ML3RFU6U1D3VUK",
+           &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  if (NULL !=
+      (ret =
+         GNUNET_FS_uri_parse
+         (
+           "gnunet://fs/sks/XQHH4R288W26EBV369F6RCE0PJVJTX2Y74Q2FJPMPGA31HJX2JG/this",
+           &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/sks/test", &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  GNUNET_CRYPTO_ecdsa_key_create (&ph);
+  GNUNET_CRYPTO_ecdsa_key_get_public (&ph,
+                                      &id);
+  sret = GNUNET_STRINGS_data_to_string (&id, sizeof(id),
+                                        ubuf, sizeof(ubuf) - 1);
+  GNUNET_assert (NULL != sret);
+  sret[0] = '\0';
+  GNUNET_snprintf (buf, sizeof(buf),
+                   "gnunet://fs/sks/%s/test",
+                   ubuf);
+  ret = GNUNET_FS_uri_parse (buf, &emsg);
+  if (NULL == ret)
+  {
+    GNUNET_free (emsg);
+    GNUNET_assert (0);
+  }
+  if (GNUNET_FS_uri_test_ksk (ret))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  if (! GNUNET_FS_uri_test_sks (ret))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+
+  uri = GNUNET_FS_uri_to_string (ret);
+  if (0 !=
+      strcmp (uri,
+              buf))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_free (uri);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (uri);
+  GNUNET_FS_uri_destroy (ret);
+  return 0;
+}
+
+
+static int
+testFile (int i)
+{
+  char *uri;
+  struct GNUNET_FS_Uri *ret;
+  char *emsg;
+
+  if (NULL !=
+      (ret =
+         GNUNET_FS_uri_parse
+         (
+           "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H00000440000.42",
+           &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  if (NULL !=
+      (ret =
+         GNUNET_FS_uri_parse
+         (
+           "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000",
+           &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  if (NULL !=
+      (ret =
+         GNUNET_FS_uri_parse
+         (
+           "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.FGH",
+           &emsg)))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (emsg);
+  ret =
+    GNUNET_FS_uri_parse
+    (
+      "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.42",
+      &emsg);
+  if (ret == NULL)
+  {
+    GNUNET_free (emsg);
+    GNUNET_assert (0);
+  }
+  if (GNUNET_FS_uri_test_ksk (ret))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  if (GNUNET_FS_uri_test_sks (ret))
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  if (GNUNET_ntohll (ret->data.chk.file_length) != 42)
+  {
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+
+  uri = GNUNET_FS_uri_to_string (ret);
+  if (0 !=
+      strcmp (uri,
+              "gnunet://fs/chk/4QZP479A9SKGFNMQ2ZBCYE71YV2QMTVGWTVPB6A10ASVCKXDHB05DKPSC7ZF6E9P9W1VE47394EQY7NXA47Q6R35M7P1MJPGP59D1Z8.D54QD1K5XCG5878T6YZ19AM60MQ6FC0YNVK7QY08KK0KM0FJJ3KQWYG112FN5T07KN7J0X35DF6WVBT9B8ZMZ3X2BXJ22X3KFQ6MV2G.42"))
+  {
+    GNUNET_free (uri);
+    GNUNET_FS_uri_destroy (ret);
+    GNUNET_assert (0);
+  }
+  GNUNET_free (uri);
+  GNUNET_FS_uri_destroy (ret);
+  return 0;
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int failureCount = 0;
+  int i;
+
+  GNUNET_log_setup ("test_fs_uri",
+                    "WARNING",
+                    NULL);
+  failureCount += testKeyword ();
+  failureCount += testLocation ();
+  for (i = 0; i < 255; i++)
+  {
+    /* fprintf (stderr, "%s",  "."); */
+    failureCount += testNamespace (i);
+    failureCount += testFile (i);
+  }
+  /* fprintf (stderr, "%s",  "\n"); */
+  GNUNET_DISK_purge_cfg_dir 
+    ("test_fs_uri.conf", 
+     "GNUNET_TEST_HOME");
+
+  if (failureCount != 0)
+    return 1;
+  return 0;
+}
+
+
+/* end of test_fs_uri.c */
diff --git a/src/service/fs/test_gnunet_fs_rec_data.tgz b/src/service/fs/test_gnunet_fs_rec_data.tgz
new file mode 100644
index 000000000..697794306
--- /dev/null
+++ b/src/service/fs/test_gnunet_fs_rec_data.tgz
diff --git a/src/service/fs/test_gnunet_service_fs_migration.c b/src/service/fs/test_gnunet_service_fs_migration.c
new file mode 100644
index 000000000..38b00f3e8
--- /dev/null
+++ b/src/service/fs/test_gnunet_service_fs_migration.c
@@ -0,0 +1,223 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2012, 2015 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_gnunet_service_fs_migration.c
+ * @brief test content migration between two peers
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fs_test_lib.h"
+#include "gnunet_testbed_service.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (2 * 32 * 1024)
+
+/**
+ * How long until we give up on transmitting the message?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120)
+
+/**
+ * How long do we give the peers for content migration?
+ */
+#define MIGRATION_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, \
+                                                       90)
+
+#define SEED 42
+
+static struct GNUNET_TESTBED_Peer *daemons[2];
+
+static int ok;
+
+static struct GNUNET_TIME_Absolute start_time;
+
+static struct GNUNET_TESTBED_Operation *op;
+
+
+struct DownloadContext
+{
+  char *fn;
+
+  struct GNUNET_FS_Uri *uri;
+};
+
+
+static void
+do_stop (void *cls)
+{
+  struct GNUNET_TIME_Relative del;
+  char *fancy;
+
+  GNUNET_SCHEDULER_shutdown ();
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time,
+                                                                    TIMEOUT)).
+      rel_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Timeout during download, shutting down with error\n");
+    ok = 1;
+  }
+  else
+  {
+    del = GNUNET_TIME_absolute_get_duration (start_time);
+    if (del.rel_value_us == 0)
+      del.rel_value_us = 1;
+    fancy =
+      GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE)
+                                      * 1000000LL / del.rel_value_us);
+    fprintf (stdout,
+             "Download speed was %s/s\n",
+             fancy);
+    GNUNET_free (fancy);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Finished download, shutting down\n");
+  }
+}
+
+
+static void
+do_download (void *cls,
+             const char *emsg)
+{
+  struct DownloadContext *dc = cls;
+  struct GNUNET_FS_Uri *uri = dc->uri;
+
+  GNUNET_TESTBED_operation_done (op);
+  op = NULL;
+  if (NULL != dc->fn)
+  {
+    GNUNET_DISK_directory_remove (dc->fn);
+    GNUNET_free (dc->fn);
+  }
+  GNUNET_free (dc);
+  if (NULL != emsg)
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Failed to stop source daemon: %s\n",
+                emsg);
+    GNUNET_FS_uri_destroy (uri);
+    ok = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Downloading %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  start_time = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_TEST_download (daemons[0],
+                           TIMEOUT,
+                           1,
+                           SEED,
+                           uri,
+                           VERBOSE,
+                           &do_stop,
+                           NULL);
+  GNUNET_FS_uri_destroy (uri);
+}
+
+
+static void
+stop_source_peer (void *cls)
+{
+  struct DownloadContext *dc = cls;
+
+  /* FIXME: We should not interact with testbed when shutting down */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Stopping source peer\n");
+  op = GNUNET_TESTBED_peer_stop (NULL,
+                                 daemons[1],
+                                 &do_download, dc);
+  GNUNET_assert (NULL != op);
+}
+
+
+static void
+do_wait (void *cls,
+         const struct GNUNET_FS_Uri *uri,
+         const char *fn)
+{
+  struct DownloadContext *dc;
+
+  if (NULL == uri)
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Timeout during upload attempt, shutting down with error\n");
+    ok = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Waiting to allow content to migrate\n");
+  dc = GNUNET_new (struct DownloadContext);
+  dc->uri = GNUNET_FS_uri_dup (uri);
+  if (NULL != fn)
+    dc->fn = GNUNET_strdup (fn);
+  (void) GNUNET_SCHEDULER_add_delayed (MIGRATION_DELAY,
+                                       &stop_source_peer,
+                                       dc);
+}
+
+
+static void
+do_publish (void *cls,
+            struct GNUNET_TESTBED_RunHandle *h,
+            unsigned int num_peers,
+            struct GNUNET_TESTBED_Peer **peers,
+            unsigned int links_succeeded,
+            unsigned int links_failed)
+{
+  unsigned int i;
+
+  GNUNET_assert (2 == num_peers);
+  for (i = 0; i < num_peers; i++)
+    daemons[i] = peers[i];
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  GNUNET_FS_TEST_publish (daemons[1], TIMEOUT, 1, GNUNET_NO,
+                          FILESIZE, SEED,
+                          VERBOSE, &do_wait, NULL);
+}
+
+
+int
+main (int argc,
+      char *argv[])
+{
+  (void) GNUNET_TESTBED_test_run ("test-gnunet-service-fs-migration",
+                                  "fs_test_lib_data.conf",
+                                  2,
+                                  0, NULL, NULL,
+                                  &do_publish,
+                                  NULL);
+  GNUNET_DISK_purge_cfg_dir 
+    ("test_gnunet_service_fs_migration_data.conf", 
+     "GNUNET_TEST_HOME");
+  return ok;
+}
+
+
+/* end of test_gnunet_service_fs_migration.c */
diff --git a/src/service/fs/test_gnunet_service_fs_migration_data.conf b/src/service/fs/test_gnunet_service_fs_migration_data.conf
new file mode 100644
index 000000000..fca6c5a29
--- /dev/null
+++ b/src/service/fs/test_gnunet_service_fs_migration_data.conf
@@ -0,0 +1,10 @@
+@INLINE@ test_fs_defaults.conf
+[PATHS]
+GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-service-fs-migration/
+
+[testbed]
+OVERLAY_TOPOLOGY = CLIQUE
+
+[ats]
+WAN_QUOTA_IN = 3932160
+WAN_QUOTA_OUT = 3932160
diff --git a/src/service/fs/test_gnunet_service_fs_p2p.c b/src/service/fs/test_gnunet_service_fs_p2p.c
new file mode 100644
index 000000000..2d1fbb788
--- /dev/null
+++ b/src/service/fs/test_gnunet_service_fs_p2p.c
@@ -0,0 +1,167 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2010, 2012 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+
+/**
+ * @file fs/test_gnunet_service_fs_p2p.c
+ * @brief test P2P routing using simple publish + download operation
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fs_test_lib.h"
+
+#define VERBOSE GNUNET_NO
+
+/**
+ * File-size we use for testing.
+ */
+#define FILESIZE (1024 * 1024 * 1)
+
+/**
+ * How long until we give up on the download?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
+
+#define NUM_DAEMONS 2
+
+#define SEED 42
+
+static const char *progname;
+
+static unsigned int anonymity_level;
+
+static struct GNUNET_TESTBED_Peer *daemons[NUM_DAEMONS];
+
+static int ok;
+
+static struct GNUNET_TIME_Absolute start_time;
+
+
+static void
+do_stop (void *cls)
+{
+  char *fn = cls;
+  struct GNUNET_TIME_Relative del;
+  char *fancy;
+
+  GNUNET_SCHEDULER_shutdown ();
+  if (0 ==
+      GNUNET_TIME_absolute_get_remaining (GNUNET_TIME_absolute_add (start_time,
+                                                                    TIMEOUT)).
+      rel_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Timeout during download, shutting down with error\n");
+    ok = 1;
+  }
+  else
+  {
+    del = GNUNET_TIME_absolute_get_duration (start_time);
+    if (0 == del.rel_value_us)
+      del.rel_value_us = 1;
+    fancy =
+      GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE)
+                                      * 1000000LL / del.rel_value_us);
+    fprintf (stdout,
+             "Download speed was %s/s\n",
+             fancy);
+    GNUNET_free (fancy);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Finished download, shutting down\n");
+  }
+  if (NULL != fn)
+  {
+    GNUNET_DISK_directory_remove (fn);
+    GNUNET_free (fn);
+  }
+}
+
+
+static void
+do_download (void *cls, const struct GNUNET_FS_Uri *uri,
+             const char *fn)
+{
+  if (NULL == uri)
+  {
+    GNUNET_SCHEDULER_shutdown ();
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Timeout during upload attempt, shutting down with error\n");
+    ok = 1;
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  start_time = GNUNET_TIME_absolute_get ();
+  GNUNET_FS_TEST_download (daemons[0], TIMEOUT,
+                           anonymity_level, SEED, uri,
+                           VERBOSE, &do_stop,
+                           (NULL == fn)
+                           ? NULL
+                           : GNUNET_strdup (fn));
+}
+
+
+static void
+do_publish (void *cls,
+            struct GNUNET_TESTBED_RunHandle *h,
+            unsigned int num_peers,
+            struct GNUNET_TESTBED_Peer **peers,
+            unsigned int links_succeeded,
+            unsigned int links_failed)
+{
+  unsigned int i;
+
+  if (NULL != strstr (progname, "cadet"))
+    anonymity_level = 0;
+  else
+    anonymity_level = 1;
+  GNUNET_assert (NUM_DAEMONS == num_peers);
+  for (i = 0; i < num_peers; i++)
+    daemons[i] = peers[i];
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n",
+              (unsigned long long) FILESIZE);
+  GNUNET_FS_TEST_publish (daemons[1], TIMEOUT,
+                          anonymity_level, GNUNET_NO,
+                          FILESIZE, SEED,
+                          VERBOSE, &do_download, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  const char *config;
+
+  progname = argv[0];
+  if (NULL != strstr (progname, "cadet"))
+    config = "test_gnunet_service_fs_p2p_cadet.conf";
+  else
+    config = "fs_test_lib_data.conf";
+  (void) GNUNET_TESTBED_test_run ("test-gnunet-service-fs-p2p",
+                                  config,
+                                  NUM_DAEMONS,
+                                  0, NULL, NULL,
+                                  &do_publish, NULL);
+  GNUNET_DISK_purge_cfg_dir (config,
+                             "GNUNET_TEST_HOME");
+  return ok;
+}
+
+
+/* end of test_gnunet_service_fs_p2p.c */
diff --git a/src/service/fs/test_gnunet_service_fs_p2p_cadet.conf b/src/service/fs/test_gnunet_service_fs_p2p_cadet.conf
new file mode 100644
index 000000000..12e106968
--- /dev/null
+++ b/src/service/fs/test_gnunet_service_fs_p2p_cadet.conf
@@ -0,0 +1,20 @@
+@INLINE@ fs_test_lib_data.conf
+
+[fs]
+# FIXME: this option needs to be set for the
+# testcase to truly work; however, as the code
+# is not finished, not setting the option should
+# allow the test to at least pass for now...
+DISABLE_ANON_TRANSFER = YES
+
+# Do we cache content from other nodes? (may improve anonymity)
+CONTENT_CACHING = NO
+
+# Do we send unsolicited data to other nodes if we have excess bandwidth?
+# (may improve anonymity, probably not a good idea if content_caching is NO)
+CONTENT_PUSHING = NO
+
+#PREFIX = valgrind
+
+[cadet]
+#PREFIX = valgrind
diff --git a/src/service/fs/test_pseudonym_data.conf b/src/service/fs/test_pseudonym_data.conf
new file mode 100644
index 000000000..5827721b8
--- /dev/null
+++ b/src/service/fs/test_pseudonym_data.conf
@@ -0,0 +1,6 @@
+# General settings
+[fs]
+
+[TESTING]
+WEAKRANDOM = YES
+