Large refactor in order to restore some sanity with respect to private defines used in headers

7 files changed, 614 insertions, 469 deletions

diff --git a/src/include/gnunet_bio_lib.h b/src/include/gnunet_bio_lib.h
index e165b9d90..6d42f554c 100644
--- a/src/include/gnunet_bio_lib.h
+++ b/src/include/gnunet_bio_lib.h
@@ -125,19 +125,6 @@ GNUNET_BIO_read_string (struct GNUNET_BIO_ReadHandle *h,
                         size_t max_length);
 
 
-/**
- * Read a metadata container.
- *
- * @param h handle to an open file
- * @param what describes what is being read (for error message creation)
- * @param result the buffer to store a pointer to the (allocated) metadata
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
- */
-int
-GNUNET_BIO_read_meta_data (struct GNUNET_BIO_ReadHandle *h,
-                           const char *what,
-                           struct GNUNET_CONTAINER_MetaData **result);
-
 
 /**
  * Read a float.
@@ -294,19 +281,6 @@ GNUNET_BIO_write_string (struct GNUNET_BIO_WriteHandle *h,
                          const char *s);
 
 
-/**
- * Write a metadata container.
- *
- * @param h the IO handle to write to
- * @param what what is being written (for error message creation)
- * @param m metadata to write
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-int
-GNUNET_BIO_write_meta_data (struct GNUNET_BIO_WriteHandle *h,
-                            const char *what,
-                            const struct GNUNET_CONTAINER_MetaData *m);
-
 
 /**
  * Write a float.
@@ -453,17 +427,6 @@ GNUNET_BIO_read_spec_string (const char *what,
                              size_t max_length);
 
 
-/**
- * Create the specification to read a metadata container.
- *
- * @param what describes what is being read (for error message creation)
- * @param result the buffer to store a pointer to the (allocated) metadata
- * @return the read spec
- */
-struct GNUNET_BIO_ReadSpec
-GNUNET_BIO_read_spec_meta_data (const char *what,
-                                struct GNUNET_CONTAINER_MetaData **result);
-
 
 /**
  * Create the specification to read an (u)int32_t.
@@ -610,18 +573,6 @@ GNUNET_BIO_write_spec_string (const char *what,
 
 
 /**
- * Create the specification to write a metadata container.
- *
- * @param what what is being written (for error message creation)
- * @param m metadata to write
- * @return the write spec
- */
-struct GNUNET_BIO_WriteSpec
-GNUNET_BIO_write_spec_meta_data (const char *what,
-                                 const struct GNUNET_CONTAINER_MetaData *m);
-
-
-/**
  * Create the specification to write an (u)int32_t.
  *
  * @param what describes what is being written (for error message creation)
diff --git a/src/include/gnunet_config.h.in b/src/include/gnunet_config.h.in
index 5b0d69a03..ce7c5a894 100644
--- a/src/include/gnunet_config.h.in
+++ b/src/include/gnunet_config.h.in
@@ -48,7 +48,4 @@
 /* Set to 1 if GnuTLS has DANE support */
 #define GNUNET_CURL_GNUTLS @curl_gnutls@
 
-/* Set to 1 is we have the byteswap header */
-#define GNUNET_HAVE_BYTESWAP_H @have_byteswap@
-
 #endif
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h
index 55774a209..1d39f813b 100644
--- a/src/include/gnunet_container_lib.h
+++ b/src/include/gnunet_container_lib.h
@@ -48,8 +48,6 @@
  *
  * @see [Documentation](https://gnunet.org/mdll-api)
  *
- * @defgroup metadata  Metadata
- * GNU libextractor key-value pairs
  *
  * @}
  */
@@ -102,114 +100,6 @@ char *
 GNUNET_decompress (const char *input, size_t input_size, size_t output_size);
 
 
-#if HAVE_EXTRACTOR_H
-
-#include <extractor.h>
-
-#else
-
-/* definitions from extractor.h we need for the build */
-
-/**
- * Enumeration defining various sources of keywords.  See also
- * http://dublincore.org/documents/1998/09/dces/
- */
-enum EXTRACTOR_MetaType
-{
-  EXTRACTOR_METATYPE_RESERVED = 0,
-  EXTRACTOR_METATYPE_MIMETYPE = 1,
-  EXTRACTOR_METATYPE_FILENAME = 2,
-  EXTRACTOR_METATYPE_COMMENT = 3,
-  EXTRACTOR_METATYPE_TITLE = 4,
-  EXTRACTOR_METATYPE_BOOK_TITLE = 5,
-  EXTRACTOR_METATYPE_JOURNAL_NAME = 8,
-  EXTRACTOR_METATYPE_AUTHOR_NAME = 13,
-  EXTRACTOR_METATYPE_PUBLICATION_DATE = 24,
-  EXTRACTOR_METATYPE_URL = 29,
-  EXTRACTOR_METATYPE_URI = 30,
-  EXTRACTOR_METATYPE_ISRC = 31,
-  EXTRACTOR_METATYPE_UNKNOWN = 45,
-  EXTRACTOR_METATYPE_DESCRIPTION = 46,
-  EXTRACTOR_METATYPE_KEYWORDS = 49,
-  EXTRACTOR_METATYPE_SUBJECT = 52,
-  EXTRACTOR_METATYPE_PACKAGE_NAME = 69,
-  EXTRACTOR_METATYPE_THUMBNAIL = 114,
-  EXTRACTOR_METATYPE_ALBUM = 129,
-  EXTRACTOR_METATYPE_ARTIST = 130,
-  EXTRACTOR_METATYPE_ORIGINAL_TITLE = 162,
-  EXTRACTOR_METATYPE_GNUNET_FULL_DATA = 174,
-  EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME = 180,
-};
-
-/**
- * Format in which the extracted meta data is presented.
- */
-enum EXTRACTOR_MetaFormat
-{
-  /**
-   * Format is unknown.
-   */
-  EXTRACTOR_METAFORMAT_UNKNOWN = 0,
-
-  /**
-   * 0-terminated, UTF-8 encoded string.  "data_len"
-   * is strlen(data)+1.
-   */
-  EXTRACTOR_METAFORMAT_UTF8 = 1,
-
-  /**
-   * Some kind of binary format, see given Mime type.
-   */
-  EXTRACTOR_METAFORMAT_BINARY = 2,
-
-  /**
-   * 0-terminated string.  The specific encoding is unknown.
-   * "data_len" is strlen (data)+1.
-   */
-  EXTRACTOR_METAFORMAT_C_STRING = 3
-};
-
-
-/**
- * Type of a function that libextractor calls for each
- * meta data item found.
- *
- * @param cls closure (user-defined)
- * @param plugin_name name of the plugin that produced this value;
- *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
- *        used in the main libextractor library and yielding
- *        meta data).
- * @param type libextractor-type describing the meta data
- * @param format basic format information about @a data
- * @param data_mime_type mime-type of @a data (not of the original file);
- *        can be NULL (if mime-type is not known)
- * @param data actual meta-data found
- * @param data_len number of bytes in @a data
- * @return 0 to continue extracting, 1 to abort
- */
-typedef int (*EXTRACTOR_MetaDataProcessor) (void *cls,
-                                            const char *plugin_name,
-                                            enum EXTRACTOR_MetaType type,
-                                            enum EXTRACTOR_MetaFormat format,
-                                            const char *data_mime_type,
-                                            const char *data,
-                                            size_t data_len);
-
-#endif
-
-#ifndef EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
-/* hack for LE < 0.6.3 */
-#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME 180
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#if 0 /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-
 /* ******************* bloomfilter ***************** */
 
 /**
@@ -435,285 +325,6 @@ GNUNET_CONTAINER_bloomfilter_resize (struct GNUNET_CONTAINER_BloomFilter *bf,
                                      unsigned int k);
 
 
-/* ****************** metadata ******************* */
-
-/**
- * @ingroup metadata
- * Meta data to associate with a file, directory or namespace.
- */
-struct GNUNET_CONTAINER_MetaData;
-
-
-/**
- * @ingroup metadata
- * Create a fresh meta data container.
- *
- * @return empty meta-data container
- */
-struct GNUNET_CONTAINER_MetaData *
-GNUNET_CONTAINER_meta_data_create (void);
-
-
-/**
- * @ingroup metadata
- * Duplicate a MetaData token.
- *
- * @param md what to duplicate
- * @return duplicate meta-data container
- */
-struct GNUNET_CONTAINER_MetaData *
-GNUNET_CONTAINER_meta_data_duplicate (
-  const struct GNUNET_CONTAINER_MetaData *md);
-
-
-/**
- * @ingroup metadata
- * Free meta data.
- *
- * @param md what to free
- */
-void
-GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md);
-
-
-/**
- * @ingroup metadata
- * Test if two MDs are equal. We consider them equal if
- * the meta types, formats and content match (we do not
- * include the mime types and plugins names in this
- * consideration).
- *
- * @param md1 first value to check
- * @param md2 other value to check
- * @return #GNUNET_YES if they are equal
- */
-int
-GNUNET_CONTAINER_meta_data_test_equal (
-  const struct GNUNET_CONTAINER_MetaData *md1,
-  const struct GNUNET_CONTAINER_MetaData *md2);
-
-
-/**
- * @ingroup metadata
- * Extend metadata.
- *
- * @param md metadata to extend
- * @param plugin_name name of the plugin that produced this value;
- *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
- *        used in the main libextractor library and yielding
- *        meta data).
- * @param type libextractor-type describing the meta data
- * @param format basic format information about data
- * @param data_mime_type mime-type of data (not of the original file);
- *        can be NULL (if mime-type is not known)
- * @param data actual meta-data found
- * @param data_size number of bytes in data
- * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists
- *         data_mime_type and plugin_name are not considered for "exists" checks
- */
-int
-GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
-                                   const char *plugin_name,
-                                   enum EXTRACTOR_MetaType type,
-                                   enum EXTRACTOR_MetaFormat format,
-                                   const char *data_mime_type,
-                                   const char *data,
-                                   size_t data_size);
-
-
-/**
- * @ingroup metadata
- * Extend metadata.  Merges the meta data from the second argument
- * into the first, discarding duplicate key-value pairs.
- *
- * @param md metadata to extend
- * @param in metadata to merge
- */
-void
-GNUNET_CONTAINER_meta_data_merge (struct GNUNET_CONTAINER_MetaData *md,
-                                  const struct GNUNET_CONTAINER_MetaData *in);
-
-
-/**
- * @ingroup metadata
- * Remove an item.
- *
- * @param md metadata to manipulate
- * @param type type of the item to remove
- * @param data specific value to remove, NULL to remove all
- *        entries of the given type
- * @param data_size number of bytes in data
- * @return #GNUNET_OK on success, #GNUNET_SYSERR if the item does not exist in md
- */
-int
-GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
-                                   enum EXTRACTOR_MetaType type,
-                                   const char *data,
-                                   size_t data_size);
-
-
-/**
- * @ingroup metadata
- * Remove all items in the container.
- *
- * @param md metadata to manipulate
- */
-void
-GNUNET_CONTAINER_meta_data_clear (struct GNUNET_CONTAINER_MetaData *md);
-
-
-/**
- * @ingroup metadata
- * Add the current time as the publication date
- * to the meta-data.
- *
- * @param md metadata to modify
- */
-void
-GNUNET_CONTAINER_meta_data_add_publication_date (
-  struct GNUNET_CONTAINER_MetaData *md);
-
-
-/**
- * @ingroup metadata
- * Iterate over MD entries.
- *
- * @param md metadata to inspect
- * @param iter function to call on each entry, return 0 to continue to iterate
- *             and 1 to abort iteration in this function (GNU libextractor API!)
- * @param iter_cls closure for @a iter
- * @return number of entries
- */
-int
-GNUNET_CONTAINER_meta_data_iterate (const struct GNUNET_CONTAINER_MetaData *md,
-                                    EXTRACTOR_MetaDataProcessor iter,
-                                    void *iter_cls);
-
-
-/**
- * @ingroup metadata
- * Get the first MD entry of the given type.  Caller
- * is responsible for freeing the return value.
- * Also, only meta data items that are strings (0-terminated)
- * are returned by this function.
- *
- * @param md metadata to inspect
- * @param type type to look for
- * @return NULL if no entry was found
- */
-char *
-GNUNET_CONTAINER_meta_data_get_by_type (
-  const struct GNUNET_CONTAINER_MetaData *md,
-  enum EXTRACTOR_MetaType type);
-
-
-/**
- * @ingroup metadata
- * Get the first matching MD entry of the given types. Caller is
- * responsible for freeing the return value.  Also, only meta data
- * items that are strings (0-terminated) are returned by this
- * function.
- *
- * @param md metadata to inspect
- * @param ... -1-terminated list of types
- * @return NULL if we do not have any such entry,
- *  otherwise client is responsible for freeing the value!
- */
-char *
-GNUNET_CONTAINER_meta_data_get_first_by_types (
-  const struct GNUNET_CONTAINER_MetaData *md,
-  ...);
-
-/**
- * @ingroup metadata
- * Get a thumbnail from the meta-data (if present).  Only matches meta
- * data with mime type "image" and binary format.
- *
- * @param md metadata to inspect
- * @param thumb will be set to the thumbnail data.  Must be
- *        freed by the caller!
- * @return number of bytes in thumbnail, 0 if not available
- */
-size_t
-GNUNET_CONTAINER_meta_data_get_thumbnail (
-  const struct GNUNET_CONTAINER_MetaData *md,
-  unsigned char **thumb);
-
-
-/**
- * @ingroup metadata
- * Options for metadata serialization.
- */
-enum GNUNET_CONTAINER_MetaDataSerializationOptions
-{
-  /**
-   * @ingroup metadata
-   * Serialize all of the data.
-   */
-  GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL = 0,
-
-  /**
-   * @ingroup metadata
-   * If not enough space is available, it is acceptable
-   * to only serialize some of the metadata.
-   */
-  GNUNET_CONTAINER_META_DATA_SERIALIZE_PART = 1,
-
-  /**
-   * @ingroup metadata
-   * Speed is of the essence, do not allow compression.
-   */
-  GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS = 2
-};
-
-
-/**
- * @ingroup metadata
- * Serialize meta-data to target.
- *
- * @param md metadata to serialize
- * @param target where to write the serialized metadata;
- *         *target can be NULL, in which case memory is allocated
- * @param max maximum number of bytes available
- * @param opt is it ok to just write SOME of the
- *        meta-data to match the size constraint,
- *        possibly discarding some data?
- * @return number of bytes written on success,
- *         -1 on error (typically: not enough
- *         space)
- */
-ssize_t
-GNUNET_CONTAINER_meta_data_serialize (
-  const struct GNUNET_CONTAINER_MetaData *md,
-  char **target,
-  size_t max,
-  enum GNUNET_CONTAINER_MetaDataSerializationOptions opt);
-
-
-/**
- * @ingroup metadata
- * Get the size of the full meta-data in serialized form.
- *
- * @param md metadata to inspect
- * @return number of bytes needed for serialization, -1 on error
- */
-ssize_t
-GNUNET_CONTAINER_meta_data_get_serialized_size (
-  const struct GNUNET_CONTAINER_MetaData *md);
-
-
-/**
- * @ingroup metadata
- * Deserialize meta-data.  Initializes md.
- *
- * @param input serialized meta-data.
- * @param size number of bytes available
- * @return MD on success, NULL on error (e.g.
- *         bad format)
- */
-struct GNUNET_CONTAINER_MetaData *
-GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size);
-
 
 /* ******************************* HashMap **************************** */
 
diff --git a/src/include/gnunet_extractor_compat.h b/src/include/gnunet_extractor_compat.h
new file mode 100644
index 000000000..d69525c9d
--- /dev/null
+++ b/src/include/gnunet_extractor_compat.h
@@ -0,0 +1,149 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2022 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @author Martin Schanzenbach
+ *
+ * @file
+ * libextractor compatibility insanity helper header
+ *
+ * @{
+ */
+#ifndef GNUNET_EXTRACTOR_COMPAT_H
+#define GNUNET_EXTRACTOR_COMPAT_H
+
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#if HAVE_EXTRACTOR_H
+
+#include <extractor.h>
+
+#else
+
+/* definitions from extractor.h we need for the build */
+
+/**
+ * Enumeration defining various sources of keywords.  See also
+ * http://dublincore.org/documents/1998/09/dces/
+ */
+enum EXTRACTOR_MetaType
+{
+  EXTRACTOR_METATYPE_RESERVED = 0,
+  EXTRACTOR_METATYPE_MIMETYPE = 1,
+  EXTRACTOR_METATYPE_FILENAME = 2,
+  EXTRACTOR_METATYPE_COMMENT = 3,
+  EXTRACTOR_METATYPE_TITLE = 4,
+  EXTRACTOR_METATYPE_BOOK_TITLE = 5,
+  EXTRACTOR_METATYPE_JOURNAL_NAME = 8,
+  EXTRACTOR_METATYPE_AUTHOR_NAME = 13,
+  EXTRACTOR_METATYPE_PUBLICATION_DATE = 24,
+  EXTRACTOR_METATYPE_URL = 29,
+  EXTRACTOR_METATYPE_URI = 30,
+  EXTRACTOR_METATYPE_ISRC = 31,
+  EXTRACTOR_METATYPE_UNKNOWN = 45,
+  EXTRACTOR_METATYPE_DESCRIPTION = 46,
+  EXTRACTOR_METATYPE_KEYWORDS = 49,
+  EXTRACTOR_METATYPE_SUBJECT = 52,
+  EXTRACTOR_METATYPE_PACKAGE_NAME = 69,
+  EXTRACTOR_METATYPE_THUMBNAIL = 114,
+  EXTRACTOR_METATYPE_ALBUM = 129,
+  EXTRACTOR_METATYPE_ARTIST = 130,
+  EXTRACTOR_METATYPE_ORIGINAL_TITLE = 162,
+  EXTRACTOR_METATYPE_GNUNET_FULL_DATA = 174,
+  EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME = 180,
+};
+
+/**
+ * Format in which the extracted meta data is presented.
+ */
+enum EXTRACTOR_MetaFormat
+{
+  /**
+   * Format is unknown.
+   */
+  EXTRACTOR_METAFORMAT_UNKNOWN = 0,
+
+  /**
+   * 0-terminated, UTF-8 encoded string.  "data_len"
+   * is strlen(data)+1.
+   */
+  EXTRACTOR_METAFORMAT_UTF8 = 1,
+
+  /**
+   * Some kind of binary format, see given Mime type.
+   */
+  EXTRACTOR_METAFORMAT_BINARY = 2,
+
+  /**
+   * 0-terminated string.  The specific encoding is unknown.
+   * "data_len" is strlen (data)+1.
+   */
+  EXTRACTOR_METAFORMAT_C_STRING = 3
+};
+
+
+/**
+ * Type of a function that libextractor calls for each
+ * meta data item found.
+ *
+ * @param cls closure (user-defined)
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about @a data
+ * @param data_mime_type mime-type of @a data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in @a data
+ * @return 0 to continue extracting, 1 to abort
+ */
+typedef int (*EXTRACTOR_MetaDataProcessor) (void *cls,
+                                            const char *plugin_name,
+                                            enum EXTRACTOR_MetaType type,
+                                            enum EXTRACTOR_MetaFormat format,
+                                            const char *data_mime_type,
+                                            const char *data,
+                                            size_t data_len);
+
+#endif
+
+#ifndef EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
+/* hack for LE < 0.6.3 */
+#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME 180
+#endif
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+/** @} */  /* end of group */
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index 6b46f1553..53ca6a9ec 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -37,6 +37,7 @@
 #define GNUNET_FS_LIB_H
 
 
+#include "gnunet_common.h"
 #include "gnunet_util_lib.h"
 
 #ifdef __cplusplus
@@ -92,6 +93,12 @@ extern "C"
  */
 struct GNUNET_FS_Uri;
 
+/**
+ * @ingroup metadata
+ * Meta data to associate with a file, directory or namespace.
+ */
+struct GNUNET_FS_MetaData;
+
 
 /**
  * Iterator over keywords
@@ -436,7 +443,7 @@ GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri);
  * @return NULL on error, otherwise a KSK URI
  */
 struct GNUNET_FS_Uri *
-GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
+GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_FS_MetaData
                                          *md);
 
 
@@ -472,7 +479,7 @@ GNUNET_FS_GETOPT_METADATA (char shortName,
                            const char *name,
                            const char *argumentHelp,
                            const char *description,
-                           struct GNUNET_CONTAINER_MetaData **meta);
+                           struct GNUNET_FS_MetaData **meta);
 
 /**
  * Command-line option parser function that allows the user to specify
@@ -481,7 +488,7 @@ GNUNET_FS_GETOPT_METADATA (char shortName,
  * the metadata must be passed as the "scls" argument.
  *
  * @param ctx command line processor context
- * @param scls must be of type `struct GNUNET_CONTAINER_MetaData **`
+ * @param scls must be of type `struct GNUNET_FS_MetaData **`
  * @param option name of the option (typically 'k')
  * @param value command line argument given
  * @return #GNUNET_OK on success
@@ -1099,7 +1106,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Known metadata for the download.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
         } start;
 
         /**
@@ -1111,7 +1118,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Known metadata for the download.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * Error message, NULL if we have not encountered any error yet.
@@ -1196,7 +1203,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the search result.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * URI for the search result.
@@ -1224,7 +1231,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the search result.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * URI for the search result.
@@ -1271,7 +1278,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the search result.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * URI for the search result.
@@ -1323,7 +1330,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the search result.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * URI for the search result.
@@ -1351,7 +1358,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the search result.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * URI for the search result.
@@ -1406,7 +1413,7 @@ struct GNUNET_FS_ProgressInfo
           /**
            * Metadata for the namespace.
            */
-          const struct GNUNET_CONTAINER_MetaData *meta;
+          const struct GNUNET_FS_MetaData *meta;
 
           /**
            * Public key of the namespace.
@@ -1706,7 +1713,7 @@ typedef int
 (*GNUNET_FS_FileInformationProcessor) (void *cls,
                                        struct GNUNET_FS_FileInformation *fi,
                                        uint64_t length,
-                                       struct GNUNET_CONTAINER_MetaData *meta,
+                                       struct GNUNET_FS_MetaData *meta,
                                        struct GNUNET_FS_Uri **uri,
                                        struct GNUNET_FS_BlockOptions *bo,
                                        int *do_index,
@@ -1771,7 +1778,7 @@ GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h,
                                              const struct
                                              GNUNET_FS_Uri *keywords,
                                              const struct
-                                             GNUNET_CONTAINER_MetaData *meta,
+                                             GNUNET_FS_MetaData *meta,
                                              int do_index,
                                              const struct
                                              GNUNET_FS_BlockOptions *bo);
@@ -1801,7 +1808,7 @@ GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h,
                                              const struct
                                              GNUNET_FS_Uri *keywords,
                                              const struct
-                                             GNUNET_CONTAINER_MetaData *meta,
+                                             GNUNET_FS_MetaData *meta,
                                              int do_index,
                                              const struct
                                              GNUNET_FS_BlockOptions *bo);
@@ -1860,7 +1867,7 @@ GNUNET_FS_file_information_create_from_reader (struct GNUNET_FS_Handle *h,
                                                const struct GNUNET_FS_Uri
                                                *keywords,
                                                const struct
-                                               GNUNET_CONTAINER_MetaData *meta,
+                                               GNUNET_FS_MetaData *meta,
                                                int do_index,
                                                const struct
                                                GNUNET_FS_BlockOptions *bo);
@@ -1884,7 +1891,7 @@ GNUNET_FS_file_information_create_empty_directory (struct GNUNET_FS_Handle *h,
                                                    const struct GNUNET_FS_Uri
                                                    *keywords,
                                                    const struct
-                                                   GNUNET_CONTAINER_MetaData
+                                                   GNUNET_FS_MetaData
                                                    *meta,
                                                    const struct
                                                    GNUNET_FS_BlockOptions *bo,
@@ -2043,7 +2050,7 @@ struct GNUNET_FS_PublishKskContext;
 struct GNUNET_FS_PublishKskContext *
 GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
                        const struct GNUNET_FS_Uri *ksk_uri,
-                       const struct GNUNET_CONTAINER_MetaData *meta,
+                       const struct GNUNET_FS_MetaData *meta,
                        const struct GNUNET_FS_Uri *uri,
                        const struct GNUNET_FS_BlockOptions *bo,
                        enum GNUNET_FS_PublishOptions options,
@@ -2085,7 +2092,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
                        const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
                        const char *identifier,
                        const char *update,
-                       const struct GNUNET_CONTAINER_MetaData *meta,
+                       const struct GNUNET_FS_MetaData *meta,
                        const struct GNUNET_FS_Uri *uri,
                        const struct GNUNET_FS_BlockOptions *bo,
                        enum GNUNET_FS_PublishOptions options,
@@ -2181,7 +2188,7 @@ typedef void (*GNUNET_FS_IdentifierProcessor) (void *cls,
                                                const struct
                                                GNUNET_FS_Uri *last_uri,
                                                const struct
-                                               GNUNET_CONTAINER_MetaData *
+                                               GNUNET_FS_MetaData *
                                                last_meta,
                                                const char *next_id);
 
@@ -2291,7 +2298,7 @@ GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc);
 struct GNUNET_FS_SearchResult *
 GNUNET_FS_probe (struct GNUNET_FS_Handle *h,
                  const struct GNUNET_FS_Uri *uri,
-                 const struct GNUNET_CONTAINER_MetaData *meta,
+                 const struct GNUNET_FS_MetaData *meta,
                  void *client_info,
                  uint32_t anonymity);
 
@@ -2379,7 +2386,7 @@ enum GNUNET_FS_DownloadOptions
 struct GNUNET_FS_DownloadContext *
 GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
                           const struct GNUNET_FS_Uri *uri,
-                          const struct GNUNET_CONTAINER_MetaData *meta,
+                          const struct GNUNET_FS_MetaData *meta,
                           const char *filename, const char *tempname,
                           uint64_t offset, uint64_t length, uint32_t anonymity,
                           enum GNUNET_FS_DownloadOptions options, void *cctx,
@@ -2473,7 +2480,7 @@ GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc);
  *  we have no mime-type information (treat as #GNUNET_NO)
  */
 int
-GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData
+GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_FS_MetaData
                                         *md);
 
 
@@ -2484,7 +2491,7 @@ GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData
  * @param md metadata to add mimetype to
  */
 void
-GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md);
+GNUNET_FS_meta_data_make_directory (struct GNUNET_FS_MetaData *md);
 
 
 /**
@@ -2495,7 +2502,7 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md);
  */
 char *
 GNUNET_FS_meta_data_suggest_filename (const struct
-                                      GNUNET_CONTAINER_MetaData *md);
+                                      GNUNET_FS_MetaData *md);
 
 
 /**
@@ -2518,7 +2525,7 @@ typedef void (*GNUNET_FS_DirectoryEntryProcessor) (void *cls,
                                                    const struct GNUNET_FS_Uri *
                                                    uri,
                                                    const struct
-                                                   GNUNET_CONTAINER_MetaData *
+                                                   GNUNET_FS_MetaData *
                                                    meta, size_t length,
                                                    const void *data);
 
@@ -2561,7 +2568,7 @@ struct GNUNET_FS_DirectoryBuilder;
  * @param mdir metadata for the directory
  */
 struct GNUNET_FS_DirectoryBuilder *
-GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData
+GNUNET_FS_directory_builder_create (const struct GNUNET_FS_MetaData
                                     *mdir);
 
 
@@ -2578,7 +2585,7 @@ GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData
 void
 GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
                                  const struct GNUNET_FS_Uri *uri,
-                                 const struct GNUNET_CONTAINER_MetaData *md,
+                                 const struct GNUNET_FS_MetaData *md,
                                  const void *data);
 
 
@@ -2693,7 +2700,7 @@ struct GNUNET_FS_ShareTreeItem
   /**
    * Metadata for this file or directory
    */
-  struct GNUNET_CONTAINER_MetaData *meta;
+  struct GNUNET_FS_MetaData *meta;
 
   /**
    * Keywords for this file or directory (derived from metadata).
@@ -2782,6 +2789,336 @@ GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel);
 void
 GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel);
 
+/**
+ * Meta data processing
+ * @defgroup metadata  Metadata
+ * GNU libextractor key-value pairs
+ */
+
+
+/* ****************** metadata ******************* */
+
+
+/**
+ * @ingroup metadata
+ * Create a fresh meta data container.
+ *
+ * @return empty meta-data container
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_create (void);
+
+
+/**
+ * @ingroup metadata
+ * Duplicate a MetaData token.
+ *
+ * @param md what to duplicate
+ * @return duplicate meta-data container
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_duplicate (
+  const struct GNUNET_FS_MetaData *md);
+
+
+/**
+ * @ingroup metadata
+ * Free meta data.
+ *
+ * @param md what to free
+ */
+void
+GNUNET_FS_meta_data_destroy (struct GNUNET_FS_MetaData *md);
+
+
+/**
+ * @ingroup metadata
+ * Test if two MDs are equal. We consider them equal if
+ * the meta types, formats and content match (we do not
+ * include the mime types and plugins names in this
+ * consideration).
+ *
+ * @param md1 first value to check
+ * @param md2 other value to check
+ * @return #GNUNET_YES if they are equal
+ */
+int
+GNUNET_FS_meta_data_test_equal (
+  const struct GNUNET_FS_MetaData *md1,
+  const struct GNUNET_FS_MetaData *md2);
+
+
+/**
+ * @ingroup metadata
+ * Extend metadata.
+ *
+ * @param md metadata to extend
+ * @param plugin_name name of the plugin that produced this value;
+ *        special values can be used (e.g. '&lt;zlib&gt;' for zlib being
+ *        used in the main libextractor library and yielding
+ *        meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ *        can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_size number of bytes in data
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists
+ *         data_mime_type and plugin_name are not considered for "exists" checks
+ */
+int
+GNUNET_FS_meta_data_insert (struct GNUNET_FS_MetaData *md,
+                                   const char *plugin_name,
+                                   enum EXTRACTOR_MetaType type,
+                                   enum EXTRACTOR_MetaFormat format,
+                                   const char *data_mime_type,
+                                   const char *data,
+                                   size_t data_size);
+
+
+/**
+ * @ingroup metadata
+ * Extend metadata.  Merges the meta data from the second argument
+ * into the first, discarding duplicate key-value pairs.
+ *
+ * @param md metadata to extend
+ * @param in metadata to merge
+ */
+void
+GNUNET_FS_meta_data_merge (struct GNUNET_FS_MetaData *md,
+                                  const struct GNUNET_FS_MetaData *in);
+
+
+/**
+ * @ingroup metadata
+ * Remove an item.
+ *
+ * @param md metadata to manipulate
+ * @param type type of the item to remove
+ * @param data specific value to remove, NULL to remove all
+ *        entries of the given type
+ * @param data_size number of bytes in data
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if the item does not exist in md
+ */
+int
+GNUNET_FS_meta_data_delete (struct GNUNET_FS_MetaData *md,
+                                   enum EXTRACTOR_MetaType type,
+                                   const char *data,
+                                   size_t data_size);
+
+
+/**
+ * @ingroup metadata
+ * Remove all items in the container.
+ *
+ * @param md metadata to manipulate
+ */
+void
+GNUNET_FS_meta_data_clear (struct GNUNET_FS_MetaData *md);
+
+
+/**
+ * @ingroup metadata
+ * Add the current time as the publication date
+ * to the meta-data.
+ *
+ * @param md metadata to modify
+ */
+void
+GNUNET_FS_meta_data_add_publication_date (
+  struct GNUNET_FS_MetaData *md);
+
+
+/**
+ * @ingroup metadata
+ * Iterate over MD entries.
+ *
+ * @param md metadata to inspect
+ * @param iter function to call on each entry, return 0 to continue to iterate
+ *             and 1 to abort iteration in this function (GNU libextractor API!)
+ * @param iter_cls closure for @a iter
+ * @return number of entries
+ */
+int
+GNUNET_FS_meta_data_iterate (const struct GNUNET_FS_MetaData *md,
+                                    EXTRACTOR_MetaDataProcessor iter,
+                                    void *iter_cls);
+
+
+/**
+ * @ingroup metadata
+ * Get the first MD entry of the given type.  Caller
+ * is responsible for freeing the return value.
+ * Also, only meta data items that are strings (0-terminated)
+ * are returned by this function.
+ *
+ * @param md metadata to inspect
+ * @param type type to look for
+ * @return NULL if no entry was found
+ */
+char *
+GNUNET_FS_meta_data_get_by_type (
+  const struct GNUNET_FS_MetaData *md,
+  enum EXTRACTOR_MetaType type);
+
+
+/**
+ * @ingroup metadata
+ * Get the first matching MD entry of the given types. Caller is
+ * responsible for freeing the return value.  Also, only meta data
+ * items that are strings (0-terminated) are returned by this
+ * function.
+ *
+ * @param md metadata to inspect
+ * @param ... -1-terminated list of types
+ * @return NULL if we do not have any such entry,
+ *  otherwise client is responsible for freeing the value!
+ */
+char *
+GNUNET_FS_meta_data_get_first_by_types (
+  const struct GNUNET_FS_MetaData *md,
+  ...);
+
+/**
+ * @ingroup metadata
+ * Get a thumbnail from the meta-data (if present).  Only matches meta
+ * data with mime type "image" and binary format.
+ *
+ * @param md metadata to inspect
+ * @param thumb will be set to the thumbnail data.  Must be
+ *        freed by the caller!
+ * @return number of bytes in thumbnail, 0 if not available
+ */
+size_t
+GNUNET_FS_meta_data_get_thumbnail (
+  const struct GNUNET_FS_MetaData *md,
+  unsigned char **thumb);
+
+
+/**
+ * @ingroup metadata
+ * Options for metadata serialization.
+ */
+enum GNUNET_FS_MetaDataSerializationOptions
+{
+  /**
+   * @ingroup metadata
+   * Serialize all of the data.
+   */
+  GNUNET_FS_META_DATA_SERIALIZE_FULL = 0,
+
+  /**
+   * @ingroup metadata
+   * If not enough space is available, it is acceptable
+   * to only serialize some of the metadata.
+   */
+  GNUNET_FS_META_DATA_SERIALIZE_PART = 1,
+
+  /**
+   * @ingroup metadata
+   * Speed is of the essence, do not allow compression.
+   */
+  GNUNET_FS_META_DATA_SERIALIZE_NO_COMPRESS = 2
+};
+
+
+/**
+ * @ingroup metadata
+ * Serialize meta-data to target.
+ *
+ * @param md metadata to serialize
+ * @param target where to write the serialized metadata;
+ *         *target can be NULL, in which case memory is allocated
+ * @param max maximum number of bytes available
+ * @param opt is it ok to just write SOME of the
+ *        meta-data to match the size constraint,
+ *        possibly discarding some data?
+ * @return number of bytes written on success,
+ *         -1 on error (typically: not enough
+ *         space)
+ */
+ssize_t
+GNUNET_FS_meta_data_serialize (
+  const struct GNUNET_FS_MetaData *md,
+  char **target,
+  size_t max,
+  enum GNUNET_FS_MetaDataSerializationOptions opt);
+
+
+/**
+ * @ingroup metadata
+ * Get the size of the full meta-data in serialized form.
+ *
+ * @param md metadata to inspect
+ * @return number of bytes needed for serialization, -1 on error
+ */
+ssize_t
+GNUNET_FS_meta_data_get_serialized_size (
+  const struct GNUNET_FS_MetaData *md);
+
+
+/**
+ * @ingroup metadata
+ * Deserialize meta-data.  Initializes md.
+ *
+ * @param input serialized meta-data.
+ * @param size number of bytes available
+ * @return MD on success, NULL on error (e.g.
+ *         bad format)
+ */
+struct GNUNET_FS_MetaData *
+GNUNET_FS_meta_data_deserialize (const char *input, size_t size);
+
+/**
+ * Write a metadata container.
+ *
+ * @param h the IO handle to write to
+ * @param what what is being written (for error message creation)
+ * @param m metadata to write
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+enum GNUNET_GenericReturnValue
+GNUNET_FS_write_meta_data (struct GNUNET_BIO_WriteHandle *h,
+                            const char *what,
+                            const struct GNUNET_FS_MetaData *m);
+
+/**
+ * Create the specification to read a metadata container.
+ *
+ * @param what describes what is being read (for error message creation)
+ * @param result the buffer to store a pointer to the (allocated) metadata
+ * @return the read spec
+ */
+struct GNUNET_BIO_ReadSpec
+GNUNET_FS_read_spec_meta_data (const char *what,
+                                struct GNUNET_FS_MetaData **result);
+
+
+
+/**
+ * Create the specification to write a metadata container.
+ *
+ * @param what what is being written (for error message creation)
+ * @param m metadata to write
+ * @return the write spec
+ */
+struct GNUNET_BIO_WriteSpec
+GNUNET_FS_write_spec_meta_data (const char *what,
+                                 const struct GNUNET_FS_MetaData *m);
+
+/**
+ * Read a metadata container.
+ *
+ * @param h handle to an open file
+ * @param what describes what is being read (for error message creation)
+ * @param result the buffer to store a pointer to the (allocated) metadata
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
+ */
+enum GNUNET_GenericReturnValue
+GNUNET_FS_read_meta_data (struct GNUNET_BIO_ReadHandle *h,
+                           const char *what,
+                           struct GNUNET_FS_MetaData **result);
+
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_mysql_compat.h b/src/include/gnunet_mysql_compat.h
new file mode 100644
index 000000000..bb1e8de8a
--- /dev/null
+++ b/src/include/gnunet_mysql_compat.h
@@ -0,0 +1,63 @@
+/*
+     This file is part of GNUnet
+     Copyright (C) 2022 GNUnet e.V.
+
+     GNUnet is free software: you can redistribute it and/or modify it
+     under the terms of the GNU Affero General Public License as published
+     by the Free Software Foundation, either version 3 of the License,
+     or (at your option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @author Martin Schanzenbach
+ *
+ * @file
+ * MySQL/MariaDB compatibility insanity helper header
+ *
+ * @defgroup mysql  MySQL library
+ * Helper library to access a MySQL database.
+ * @{
+ */
+#ifndef GNUNET_MYSQL_COMPAT_H
+#define GNUNET_MYSQL_COMPAT_H
+
+
+#include <mysql/mysql.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#ifndef LIBMARIADB
+#if MYSQL_VERSION_ID < 80000
+#define MYSQL_BOOL bool;
+#else
+#define MYSQL_BOOL my_bool; //MySQL < 8 wants this
+#endif
+#else
+#define MYSQL_BOOL my_bool //MariaDB still uses my_bool
+#endif
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+/** @} */  /* end of group */
diff --git a/src/include/gnunet_util_lib.h b/src/include/gnunet_util_lib.h
index 52fa5c373..337552a0c 100644
--- a/src/include/gnunet_util_lib.h
+++ b/src/include/gnunet_util_lib.h
@@ -92,6 +92,43 @@ extern "C"
 #include "gnunet_child_management_lib.h"
 #include "gnunet_error_codes.h"
 
+/**
+ * Try to compress the given block of data using libz.  Only returns
+ * the compressed block if compression worked and the new block is
+ * actually smaller.  Decompress using #GNUNET_decompress().
+ *
+ * @param data block to compress; if compression
+ *        resulted in a smaller block, the first
+ *        bytes of data are updated to the compressed
+ *        data
+ * @param old_size number of bytes in data
+ * @param[out] result set to the compressed data, if compression worked
+ * @param[out] new_size set to size of result, if compression worked
+ * @return #GNUNET_YES if compression reduce the size,
+ *         #GNUNET_NO if compression did not help
+ */
+int
+GNUNET_try_compression (const char *data,
+                        size_t old_size,
+                        char **result,
+                        size_t *new_size);
+
+/**
+ * Decompress input, return the decompressed data as output.  Dual to
+ * #GNUNET_try_compression(). Caller must set @a output_size to the
+ * number of bytes that were originally compressed.
+ *
+ * @param input compressed data
+ * @param input_size number of bytes in input
+ * @param output_size expected size of the output
+ * @return NULL on error, buffer of @a output_size decompressed bytes otherwise
+ */
+char *
+GNUNET_decompress (const char *input,
+                   size_t input_size,
+                   size_t output_size);
+
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif