splitting of set intersection functionality from set service (not yet finished, FTBFS)

3 files changed, 451 insertions, 16 deletions

diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index c3fcde0b9..e9a2b1c0e 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -1713,80 +1713,146 @@ extern "C" {
  * Demand the whole element from the other
  * peer, given only the hash code.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL 565
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL 559
 
 /**
  * Demand the whole element from the other
  * peer, given only the hash code.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND 566
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND 560
 
 /**
  * Tell the other peer to send us a list of
  * hashes that match an IBF key.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY 567
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY 561
 
 /**
  * Tell the other peer which hashes match a
  * given IBF key.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER 568
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER 562
 
 /**
  * Request a set union operation from a remote peer.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST 581
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST 563
 
 /**
  * Strata estimator.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_SE 582
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_SE 564
 
 /**
  * Invertible bloom filter.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_IBF 583
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_IBF 565
 
 /**
  * Actual set elements.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS 584
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS 566
 
 /**
  * Requests for the elements with the given hashes.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENT_REQUESTS 585
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENT_REQUESTS 567
 
 /**
  * Set operation is done.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_DONE 586
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_DONE 568
 
 /**
  * Compressed strata estimator.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_SEC 590
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_SEC 569
 
 /**
  * Request all missing elements from the other peer,
  * based on their sets and the elements we previously sent
  * with #GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE 597
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE 570
 
 /**
  * Send a set element, not as response to a demand but because
  * we're sending the full set.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT 598
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT 571
 
 /**
  * Request all missing elements from the other peer,
  * based on their sets and the elements we previously sent
  * with #GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS.
  */
-#define GNUNET_MESSAGE_TYPE_SETU_P2P_OVER 599
+#define GNUNET_MESSAGE_TYPE_SETU_P2P_OVER 572
+
+
+/*******************************************************************************
+ * SETI message types
+ ******************************************************************************/
+
+
+/**
+ * Cancel a set operation
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_CANCEL 580
+
+/**
+ * Add element to set.
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_ADD 581
+
+/**
+ * Create a new local set
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_CREATE 582
+
+/**
+ * Handle result message from operation
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_RESULT 583
+
+/**
+ * Evaluate a set operation
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_EVALUATE 584
+
+/**
+ * Listen for operation requests
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_LISTEN 585
+
+/**
+ * Reject a set request.
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_REJECT 586
+
+/**
+ * Accept an incoming set request
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_ACCEPT 587
+
+/**
+ * Notify the client of an incoming request from a remote peer
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_REQUEST 588
+
+/**
+ * Information about the element count for intersection
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_P2P_ELEMENT_INFO 591
+
+/**
+ * Bloom filter message for intersection exchange started by Bob.
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_P2P_BF 592
+
+/**
+ * Intersection operation is done.
+ */
+#define GNUNET_MESSAGE_TYPE_SETI_P2P_DONE 593
 
 
 /*******************************************************************************
diff --git a/src/include/gnunet_seti_service.h b/src/include/gnunet_seti_service.h
new file mode 100644
index 000000000..c0b6f41a5
--- /dev/null
+++ b/src/include/gnunet_seti_service.h
@@ -0,0 +1,369 @@
+/*
+      This file is part of GNUnet
+      Copyright (C) 2013, 2014, 2020 GNUnet e.V.
+
+      GNUnet is free software: you can redistribute it and/or modify it
+      under the terms of the GNU Affero General Public License as published
+      by the Free Software Foundation, either version 3 of the License,
+      or (at your option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      Affero General Public License for more details.
+
+      You should have received a copy of the GNU Affero General Public License
+      along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+     SPDX-License-Identifier: AGPL3.0-or-later
+ */
+/**
+ * @author Florian Dold
+ * @author Christian Grothoff
+ *
+ * @file
+ * Two-peer set intersection operations
+ *
+ * @defgroup set  Set intersection service
+ * Two-peer set operations
+ *
+ * @{
+ */
+
+#ifndef GNUNET_SETI_SERVICE_H
+#define GNUNET_SETI_SERVICE_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#if 0                           /* keep Emacsens' auto-indent happy */
+}
+#endif
+#endif
+
+#include "gnunet_common.h"
+#include "gnunet_time_lib.h"
+#include "gnunet_configuration_lib.h"
+
+
+/**
+ * Maximum size of a context message for set operation requests.
+ */
+#define GNUNET_SETI_CONTEXT_MESSAGE_MAX_SIZE ((1 << 16) - 1024)
+
+/**
+ * Opaque handle to a set.
+ */
+struct GNUNET_SETI_Handle;
+
+/**
+ * Opaque handle to a set operation request from another peer.
+ */
+struct GNUNET_SETI_Request;
+
+/**
+ * Opaque handle to a listen operation.
+ */
+struct GNUNET_SETI_ListenHandle;
+
+/**
+ * Opaque handle to a set operation.
+ */
+struct GNUNET_SETI_OperationHandle;
+
+
+/**
+ * Status for the result callback
+ */
+enum GNUNET_SETI_Status
+{
+
+  /**
+   * Element should be added to the result set of the local peer, i.e. the
+   * element is in the intersection.
+   */
+  GNUNET_SETI_STATUS_ADD_LOCAL,
+
+  /**
+   * Element should be delete from the result set of the local peer, i.e. the
+   * local peer is having an element that is not in the intersection.
+   */
+  GNUNET_SETI_STATUS_DEL_LOCAL,
+
+  /**
+   * The other peer refused to do the operation with us, or something went
+   * wrong.
+   */
+  GNUNET_SETI_STATUS_FAILURE,
+
+  /**
+   * Success, all elements have been sent (and received).
+   */
+  GNUNET_SETI_STATUS_DONE
+};
+
+
+/**
+ * Element stored in a set.
+ */
+struct GNUNET_SETI_Element
+{
+  /**
+   * Number of bytes in the buffer pointed to by data.
+   */
+  uint16_t size;
+
+  /**
+   * Application-specific element type.
+   */
+  uint16_t element_type;
+
+  /**
+   * Actual data of the element
+   */
+  const void *data;
+};
+
+
+/**
+ * Possible options to pass to a set operation.
+ *
+ * Used as tag for struct #GNUNET_SETI_Option.
+ */
+enum GNUNET_SETI_OptionType
+{
+  /**
+   * List terminator.
+   */
+  GNUNET_SETI_OPTION_END = 0,
+
+  /**
+   * Return the elements remaining in the intersection
+   * (#GNUNET_SETI_STATUS_ADD_LOCAL). If not given, the default is to return a
+   * list of the elements to be removed (#GNUNET_SETI_STATUS_DEL_LOCAL).
+   */
+  GNUNET_SETI_OPTION_RETURN_INTERSECTION = 1,
+};
+
+
+/**
+ * Option for set operations.
+ */
+struct GNUNET_SETI_Option
+{
+  /**
+   * Type of the option.
+   */
+  enum GNUNET_SETI_OptionType type;
+
+  /**
+   * Value for the option, only used with some options.
+   */
+  union
+  {
+    uint64_t num;
+  } v;
+};
+
+
+/**
+ * Callback for set union operation results. Called for each element
+ * in the result set.
+ *
+ * @param cls closure
+ * @param element a result element, only valid if status is #GNUNET_SETI_STATUS_OK
+ * @param current_size current set size
+ * @param status see `enum GNUNET_SETI_Status`
+ */
+typedef void
+(*GNUNET_SETI_ResultIterator) (void *cls,
+                               const struct GNUNET_SETI_Element *element,
+                               uint64_t current_size,
+                               enum GNUNET_SETI_Status status);
+
+
+/**
+ * Called when another peer wants to do a set operation with the
+ * local peer. If a listen error occurs, the @a request is NULL.
+ *
+ * @param cls closure
+ * @param other_peer the other peer
+ * @param context_msg message with application specific information from
+ *        the other peer
+ * @param request request from the other peer (never NULL), use GNUNET_SETI_accept()
+ *        to accept it, otherwise the request will be refused
+ *        Note that we can't just return value from the listen callback,
+ *        as it is also necessary to specify the set we want to do the
+ *        operation with, whith sometimes can be derived from the context
+ *        message. It's necessary to specify the timeout.
+ */
+typedef void
+(*GNUNET_SETI_ListenCallback) (void *cls,
+                               const struct GNUNET_PeerIdentity *other_peer,
+                               const struct GNUNET_MessageHeader *context_msg,
+                               struct GNUNET_SETI_Request *request);
+
+
+/**
+ * Create an empty set, supporting the specified operation.
+ *
+ * @param cfg configuration to use for connecting to the
+ *        set service
+ * @return a handle to the set
+ */
+struct GNUNET_SETI_Handle *
+GNUNET_SETI_create (const struct GNUNET_CONFIGURATION_Handle *cfg);
+
+
+/**
+ * Add an element to the given set.
+ *
+ * @param set set to add element to
+ * @param element element to add to the set
+ * @param cb function to call when finished, can be NULL
+ * @param cb_cls closure for @a cb
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
+ *         set is invalid (e.g. the set service crashed)
+ */
+int
+GNUNET_SETI_add_element (struct GNUNET_SETI_Handle *set,
+                         const struct GNUNET_SETI_Element *element,
+                         GNUNET_SCHEDULER_TaskCallback cb,
+                         void *cb_cls);
+
+
+/**
+ * Destroy the set handle, and free all associated resources.  Operations may
+ * still be pending when a set is destroyed (and will be allowed to complete).
+ *
+ * @param set set to destroy
+ */
+void
+GNUNET_SETI_destroy (struct GNUNET_SETI_Handle *set);
+
+
+/**
+ * Prepare a set operation to be evaluated with another peer.  The evaluation
+ * will not start until the client provides a local set with
+ * GNUNET_SETI_commit().
+ *
+ * @param other_peer peer with the other set
+ * @param app_id hash for the application using the set
+ * @param context_msg additional information for the request
+ * @param options options to use when processing the request
+ * @param result_cb called on error or success
+ * @param result_cls closure for @a result_cb
+ * @return a handle to cancel the operation
+ */
+struct GNUNET_SETI_OperationHandle *
+GNUNET_SETI_prepare (const struct GNUNET_PeerIdentity *other_peer,
+                     const struct GNUNET_HashCode *app_id,
+                     const struct GNUNET_MessageHeader *context_msg,
+                     const struct GNUNET_SETI_Option options[],
+                     GNUNET_SETI_ResultIterator result_cb,
+                     void *result_cls);
+
+
+/**
+ * Wait for set operation requests for the given application ID.
+ * If the connection to the set service is lost, the listener is
+ * re-created transparently with exponential backoff.
+ *
+ * @param cfg configuration to use for connecting to
+ *            the set service
+ * @param app_id id of the application that handles set operation requests
+ * @param listen_cb called for each incoming request matching the operation
+ *                  and application id
+ * @param listen_cls handle for @a listen_cb
+ * @return a handle that can be used to cancel the listen operation
+ */
+struct GNUNET_SETI_ListenHandle *
+GNUNET_SETI_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                    const struct GNUNET_HashCode *app_id,
+                    GNUNET_SETI_ListenCallback listen_cb,
+                    void *listen_cls);
+
+
+/**
+ * Cancel the given listen operation.  After calling cancel, the
+ * listen callback for this listen handle will not be called again.
+ * Note that cancelling a listen operation will automatically reject
+ * all operations that have not yet been accepted.
+ *
+ * @param lh handle for the listen operation
+ */
+void
+GNUNET_SETI_listen_cancel (struct GNUNET_SETI_ListenHandle *lh);
+
+
+/**
+ * Accept a request we got via GNUNET_SETI_listen().  Must be called during
+ * GNUNET_SETI_listen(), as the `struct GNUNET_SETI_Request` becomes invalid
+ * afterwards.
+ * Call GNUNET_SETI_commit() to provide the local set to use for the operation,
+ * and to begin the exchange with the remote peer.
+ *
+ * @param request request to accept
+ * @param options options to use when processing the request
+ * @param result_cb callback for the results
+ * @param result_cls closure for @a result_cb
+ * @return a handle to cancel the operation
+ */
+struct GNUNET_SETI_OperationHandle *
+GNUNET_SETI_accept (struct GNUNET_SETI_Request *request,
+                    const struct GNUNET_SETI_Option options[],
+                    GNUNET_SETI_ResultIterator result_cb,
+                    void *result_cls);
+
+
+/**
+ * Commit a set to be used with a set operation.
+ * This function is called once we have fully constructed
+ * the set that we want to use for the operation.  At this
+ * time, the P2P protocol can then begin to exchange the
+ * set information and call the result callback with the
+ * result information.
+ *
+ * @param oh handle to the set operation
+ * @param set the set to use for the operation
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
+ *         set is invalid (e.g. the set service crashed)
+ */
+int
+GNUNET_SETI_commit (struct GNUNET_SETI_OperationHandle *oh,
+                    struct GNUNET_SETI_Handle *set);
+
+
+/**
+ * Cancel the given set operation.  May not be called after the operation's
+ * `GNUNET_SETI_ResultIterator` has been called with a status of
+ * #GNUNET_SETI_STATUS_FAILURE or #GNUNET_SETI_STATUS_DONE.
+ *
+ * @param oh set operation to cancel
+ */
+void
+GNUNET_SETI_operation_cancel (struct GNUNET_SETI_OperationHandle *oh);
+
+
+/**
+ * Hash a set element.
+ *
+ * @param element the element that should be hashed
+ * @param[out] ret_hash a pointer to where the hash of @a element
+ *        should be stored
+ */
+void
+GNUNET_SETI_element_hash (const struct GNUNET_SETI_Element *element,
+                          struct GNUNET_HashCode *ret_hash);
+
+
+#if 0                           /* keep Emacsens' auto-indent happy */
+{
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+/** @} */  /* end of group */
diff --git a/src/include/gnunet_setu_service.h b/src/include/gnunet_setu_service.h
index 092c03198..d737b97c1 100644
--- a/src/include/gnunet_setu_service.h
+++ b/src/include/gnunet_setu_service.h
@@ -22,9 +22,9 @@
  * @author Christian Grothoff
  *
  * @file
- * Two-peer set operations
+ * Two-peer set union operations
  *
- * @defgroup set  Set service
+ * @defgroup set  Set union service
  * Two-peer set operations
  *
  * @{