multicast: message fragmentation

4 files changed, 162 insertions, 76 deletions

diff --git a/src/include/gnunet_multicast_service.h b/src/include/gnunet_multicast_service.h
index df677784a..5803f551e 100644
--- a/src/include/gnunet_multicast_service.h
+++ b/src/include/gnunet_multicast_service.h
@@ -94,26 +94,46 @@ enum GNUNET_MULTICAST_JoinPolicy
 
 };
 
+enum GNUNET_MULTICAST_MessageFlags
+{
+  /**
+   * First fragment of a message.
+   */
+  GNUNET_MULTICAST_MESSAGE_FIRST_FRAGMENT = 1 << 0,
+
+  /**
+   * Last fragment of a message.
+   */
+  GNUNET_MULTICAST_MESSAGE_LAST_FRAGMENT = 1 << 1,
+
+  /** 
+   * OR'ed flags if message is not fragmented.
+   */
+  GNUNET_MULTICAST_MESSAGE_NOT_FRAGMENTED
+    = GNUNET_MULTICAST_MESSAGE_FIRST_FRAGMENT
+    | GNUNET_MULTICAST_MESSAGE_LAST_FRAGMENT
+};
+
 
 GNUNET_NETWORK_STRUCT_BEGIN
 
 /** 
- * Header of a multicast message.
+ * Header of a multicast message fragment.
  *
- * This format is public as the replay mechanism must replay messages using the
- * same format.  This is needed as we want to integrity-check messages within
+ * This format is public as the replay mechanism must replay message fragments using the
+ * same format.  This is needed as we want to integrity-check message fragments within
  * the multicast layer to avoid multicasting mal-formed messages.
  */
 struct GNUNET_MULTICAST_MessageHeader
 {
 
   /** 
-   * Header for all multicast messages from the origin.
+   * Header for all multicast message fragments from the origin.
    */
   struct GNUNET_MessageHeader header;
 
   /** 
-   * Number of hops this message has taken since the origin.
+   * Number of hops this message fragment has taken since the origin.
    *
    * Helpful to determine shortest paths to the origin for responses among
    * honest peers; updated at each hop and thus not signed and not secure.
@@ -121,23 +141,33 @@ struct GNUNET_MULTICAST_MessageHeader
   uint32_t hop_counter GNUNET_PACKED;
 
   /** 
-   * ECC signature of the message.
+   * ECC signature of the message fragment.
    *
    * Signature must match the public key of the multicast group.
    */
   struct GNUNET_CRYPTO_EccSignature signature;
 
   /** 
-   * Signature of the multicast message.
+   * Signature of the multicast message fragment.
    */
   struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
 
   /** 
-   * Number of the message, monotonically increasing.
+   * Number of the message fragment, monotonically increasing.
+   */
+  uint64_t fragment_id GNUNET_PACKED;
+
+  /** 
+   * Number of the message this fragment belongs to.
    */
   uint64_t message_id GNUNET_PACKED;
 
   /** 
+   * Byte offset of this @e fragment of the @e message.
+   */
+  uint64_t fragment_offset GNUNET_PACKED;
+
+  /** 
    * Counter that monotonically increases whenever a member parts the group.
    *
    * It has significance in case of replay requests: when a member has missed
@@ -148,12 +178,12 @@ struct GNUNET_MULTICAST_MessageHeader
   uint64_t group_generation GNUNET_PACKED;
 
   /** 
-   * Difference between the current @a message_id and the @a message_id of the
+   * Difference between the current @a fragment_id and the @a fragment_id of the
    * preceeding non-transient message.
    * 
    * Zero for transient messages, @c UINT64_MAX for the first message, or any
    * other message creating a full state reset by the origin.  By subtracting
-   * @a state_delta from @a message_id, it is possible to calculate the message
+   * @a state_delta from @a fragment_id, it is possible to calculate the message
    * ID of the preceeding non-transient message and thus quickly traverse all
    * state changes up to the last full state reset by the origin.  This is
    * useful as it allows joining clients to quickly reassemble the state while
@@ -170,6 +200,11 @@ struct GNUNET_MULTICAST_MessageHeader
    */
   uint64_t state_delta GNUNET_PACKED;
 
+  /**
+   * Flags for this message.
+   */
+  enum GNUNET_MULTICAST_MessageFlags flags GNUNET_PACKED;
+
   /** 
    * Header for the message body.
    *
@@ -180,7 +215,44 @@ struct GNUNET_MULTICAST_MessageHeader
    */
   struct GNUNET_MessageHeader body;
 
-  /* followed by message body */
+  /* Followed by message body. */
+};
+
+GNUNET_NETWORK_STRUCT_END
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/** 
+ * Header of a request from a member to the origin.
+ *
+ * This format is public as the replay mechanism must replay message fragments using the
+ * same format.  This is needed as we want to integrity-check message fragments within
+ * the multicast layer to avoid multicasting mal-formed messages.
+ */
+struct GNUNET_MULTICAST_RequestHeader
+{
+
+  /** 
+   * Header for all requests from a member to the origin.
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Flags for this request.
+   */
+  enum GNUNET_MULTICAST_MessageFlags flags GNUNET_PACKED;
+
+  /** 
+   * Header for the request body.
+   *
+   * Two request types are specifically understood by multicast, namely "peer
+   * join", "peer part".  Multicast will use those messages to update its list
+   * of candidates for content distribution.  All other message types are
+   * application-specific.
+   */
+  struct GNUNET_MessageHeader body;
+
+  /* Followed by request body. */
 };
 
 GNUNET_NETWORK_STRUCT_END
@@ -304,32 +376,30 @@ GNUNET_MULTICAST_membership_test_answer (struct GNUNET_MULTICAST_MembershipTestH
  *
  * @param cls Closure.
  * @param peer Identity of the peer that we want to test.
- * @param message_id Message ID for which we want to do the test.
+ * @param fragment_id Message fragment ID for which we want to do the test.
  * @param mth Handle to give to GNUNET_MULTICAST_membership_test_answer().
  */
 typedef void (*GNUNET_MULTICAST_MembershipTestCallback)(void *cls,
                                                         const struct GNUNET_PeerIdentity *peer,
-                                                        uint64_t message_id,
+                                                        uint64_t fragment_id,
                                                         struct GNUNET_MULTICAST_MembershipTestHandle *mth);
 
 
 /** 
- * Function called whenever a group member has transmitted a message
+ * Function called whenever a group member has transmitted a request
  * to the origin (other than joining or leaving).
  *
  * @param cls Closure (set from GNUNET_MULTICAST_origin_start).
  * @param sender Identity of the sender.
- * @param request_id Unique counter for the request from this sender to this origin. FIXME: needed?
- * @param msg Message to the origin.
+ * @param req Request to the origin.
  */
 typedef void (*GNUNET_MULTICAST_RequestCallback) (void *cls,
                                                   const struct GNUNET_PeerIdentity *sender,
-                                                  uint64_t request_id,
-                                                  const struct GNUNET_MessageHeader *msg);
+                                                  const struct GNUNET_MULTICAST_RequestHeader *req);
 
 
 /** 
- * Function called whenever a group member is receiving a message from
+ * Function called whenever a group member is receiving a message fragment from
  * the origin.
  *
  * If admission to the group is denied, this function is called once with the
@@ -337,14 +407,11 @@ typedef void (*GNUNET_MULTICAST_RequestCallback) (void *cls,
  * then a second time with NULL to indicate that the connection failed for good.
  *
  * @param cls Closure (set from GNUNET_MULTICAST_member_join())
- * @param message_id Unique number of the message, 0 for response to join request,
- *        normal message IDs in either direction start at 1.
  * @param msg Message from the origin, NULL if the origin shut down
  *        (or we were kicked out, and we should thus call
  *        GNUNET_MULTICAST_member_part() next)
  */
 typedef void (*GNUNET_MULTICAST_MessageCallback) (void *cls,
-                                                  uint64_t message_id,
                                                   const struct GNUNET_MULTICAST_MessageHeader *msg);
 
 
@@ -364,11 +431,11 @@ struct GNUNET_MULTICAST_ReplayHandle;
  *
  * @param cls Closure (set from GNUNET_MULTICAST_origin_start()
  *            or GNUNET_MULTICAST_member_join()).
- * @param message_id Which message should be replayed.
+ * @param fragment_id Which message fragment should be replayed.
  * @param rh Handle to pass to message transmit function.
  */
 typedef void (*GNUNET_MULTICAST_ReplayCallback) (void *cls,
-                                                 uint64_t message_id,
+                                                 uint64_t fragment_id,
                                                  struct GNUNET_MULTICAST_ReplayHandle *rh);
 
 
@@ -384,12 +451,12 @@ enum GNUNET_MULTICAST_ReplayErrorCode
   GNUNET_MULTICAST_REC_OK = 0,
 
   /** 
-   * Message has been discarded (likely transient message that was too old).
+   * Message fragment has been discarded (likely transient message that was too old).
    */ 
   GNUNET_MULTICAST_REC_TRANSIENT_LOST = 1,
 
   /** 
-   * Message ID counter was larger than the highest counter this
+   * Fragment ID counter was larger than the highest counter this
    * replay function has ever encountered; thus it is likely the
    * origin never sent it and we're at the HEAD of the multicast
    * stream as far as this node is concerned.
@@ -408,7 +475,7 @@ enum GNUNET_MULTICAST_ReplayErrorCode
  * Replay a message from the multicast group.
  *
  * @param rh Replay handle identifying which replay operation was requested.
- * @param msg Replayed message, NULL if unknown/error.
+ * @param msg Replayed message fragment, NULL if unknown/error.
  * @param ec Error code.
  */
 void
@@ -509,15 +576,14 @@ GNUNET_MULTICAST_origin_end (struct GNUNET_MULTICAST_Origin *origin);
  * members of the group.
  *
  * @param cfg Configuration to use.
- * @param cls Closure for callbacks.
  * @param pub_key ECC key that identifies the group.
  * @param origin Peer identity of the origin.
- * @param max_known_message_id Largest known message ID to the replay service;
- *        all messages with IDs larger than this ID will be replayed if
+ * @param max_known_fragment_id Largest known message fragment ID to the replay
+          service; all messages with IDs larger than this ID will be replayed if
  *        possible (lower IDs will be considered known and thus only
  *        be replayed upon explicit request).
- * @param max_known_state_message_id Largest known message ID with a non-zero
- *        value for the @e state_delta; state messages with
+ * @param max_known_state_fragment_id Largest known message fragment ID with a
+          non-zero value for the @e state_delta; state messages with
  *        larger IDs than this value will be replayed with high priority
  *        (lower IDs will be considered known and thus only
  *        be replayed upon explicit request).
@@ -525,9 +591,10 @@ GNUNET_MULTICAST_origin_end (struct GNUNET_MULTICAST_Origin *origin);
  *        this peer already knows from this group; NULL if this
  *        client is unable to support replay.
  * @param test_cb Function multicast can use to test group membership.
- * @param message_cb Function to be called for all messages we 
+ * @param message_cb Function to be called for all message fragments we 
  *        receive from the group, excluding those our @a replay_cb
  *        already has.
+ * @param cls Closure for callbacks.
  * @param join_req Application-dependent join message to be passed to origin
  *        (might, for example, contain a user 
  *        bind user identity/pseudonym to peer identity, application-level
@@ -536,14 +603,14 @@ GNUNET_MULTICAST_origin_end (struct GNUNET_MULTICAST_Origin *origin);
  */
 struct GNUNET_MULTICAST_Member *
 GNUNET_MULTICAST_member_join (const struct GNUNET_CONFIGURATION_Handle *cfg, 
-                              void *cls,
                               const struct GNUNET_CRYPTO_EccPublicKey *pub_key,
                               const struct GNUNET_PeerIdentity *origin,
-                              uint64_t max_known_message_id,
-                              uint64_t max_known_state_message_id,
+                              uint64_t max_known_fragment_id,
+                              uint64_t max_known_state_fragment_id,
                               GNUNET_MULTICAST_ReplayCallback replay_cb,
                               GNUNET_MULITCAST_MembershipTestCallback test_cb,
                               GNUNET_MULTICAST_MessageCallback message_cb,
+                              void *cls,
                               const struct GNUNET_MessageHeader *join_req);
 
 
@@ -560,14 +627,14 @@ struct GNUNET_MULTICAST_MemberReplayHandle;
  * needed and not known to the client.
  *
  * @param member Membership handle.
- * @param message_id ID of a message that this client would like to see replayed.
+ * @param fragment_id ID of a message fragment that this client would like to see replayed.
  * @param message_cb Function to be called for the replayed message.
  * @param message_cb_cls Closure for @a message_cb.
  * @return Replay request handle, NULL on error.
  */
 struct GNUNET_MULTICAST_MemberReplayHandle *
 GNUNET_MULTICAST_member_request_replay (struct GNUNET_MULTICAST_Member *member,
-                                        uint64_t message_id,
+                                        uint64_t fragment_id,
                                         GNUNET_MULTICAST_MessageCallback message_cb,
                                         void *message_cb_cls);
 
diff --git a/src/include/gnunet_psyc_service.h b/src/include/gnunet_psyc_service.h
index e6d47dc4e..e23eef669 100644
--- a/src/include/gnunet_psyc_service.h
+++ b/src/include/gnunet_psyc_service.h
@@ -101,25 +101,22 @@ extern "C"
 #define GNUNET_PSYC_VERSION 0x00000000
 
 
-/** 
- * Information flags for data fragments set via PSYC.
- */
-enum GNUNET_PSYC_FragmentStatus
+enum GNUNET_PSYC_MessageFlags
 {
-  /** 
-   * This is the first part of data for the given method call.
+  /**
+   * First fragment of a message.
    */
-  GNUNET_PSYC_FS_FIRST = 1,
-  
-  /** 
-   * This is the last part of data for the given method call.
+  GNUNET_PSYC_MESSAGE_FIRST_FRAGMENT = GNUNET_MULTICAST_MESSAGE_FIRST_FRAGMENT,
+
+  /**
+   * Last fragment of a message.
    */
-  GNUNET_PSYC_FS_LAST = 2,
+  GNUNET_PSYC_MESSAGE_LAST_FRAGMENT = GNUNET_MULTICAST_MESSAGE_LAST_FRAGMENT,
 
   /** 
-   * OR'ed flags if payload is not fragmented.
+   * OR'ed flags if message is not fragmented.
    */
-  GNUNET_PSYC_FS_NOT_FRAGMENTED = (GNUNET_PSYC_FS_FIRST | GNUNET_PSYC_FS_LAST)
+  GNUNET_PSYC_MESSAGE_NOT_FRAGMENTED = GNUNET_MULTICAST_MESSAGE_NOT_FRAGMENTED
 };
 
 
@@ -157,7 +154,7 @@ struct GNUNET_PSYC_PartHandle;
  *        FIXME: no try-and-slice for methods defined here.
  * @param header_length Number of modifiers in header.
  * @param header Modifiers present in the message.
- * @param data_off Byte offset of @a data in the overall data of the method.
+ * @param data_offset Byte offset of @a data in the overall data of the method.
  * @param data_size Number of bytes in @a data.
  * @param data Data stream given to the method (might not be zero-terminated 
  *             if data is binary).
@@ -168,10 +165,12 @@ typedef int (*GNUNET_PSYC_Method)(void *cls,
                                   uint64_t message_id,
                                   uint64_t group_generation,
                                   const char *method_name,
-                                  uint64_t data_off,
+                                  size_t header_length,
+                                  GNUNET_PSYC_Modifier *header,                                 
+                                  uint64_t data_offset,
                                   size_t data_size,
                                   const void *data,
-                                  enum GNUNET_PSYC_FragmentStatus frag);
+                                  enum GNUNET_PSYC_MessageFlags flags);
 
 
 /** 
diff --git a/src/include/gnunet_psycstore_service.h b/src/include/gnunet_psycstore_service.h
index 46034dae6..8b84340fe 100644
--- a/src/include/gnunet_psycstore_service.h
+++ b/src/include/gnunet_psycstore_service.h
@@ -111,7 +111,7 @@ GNUNET_PSYCSTORE_membership_store (struct GNUNET_PSYCSTORE_Handle *h,
 
 
 /** 
- * Test if a peer was a member of the channel when the message with the
+ * Test if a peer was a member of the channel when the message fragment with the
  * specified ID was sent to the channel.
  *
  * This is useful in case of retransmissions to check if the peer was authorized
@@ -119,7 +119,7 @@ GNUNET_PSYCSTORE_membership_store (struct GNUNET_PSYCSTORE_Handle *h,
  *
  * @param h Handle for the PSYCstore.
  * @param channel_id The channel we are interested in.
- * @param message_id Message ID to check.
+ * @param fragment_id Message fragment ID to check.
  * @param peer Peer whose membership to check.
  * @param ccb Callback to call with the test result.
  * @param ccb_cls Closure for the callback.
@@ -129,14 +129,14 @@ GNUNET_PSYCSTORE_membership_store (struct GNUNET_PSYCSTORE_Handle *h,
 struct GNUNET_PSYCSTORE_OperationHandle *
 GNUNET_PSYCSTORE_membership_test (struct GNUNET_PSYCSTORE_Handle *h,
                                   const struct GNUNET_HashCode *channel_id,
-                                  uint64_t message_id,
+                                  uint64_t fragment_id,
                                   const struct GNUNET_PeerIdentity *peer,
                                   GNUNET_PSYCSTORE_ContinuationCallback ccb,
                                   void *ccb_cls);
 
 
 /** 
- * Store a message sent to a channel.
+ * Store a message fragment sent to a channel.
  *
  * @param h Handle for the PSYCstore.
  * @param channel_id The channel the message belongs to.
@@ -147,23 +147,43 @@ GNUNET_PSYCSTORE_membership_test (struct GNUNET_PSYCSTORE_Handle *h,
  * @return Handle that can be used to cancel the operation.
  */
 struct GNUNET_PSYCSTORE_OperationHandle *
-GNUNET_PSYCSTORE_message_store (struct GNUNET_PSYCSTORE_Handle *h,
-                                const struct GNUNET_HashCode *channel_id,
-                                const struct GNUNET_MULTICAST_MessageHeader *message,
-                                GNUNET_PSYCSTORE_ContinuationCallback ccb,
-                                void *ccb_cls);
+GNUNET_PSYCSTORE_fragment_store (struct GNUNET_PSYCSTORE_Handle *h,
+                                 const struct GNUNET_HashCode *channel_id,
+                                 const struct GNUNET_MULTICAST_MessageHeader *message,
+                                 GNUNET_PSYCSTORE_ContinuationCallback ccb,
+                                 void *ccb_cls);
 
 
 /** 
- * Function called with the result of a GNUNET_PSYCSTORE_message_get() call.
+ * Function called with one message fragment, as the result of a
+ * GNUNET_PSYCSTORE_fragment_get() or GNUNET_PSYCSTORE_message_get() call.
  *
  * @param cls Closure.
- * @param message_id ID of the message.
- * @param message The retrieved message.
+ * @param message The retrieved message fragment.
+ * @param flags Message flags indicating fragmentation status.
+ */
+typedef void (*GNUNET_PSYCSTORE_FragmentResultCallback)(void *cls,      
+                                                       const struct GNUNET_MULTICAST_MessageHeader *message,
+                                                       enum GNUNET_PSYC_MessageFlags flags);
+
+
+/** 
+ * Retrieve a message fragment by fragment ID.
+ *
+ * @param h Handle for the PSYCstore.
+ * @param channel_id The channel we are interested in.
+ * @param fragment_id Fragment ID to check.  Use 0 to get the latest message fragment.
+ * @param rcb Callback to call with the result of the operation.
+ * @param rcb_cls Closure for the callback.
+ * 
+ * @return Handle that can be used to cancel the operation.
  */
-typedef void (*GNUNET_PSYCSTORE_MessageResultCallback)(void *cls,       
-                                                       uint64_t message_id,                                    
-                                                       const struct GNUNET_MULTICAST_MessageHeader *message);
+struct GNUNET_PSYCSTORE_OperationHandle *
+GNUNET_PSYCSTORE_fragment_get (struct GNUNET_PSYCSTORE_Handle *h,
+                               const struct GNUNET_HashCode *channel_id,
+                               uint64_t message_id,
+                               GNUNET_PSYCSTORE_FragmentResultCallback rcb,
+                               void *rcb_cls);
 
 
 /** 
@@ -181,7 +201,7 @@ struct GNUNET_PSYCSTORE_OperationHandle *
 GNUNET_PSYCSTORE_message_get (struct GNUNET_PSYCSTORE_Handle *h,
                               const struct GNUNET_HashCode *channel_id,
                               uint64_t message_id,
-                              GNUNET_PSYCSTORE_MessageResultCallback rcb,
+                              GNUNET_PSYCSTORE_FragmentResultCallback rcb,
                               void *rcb_cls);
 
 
@@ -191,7 +211,7 @@ GNUNET_PSYCSTORE_message_get (struct GNUNET_PSYCSTORE_Handle *h,
  * @param h Handle for the PSYCstore.
  * @param channel_id The channel we are interested in.
  * @param name Name of variable.
- * @param size Size of @a value.
+ * @param value_size Size of @a value.
  * @param value Value of variable.
  * @param ccb Callback to call with the result of the operation.
  * @param ccb_cls Closure for the callback.
@@ -202,7 +222,7 @@ struct GNUNET_PSYCSTORE_OperationHandle *
 GNUNET_PSYCSTORE_state_set (struct GNUNET_PSYCSTORE_Handle *h,
                             const struct GNUNET_HashCode *channel_id,
                             const char *name,
-                            size_t size,
+                            size_t value_size,
                             const void *value,
                             GNUNET_PSYCSTORE_ContinuationCallback ccb,
                             void *ccb_cls);
diff --git a/src/include/gnunet_social_service.h b/src/include/gnunet_social_service.h
index 1d2b2096c..d4d9c437c 100644
--- a/src/include/gnunet_social_service.h
+++ b/src/include/gnunet_social_service.h
@@ -86,21 +86,21 @@ struct GNUNET_SOCIAL_Slicer;
  *                    this channel).
  * @param header_length Number of modifiers in header.
  * @param header Modifiers present in the message.
- * @param data_off Byte offset of @a data in the overall data of the method.
+ * @param data_offset Byte offset of @a data in the overall data of the method.
  * @param data_size Number of bytes in @a data.
  * @param data Data stream given to the method (might not be zero-terminated 
  *             if data is binary).
- * @param frag Fragmentation status for the data.
+ * @param flags Message flags indicating fragmentation status.
  */
 typedef int (*GNUNET_SOCIAL_Method)(void *cls,
                                     const char *full_method_name,
                                     uint64_t message_id,
                                     size_t header_length,
                                     GNUNET_PSYC_Modifier *header,
-                                    uint64_t data_off,
+                                    uint64_t data_offset,
                                     size_t data_size,
                                     const void *data,
-                                    enum GNUNET_PSYC_FragmentStatus frag);
+                                    enum GNUNET_PSYC_MessageFlags flags);
 
 
 /**