-use hash in prd query to reduce memory consumption of CP request_map

4 files changed, 83 insertions, 51 deletions

diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c
index 7066b0698..1d52f28f9 100644
--- a/src/fs/gnunet-service-fs_cp.c
+++ b/src/fs/gnunet-service-fs_cp.c
@@ -250,7 +250,7 @@ struct GSF_ConnectedPeer
   struct GNUNET_SCHEDULER_Task *rc_delay_task;
 
   /**
-   * Active requests from this neighbour, map of query to 'struct PeerRequest'.
+   * Active requests from this neighbour, map of query to `struct PeerRequest`.
    */
   struct GNUNET_CONTAINER_MultiHashMap *request_map;
 
@@ -646,7 +646,8 @@ GSF_peer_connect_handler_ (const struct GNUNET_PeerIdentity *peer,
   cp->rc =
       GNUNET_ATS_reserve_bandwidth (GSF_ats, peer, DBLOCK_SIZE,
                                     &ats_reserve_callback, cp);
-  cp->request_map = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_NO);
+  cp->request_map = GNUNET_CONTAINER_multihashmap_create (128,
+                                                          GNUNET_YES);
   GNUNET_break (GNUNET_OK ==
                 GNUNET_CONTAINER_multipeermap_put (cp_map,
                GSF_connected_peer_get_identity2_ (cp),
@@ -811,8 +812,10 @@ free_pending_request (struct PeerRequest *peerreq,
     GNUNET_SCHEDULER_cancel (peerreq->kill_task);
     peerreq->kill_task = NULL;
   }
-  GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"),
-                            -1, GNUNET_NO);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# P2P searches active"),
+                            -1,
+                            GNUNET_NO);
   GNUNET_break (GNUNET_YES ==
                 GNUNET_CONTAINER_multihashmap_remove (cp->request_map,
                                                       query,
@@ -1288,6 +1291,7 @@ GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other,
   uint32_t ttl_decrement;
   struct TestExistClosure tec;
   GNUNET_PEER_Id spid;
+  const struct GSF_PendingRequestData *prd;
 
   msize = ntohs (message->size);
   if (msize < sizeof (struct GetMessage))
@@ -1452,20 +1456,25 @@ GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other,
                                     spid,
                                     GNUNET_PEER_intern (other),
                                     NULL, 0,        /* replies_seen */
-                                    &handle_p2p_reply, peerreq);
+                                    &handle_p2p_reply,
+                                    peerreq);
   GNUNET_assert (NULL != pr);
+  prd = GSF_pending_request_get_data_ (pr);
   peerreq->pr = pr;
   GNUNET_break (GNUNET_OK ==
                 GNUNET_CONTAINER_multihashmap_put (cp->request_map,
-                                                   &gm->query,
+                                                   &prd->query,
                                                    peerreq,
                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
   GNUNET_STATISTICS_update (GSF_stats,
                             gettext_noop
-                            ("# P2P query messages received and processed"), 1,
+                            ("# P2P query messages received and processed"),
+                            1,
+                            GNUNET_NO);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# P2P searches active"),
+                            1,
                             GNUNET_NO);
-  GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"),
-                            1, GNUNET_NO);
   return pr;
 }
 
diff --git a/src/fs/gnunet-service-fs_pe.c b/src/fs/gnunet-service-fs_pe.c
index ede109c2d..9743d1cb2 100644
--- a/src/fs/gnunet-service-fs_pe.c
+++ b/src/fs/gnunet-service-fs_pe.c
@@ -59,7 +59,7 @@ struct PeerPlan;
  * to a given plan entry.)
  *
  * Similarly head and tail of the "PR" MDLL are stored
- * with the 'struct GSF_PendingRequest'.  (We need
+ * with the `struct GSF_PendingRequest`.  (We need
  * to be able to lookup all plan entries corresponding
  * to a given pending request.)
  */
@@ -87,12 +87,14 @@ struct GSF_PendingRequestPlanBijection
   struct GSF_PendingRequestPlanBijection *prev_PE;
 
   /**
-   * Associated request plan.
+   * Associated request plan (tells us one of the peers that
+   * we plan to forward the request to).
    */
   struct GSF_RequestPlan *rp;
 
   /**
-   * Associated pending request.
+   * Associated pending request (identifies request details
+   * and one of the origins of the request).
    */
   struct GSF_PendingRequest *pr;
 
@@ -101,8 +103,8 @@ struct GSF_PendingRequestPlanBijection
 
 /**
  * Information we keep per request per peer.  This is a doubly-linked
- * list (with head and tail in the 'struct GSF_PendingRequestData')
- * with one entry in each heap of each 'struct PeerPlan'.  Each
+ * list (with head and tail in the `struct GSF_PendingRequestData`)
+ * with one entry in each heap of each `struct PeerPlan`.  Each
  * entry tracks information relevant for this request and this peer.
  */
 struct GSF_RequestPlan
@@ -129,7 +131,9 @@ struct GSF_RequestPlan
   struct PeerPlan *pp;
 
   /**
-   * Head of list of associated pending requests.
+   * Head of list of associated pending requests.  This tells us
+   * which incoming requests from other peers this plan entry
+   * corresponds to.
    */
   struct GSF_PendingRequestPlanBijection *pe_head;
 
@@ -167,19 +171,20 @@ struct GSF_RequestPlan
 struct PeerPlan
 {
   /**
-   * Heap with pending queries (struct GSF_RequestPlan), higher weights mean higher priority.
+   * Heap with pending queries (`struct GSF_RequestPlan`), higher weights mean higher priority.
    */
   struct GNUNET_CONTAINER_Heap *priority_heap;
 
   /**
-   * Heap with pending queries (struct GSF_RequestPlan), by transmission time, lowest first.
+   * Heap with pending queries (`struct GSF_RequestPlan`), by transmission time, lowest first.
    */
   struct GNUNET_CONTAINER_Heap *delay_heap;
 
   /**
-   * Map of queries to plan entries.  All entries in the priority_heap or delay_heap
-   * should be in the plan map.  Note that it IS possible for the plan map to have
-   * multiple entries for the same query.
+   * Map of queries to plan entries.  All entries in the @e priority_heap
+   * or @e delay_heap should be in the @e plan_map.  Note that it is
+   * possible for the @e plan_map to have multiple entries for the same
+   * query.
    */
   struct GNUNET_CONTAINER_MultiHashMap *plan_map;
 
@@ -196,7 +201,7 @@ struct PeerPlan
   /**
    * Current task for executing the plan.
    */
-  struct GNUNET_SCHEDULER_Task * task;
+  struct GNUNET_SCHEDULER_Task *task;
 };
 
 
@@ -222,7 +227,7 @@ static unsigned long long plan_count;
  * requests for the same key and we just return _one_ of them; this
  * particular one might complete while another one might still be
  * active, hence the lifetime of the returned hash code is NOT
- * necessarily identical to that of the 'struct GSF_RequestPlan'
+ * necessarily identical to that of the `struct GSF_RequestPlan`
  * given.
  *
  * @param rp a request plan
@@ -253,7 +258,8 @@ schedule_peer_transmission (void *cls,
  * @param rp request to plan
  */
 static void
-plan (struct PeerPlan *pp, struct GSF_RequestPlan *rp)
+plan (struct PeerPlan *pp,
+      struct GSF_RequestPlan *rp)
 {
 #define N ((double)128.0)
   /**
@@ -366,8 +372,7 @@ get_latest (const struct GSF_RequestPlan *rp)
   for (bi = bi->next_PE; NULL != bi; bi = bi->next_PE)
   {
     prd = GSF_pending_request_get_data_ (bi->pr);
-    if (prd->ttl.abs_value_us >
-        rprd->ttl.abs_value_us)
+    if (prd->ttl.abs_value_us > rprd->ttl.abs_value_us)
     {
       ret = bi->pr;
       rprd = prd;
@@ -383,10 +388,12 @@ get_latest (const struct GSF_RequestPlan *rp)
  * @param cls closure
  * @param buf_size number of bytes available in @a buf
  * @param buf where to copy the message, NULL on error (peer disconnect)
- * @return number of bytes copied to 'buf', can be 0 (without indicating an error)
+ * @return number of bytes copied to @a buf, can be 0 (without indicating an error)
  */
 static size_t
-transmit_message_callback (void *cls, size_t buf_size, void *buf)
+transmit_message_callback (void *cls,
+                           size_t buf_size,
+                           void *buf)
 {
   struct PeerPlan *pp = cls;
   struct GSF_RequestPlan *rp;
@@ -414,7 +421,9 @@ transmit_message_callback (void *cls, size_t buf_size, void *buf)
     pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp);
     return 0;
   }
-  msize = GSF_pending_request_get_message_ (get_latest (rp), buf_size, buf);
+  msize = GSF_pending_request_get_message_ (get_latest (rp),
+                                            buf_size,
+                                            buf);
   if (msize > buf_size)
   {
     if (NULL != pp->task)
@@ -476,7 +485,8 @@ schedule_peer_transmission (void *cls,
     rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap);
     if (NULL == rp)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "No active requests for plan %p.\n",
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "No active requests for plan %p.\n",
                   pp);
       return;                   /* both queues empty */
     }
@@ -486,11 +496,14 @@ schedule_peer_transmission (void *cls,
                 GNUNET_STRINGS_relative_time_to_string (delay,
                                                         GNUNET_YES),
                 pp);
-    GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# delay heap timeout (ms)"),
+    GNUNET_STATISTICS_set (GSF_stats,
+                           gettext_noop ("# delay heap timeout (ms)"),
                            delay.rel_value_us / 1000LL, GNUNET_NO);
 
     pp->task =
-        GNUNET_SCHEDULER_add_delayed (delay, &schedule_peer_transmission, pp);
+        GNUNET_SCHEDULER_add_delayed (delay,
+                                      &schedule_peer_transmission,
+                                      pp);
     return;
   }
 #if INSANE_STATISTICS
@@ -503,8 +516,10 @@ schedule_peer_transmission (void *cls,
   GNUNET_assert (NULL != rp);
   msize = GSF_pending_request_get_message_ (get_latest (rp), 0, NULL);
   pp->pth =
-      GSF_peer_transmit_ (pp->cp, GNUNET_YES, rp->priority,
-                          GNUNET_TIME_UNIT_FOREVER_REL, msize,
+      GSF_peer_transmit_ (pp->cp, GNUNET_YES,
+                          rp->priority,
+                          GNUNET_TIME_UNIT_FOREVER_REL,
+                          msize,
                           &transmit_message_callback, pp);
   GNUNET_assert (NULL != pp->pth);
 }
@@ -516,8 +531,14 @@ schedule_peer_transmission (void *cls,
 struct MergeContext
 {
 
+  /**
+   * Request we are trying to merge.
+   */
   struct GSF_PendingRequest *pr;
 
+  /**
+   * Set to #GNUNET_YES if we succeeded to merge.
+   */
   int merged;
 
 };
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index b507992b4..21b00e80e 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -289,7 +289,8 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
                              enum GNUNET_BLOCK_Type type,
                              const struct GNUNET_HashCode *query,
                              const struct GNUNET_PeerIdentity *target,
-                             const char *bf_data, size_t bf_size,
+                             const char *bf_data,
+                             size_t bf_size,
                              uint32_t mingle,
                              uint32_t anonymity_level,
                              uint32_t priority,
@@ -325,7 +326,9 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
   if (NULL != target)
   {
     pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr;
-    memcpy (eptr, target, sizeof (struct GNUNET_PeerIdentity));
+    memcpy (eptr,
+            target,
+            sizeof (struct GNUNET_PeerIdentity));
   }
   pr->public_data.anonymity_level = anonymity_level;
   pr->public_data.priority = priority;
@@ -397,7 +400,9 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
                  UINT32_MAX,
                  GNUNET_TIME_UNIT_FOREVER_ABS,
                  GNUNET_TIME_UNIT_FOREVER_ABS,
-                 GNUNET_BLOCK_TYPE_ANY, NULL, 0);
+                 GNUNET_BLOCK_TYPE_ANY,
+                 NULL,
+                 0);
       GSF_pending_request_cancel_ (dpr,
                                    GNUNET_YES);
     }
diff --git a/src/fs/gnunet-service-fs_pr.h b/src/fs/gnunet-service-fs_pr.h
index d022d8640..547595b67 100644
--- a/src/fs/gnunet-service-fs_pr.h
+++ b/src/fs/gnunet-service-fs_pr.h
@@ -179,25 +179,22 @@ struct GSF_PendingRequestData
  * @param eval evaluation of the result
  * @param pr handle to the original pending request
  * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown"
- * @param expiration when does 'data' expire?
+ * @param expiration when does @a data expire?
  * @param last_transmission the last time we've tried to get this block (FOREVER if unknown)
  * @param type type of the block
  * @param data response data, NULL on request expiration
- * @param data_len number of bytes in data
+ * @param data_len number of bytes in @a data
  */
-typedef void (*GSF_PendingRequestReplyHandler) (void *cls,
-                                                enum
-                                                GNUNET_BLOCK_EvaluationResult
-                                                eval,
-                                                struct GSF_PendingRequest * pr,
-                                                uint32_t reply_anonymity_level,
-                                                struct GNUNET_TIME_Absolute
-                                                expiration,
-                                                struct GNUNET_TIME_Absolute
-                                                last_transmission,
-                                                enum GNUNET_BLOCK_Type type,
-                                                const void *data,
-                                                size_t data_len);
+typedef void
+(*GSF_PendingRequestReplyHandler) (void *cls,
+                                   enum GNUNET_BLOCK_EvaluationResult eval,
+                                   struct GSF_PendingRequest *pr,
+                                   uint32_t reply_anonymity_level,
+                                   struct GNUNET_TIME_Absolute expiration,
+                                   struct GNUNET_TIME_Absolute last_transmission,
+                                   enum GNUNET_BLOCK_Type type,
+                                   const void *data,
+                                   size_t data_len);
 
 
 /**