5 files changed, 137 insertions, 67 deletions

diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c
index 29fc0cdd7..302020d7d 100644
--- a/src/fs/fs_api.c
+++ b/src/fs/fs_api.c
@@ -65,9 +65,13 @@ start_job (struct GNUNET_FS_QueueEntry *qe)
               "Starting job %p (%u active)\n",
               qe,
               qe->h->active_downloads);
-  GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe);
-  GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head, qe->h->running_tail,
-                                     qe->h->running_tail, qe);
+  GNUNET_CONTAINER_DLL_remove (qe->h->pending_head,
+                               qe->h->pending_tail,
+                               qe);
+  GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head,
+                                     qe->h->running_tail,
+                                     qe->h->running_tail,
+                                     qe);
 }
 
 
@@ -207,19 +211,25 @@ process_job_queue (void *cls,
               num_downloads_expired,
               num_downloads_waiting);
   /* calculate start/stop decisions */
-  if (h->active_downloads + num_downloads_waiting > h->max_parallel_requests)
+  if (h->active_downloads + num_downloads_waiting > h->max_parallel_downloads)
   {
-    /* stop probes if possible */
-    num_probes_change = - num_probes_active;
-    num_downloads_change = h->max_parallel_requests - h->active_downloads;
+    /* stop as many probes as there are downloads and probes */
+    num_probes_change = - GNUNET_MIN (num_probes_active,
+                                      num_downloads_waiting);
+    /* start as many downloads as there are free slots, including those
+       we just opened up */
+    num_downloads_change = h->max_parallel_downloads - h->active_downloads - num_probes_change;
   }
   else
   {
-    /* start all downloads */
+    /* start all downloads (we can) */
     num_downloads_change = num_downloads_waiting;
-    /* start as many probes as we can */
-    num_probes_change = GNUNET_MIN (num_probes_waiting,
-                                    h->max_parallel_requests - (h->active_downloads + num_downloads_waiting));
+    /* also start probes if there is room, but use a lower cap of (mpd/4) + 1 */
+    if (h->max_parallel_downloads / 2 >= (h->active_downloads + num_downloads_change))
+      num_probes_change = GNUNET_MIN (num_probes_waiting,
+                                      (1 + h->max_parallel_downloads / 4) - (h->active_downloads + num_downloads_change));
+    else
+      num_probes_change = 0;
   }
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h
index d43da2028..46f398e86 100644
--- a/src/fs/fs_api.h
+++ b/src/fs/fs_api.h
@@ -531,6 +531,16 @@ struct GNUNET_FS_SearchResult
   struct GNUNET_FS_Handle *h;
 
   /**
+   * Kept in a DLL while probing.
+   */
+  struct GNUNET_FS_SearchResult *next;
+
+  /**
+   * Kept in a DLL while probing.
+   */
+  struct GNUNET_FS_SearchResult *prev;
+
+  /**
    * Search context this result belongs to; can be NULL
    * for probes that come from a directory result.
    */
@@ -592,12 +602,6 @@ struct GNUNET_FS_SearchResult
   GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task;
 
   /**
-   * Task we use to report periodically to the application that the
-   * probe is still running.
-   */
-  GNUNET_SCHEDULER_TaskIdentifier probe_ping_task;
-
-  /**
    * When did the current probe become active?
    */
   struct GNUNET_TIME_Absolute probe_active_time;
@@ -651,8 +655,11 @@ struct GNUNET_FS_SearchResult
  * @return queue handle
  */
 struct GNUNET_FS_QueueEntry *
-GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start,
-                  GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks,
+GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h,
+                  GNUNET_FS_QueueStart start,
+                  GNUNET_FS_QueueStop stop,
+                  void *cls,
+                  unsigned int blocks,
                   enum GNUNET_FS_QueuePriority priority);
 
 
@@ -903,7 +910,8 @@ GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext,
  * @param uni unique name of parent
  */
 void
-GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext,
+GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h,
+                            const char *ext,
                             const char *uni);
 
 
@@ -1003,7 +1011,7 @@ GNUNET_FS_search_signal_suspend_ (void *cls);
  * Create SUSPEND event for the given download operation
  * and then clean up our state (without stop signal).
  *
- * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for
+ * @param cls the `struct GNUNET_FS_DownloadContext` to signal for
  */
 void
 GNUNET_FS_download_signal_suspend_ (void *cls);
@@ -1138,12 +1146,28 @@ struct GNUNET_FS_Handle
   struct GNUNET_FS_QueueEntry *pending_tail;
 
   /**
+   * Head of active probes.
+   */
+  struct GNUNET_FS_SearchResult *probes_head;
+
+  /**
+   * Tail of active probes.
+   */
+  struct GNUNET_FS_SearchResult *probes_tail;
+
+  /**
    * Task that processes the jobs in the running and pending queues
    * (and moves jobs around as needed).
    */
   GNUNET_SCHEDULER_TaskIdentifier queue_job;
 
   /**
+   * Task we use to report periodically to the application that
+   * certain search probes (from @e probes_head) are still running.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier probe_ping_task;
+
+  /**
    * Average time we take for a single request to be satisfied.
    * FIXME: not yet calcualted properly...
    */
@@ -1563,7 +1587,7 @@ struct GNUNET_FS_SearchContext
   char *emsg;
 
   /**
-   * Map that contains a "struct GNUNET_FS_SearchResult" for each result that
+   * Map that contains a `struct GNUNET_FS_SearchResult` for each result that
    * was found in the search.  The key for each entry is the XOR of
    * the key and query in the CHK URI (as a unique identifier for the
    * search result).
@@ -1769,6 +1793,15 @@ GNUNET_FS_free_download_request_ (struct DownloadRequest *dr);
 
 
 /**
+ * Stop the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+void
+GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr);
+
+
+/**
  * Context for controlling a download.
  */
 struct GNUNET_FS_DownloadContext
@@ -1912,7 +1945,7 @@ struct GNUNET_FS_DownloadContext
   /**
    * ID of a task that is using this struct and that must be cancelled
    * when the download is being stopped (if not
-   * GNUNET_SCHEDULER_NO_TASK).  Used for the task that adds some
+   * #GNUNET_SCHEDULER_NO_TASK).  Used for the task that adds some
    * artificial delay when trying to reconnect to the FS service or
    * the task processing incrementally the data on disk, or the
    * task requesting blocks, etc.
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index b3726e9bb..348a0bc17 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -2257,11 +2257,7 @@ GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h,
   {
     GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
     sr->probe_ctx = NULL;
-  }
-  if (GNUNET_SCHEDULER_NO_TASK != sr->probe_ping_task)
-  {
-    GNUNET_SCHEDULER_cancel (sr->probe_ping_task);
-    sr->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
   }
   return dc;
 }
@@ -2314,14 +2310,14 @@ GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc)
 
     pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
     GNUNET_FS_download_make_status_ (&pi, dc);
-  
+
     dc->job_queue =
       GNUNET_FS_queue_ (dc->h, &activate_fs_download, &deactivate_fs_download,
                         dc, (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
                         (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
                         ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
                         : GNUNET_FS_QUEUE_PRIORITY_PROBE);
-                        
+
 }
 
 
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c
index a9c9389c4..c85d6d3df 100644
--- a/src/fs/fs_search.c
+++ b/src/fs/fs_search.c
@@ -171,7 +171,9 @@ struct GetResultContext
  * @return #GNUNET_OK
  */
 static int
-get_result_present (void *cls, const struct GNUNET_HashCode * key, void *value)
+get_result_present (void *cls,
+                    const struct GNUNET_HashCode *key,
+                    void *value)
 {
   struct GetResultContext *grc = cls;
   struct GNUNET_FS_SearchResult *sr = value;
@@ -216,7 +218,8 @@ signal_probe_result (struct GNUNET_FS_SearchResult *sr)
  * @param tc scheduler context
  */
 static void
-probe_failure_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+probe_failure_handler (void *cls,
+                       const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_FS_SearchResult *sr = cls;
 
@@ -224,11 +227,7 @@ probe_failure_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   sr->availability_trials++;
   GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
   sr->probe_ctx = NULL;
-  if (GNUNET_SCHEDULER_NO_TASK != sr->probe_ping_task)
-  {
-    GNUNET_SCHEDULER_cancel (sr->probe_ping_task);
-    sr->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
-  }
+  GNUNET_FS_stop_probe_ping_task_ (sr);
   GNUNET_FS_search_result_sync_ (sr);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Probe #%u for search result %p failed\n",
@@ -245,7 +244,8 @@ probe_failure_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
  * @param tc scheduler context
  */
 static void
-probe_success_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+probe_success_handler (void *cls,
+                       const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_FS_SearchResult *sr = cls;
 
@@ -254,11 +254,7 @@ probe_success_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   sr->availability_success++;
   GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
   sr->probe_ctx = NULL;
-  if (GNUNET_SCHEDULER_NO_TASK != sr->probe_ping_task)
-  {
-    GNUNET_SCHEDULER_cancel (sr->probe_ping_task);
-    sr->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
-  }
+  GNUNET_FS_stop_probe_ping_task_ (sr);
   GNUNET_FS_search_result_sync_ (sr);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Probe #%u for search result %p succeeded\n",
@@ -367,20 +363,64 @@ GNUNET_FS_search_probe_progress_ (void *cls,
 /**
  * Task run periodically to remind clients that a probe is active.
  *
- * @param cls the 'struct GNUNET_FS_SearchResult' that we are probing for
+ * @param cls the `struct GNUNET_FS_SearchResult` that we are probing for
  * @param tc scheduler context
  */
 static void
-probe_ping_task (void *cls,
-                 const struct GNUNET_SCHEDULER_TaskContext *tc)
+probe_ping_task_cb (void *cls,
+                    const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct GNUNET_FS_SearchResult *sr = cls;
+  struct GNUNET_FS_Handle *h = cls;
+  struct GNUNET_FS_SearchResult *sr;
 
-  signal_probe_result (sr);
-  sr->probe_ping_task
+  for (sr = h->probes_head; NULL != sr; sr = sr->next)
+    if (NULL != sr->probe_ctx->client)
+      signal_probe_result (sr);
+  h->probe_ping_task
     = GNUNET_SCHEDULER_add_delayed (GNUNET_FS_PROBE_UPDATE_FREQUENCY,
-                                    &probe_ping_task,
-                                    sr);
+                                    &probe_ping_task_cb,
+                                    h);
+}
+
+
+/**
+ * Start the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+static void
+start_probe_ping_task (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_Handle *h = sr->h;
+
+  GNUNET_CONTAINER_DLL_insert (h->probes_head,
+                               h->probes_tail,
+                               sr);
+  if (GNUNET_SCHEDULER_NO_TASK == h->probe_ping_task)
+    h->probe_ping_task
+      = GNUNET_SCHEDULER_add_now (&probe_ping_task_cb,
+                                  h);
+}
+
+
+/**
+ * Stop the ping task for this search result.
+ *
+ * @param sr result to start pinging for.
+ */
+void
+GNUNET_FS_stop_probe_ping_task_ (struct GNUNET_FS_SearchResult *sr)
+{
+  struct GNUNET_FS_Handle *h = sr->h;
+
+  GNUNET_CONTAINER_DLL_remove (h->probes_head,
+                               h->probes_tail,
+                               sr);
+  if (NULL == h->probes_head)
+  {
+    GNUNET_SCHEDULER_cancel (h->probe_ping_task);
+    h->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
+  }
 }
 
 
@@ -431,9 +471,7 @@ GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr)
                                 len, sr->anonymity,
                                 GNUNET_FS_DOWNLOAD_NO_TEMPORARIES |
                                 GNUNET_FS_DOWNLOAD_IS_PROBE, sr, NULL);
-  sr->probe_ping_task
-    = GNUNET_SCHEDULER_add_now (&probe_ping_task,
-                                sr);
+  start_probe_ping_task (sr);
 }
 
 
@@ -480,11 +518,7 @@ GNUNET_FS_search_stop_probe_ (struct GNUNET_FS_SearchResult *sr)
   {
     GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
     sr->probe_ctx = NULL;
-  }
-  if (GNUNET_SCHEDULER_NO_TASK != sr->probe_ping_task)
-  {
-    GNUNET_SCHEDULER_cancel (sr->probe_ping_task);
-    sr->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
   }
   if (GNUNET_SCHEDULER_NO_TASK != sr->probe_cancel_task)
   {
@@ -1410,11 +1444,7 @@ search_result_freeze_probes (void *cls,
   {
     GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES);
     sr->probe_ctx = NULL;
-  }
-  if (GNUNET_SCHEDULER_NO_TASK != sr->probe_ping_task)
-  {
-    GNUNET_SCHEDULER_cancel (sr->probe_ping_task);
-    sr->probe_ping_task = GNUNET_SCHEDULER_NO_TASK;
+    GNUNET_FS_stop_probe_ping_task_ (sr);
   }
   if (GNUNET_SCHEDULER_NO_TASK != sr->probe_cancel_task)
   {
@@ -1692,7 +1722,6 @@ search_result_free (void *cls,
   }
   GNUNET_break (NULL == sr->probe_ctx);
   GNUNET_break (GNUNET_SCHEDULER_NO_TASK == sr->probe_cancel_task);
-  GNUNET_break (GNUNET_SCHEDULER_NO_TASK == sr->probe_ping_task);
   GNUNET_break (NULL == sr->client_info);
   GNUNET_free_non_null (sr->serialization);
   GNUNET_FS_uri_destroy (sr->uri);
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index aa2f2076c..9f3ff0d12 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -88,8 +88,10 @@ struct GNUNET_FS_Uri;
  * @param is_mandatory is the keyword mandatory (in a search)
  * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to abort
  */
-typedef int (*GNUNET_FS_KeywordIterator) (void *cls, const char *keyword,
-                                          int is_mandatory);
+typedef int
+(*GNUNET_FS_KeywordIterator) (void *cls,
+                              const char *keyword,
+                              int is_mandatory);