-doxygen, code cleanup

1 files changed, 1710 insertions, 0 deletions

diff --git a/src/hostlist/gnunet-daemon-hostlist_client.c b/src/hostlist/gnunet-daemon-hostlist_client.c
new file mode 100644
index 000000000..20bd8bedd
--- /dev/null
+++ b/src/hostlist/gnunet-daemon-hostlist_client.c
@@ -0,0 +1,1710 @@
+/*
+     This file is part of GNUnet.
+     (C) 2001-2010, 2014 Christian Grothoff (and other contributing authors)
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+/**
+ * @file hostlist/gnunet-daemon-hostlist_client.c
+ * @brief hostlist support.  Downloads HELLOs via HTTP.
+ * @author Christian Grothoff
+ * @author Matthias Wachs
+ */
+#include "platform.h"
+#include "gnunet-daemon-hostlist_client.h"
+#include "gnunet_hello_lib.h"
+#include "gnunet_statistics_service.h"
+#include "gnunet_transport_service.h"
+#include "gnunet-daemon-hostlist.h"
+#include <curl/curl.h>
+
+
+/**
+ * Number of connections that we must have to NOT download
+ * hostlists anymore.
+ */
+#define MIN_CONNECTIONS 4
+
+/**
+ * Maximum number of hostlist that are saved
+ */
+#define MAX_NUMBER_HOSTLISTS 30
+
+/**
+ * Time interval hostlists are saved to disk
+ */
+#define SAVING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
+
+/**
+ * Time interval between two hostlist tests
+ */
+#define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)
+
+/**
+ * Time interval for download dispatcher before a download is re-scheduled
+ */
+#define WAITING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
+
+/**
+ * Defines concerning the hostlist quality metric
+ */
+
+/**
+ * Initial quality of a new created hostlist
+ */
+#define HOSTLIST_INITIAL 10000
+
+/**
+ * Value subtracted each time a hostlist download fails
+ */
+#define HOSTLIST_FAILED_DOWNLOAD 100
+
+/**
+ * Value added each time a hostlist download is successful
+ */
+#define HOSTLIST_SUCCESSFUL_DOWNLOAD 100
+
+/**
+ * Value added for each valid HELLO recived during a hostlist download
+ */
+#define HOSTLIST_SUCCESSFUL_HELLO 1
+
+
+
+/**
+ * A single hostlist obtained by hostlist advertisements
+ */
+struct Hostlist
+{
+  /**
+   * previous entry, used to manage entries in a double linked list
+   */
+  struct Hostlist *prev;
+
+  /**
+   * next entry, used to manage entries in a double linked list
+   */
+  struct Hostlist *next;
+
+  /**
+   * URI where hostlist can be obtained
+   */
+  const char *hostlist_uri;
+
+  /**
+   * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
+   * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
+   * intial value = HOSTLIST_INITIAL
+   * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
+   * increased every successful download by number of obtained HELLO messages
+   * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
+   */
+  uint64_t quality;
+
+  /**
+   * Time the hostlist advertisement was recieved and the entry was created
+   */
+  struct GNUNET_TIME_Absolute time_creation;
+
+  /**
+   * Last time the hostlist was obtained
+   */
+  struct GNUNET_TIME_Absolute time_last_usage;
+
+  /**
+   * Number of HELLO messages obtained during last download
+   */
+  uint32_t hello_count;
+
+  /**
+   * Number of times the hostlist was successfully obtained
+   */
+  uint32_t times_used;
+
+};
+
+
+/**
+ * Our configuration.
+ */
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
+ * Statistics handle.
+ */
+static struct GNUNET_STATISTICS_Handle *stats;
+
+/**
+ * Transport handle.
+ */
+static struct GNUNET_TRANSPORT_Handle *transport;
+
+/**
+ * Proxy hostname or ip we are using (can be NULL).
+ */
+static char *proxy;
+
+/**
+ * Proxy username we are using (can be NULL).
+ */
+static char *proxy_username;
+
+/**
+ * Proxy password we are using (can be NULL).
+ */
+static char *proxy_password;
+
+/**
+ * Proxy type we are using (can be NULL).
+ */
+static curl_proxytype proxy_type;
+
+/**
+ * Number of bytes valid in 'download_buffer'.
+ */
+static size_t download_pos;
+
+/**
+ * Current URL that we are using.
+ */
+static char *current_url;
+
+/**
+ * Current CURL handle.
+ */
+static CURL *curl;
+
+/**
+ * Current multi-CURL handle.
+ */
+static CURLM *multi;
+
+/**
+ * How many bytes did we download from the current hostlist URL?
+ */
+static uint32_t stat_bytes_downloaded;
+
+/**
+ * Amount of time we wait between hostlist downloads.
+ */
+static struct GNUNET_TIME_Relative hostlist_delay;
+
+/**
+ * ID of the task, checking if hostlist download should take plate
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;
+
+/**
+ * ID of the task downloading the hostlist
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_download;
+
+/**
+ * ID of the task saving the hostlsit in a regular intervall
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;
+
+/**
+ * ID of the task called to initiate a download
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;
+
+/**
+ * ID of the task controlling the locking between two hostlist tests
+ */
+static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;
+
+/**
+ * At what time MUST the current hostlist request be done?
+ */
+static struct GNUNET_TIME_Absolute end_time;
+
+/**
+ * Head of the linked list used to store hostlists
+ */
+static struct Hostlist *linked_list_head;
+
+/**
+ *  Tail of the linked list used to store hostlists
+ */
+static struct Hostlist *linked_list_tail;
+
+/**
+ *  Current hostlist used for downloading
+ */
+static struct Hostlist *current_hostlist;
+
+/**
+ *  Size of the linke list  used to store hostlists
+ */
+static unsigned int linked_list_size;
+
+/**
+ * Head of the linked list used to store hostlists
+ */
+static struct Hostlist *hostlist_to_test;
+
+/**
+ * Handle for our statistics GET operation.
+ */
+static struct GNUNET_STATISTICS_GetHandle *sget;
+
+/**
+ * Set to GNUNET_YES if the current URL had some problems.
+ */
+static int stat_bogus_url;
+
+/**
+ * Value controlling if a hostlist is tested at the moment
+ */
+static int stat_testing_hostlist;
+
+/**
+ * Value controlling if a hostlist testing is allowed at the moment
+ */
+static int stat_testing_allowed;
+
+/**
+ * Value controlling if a hostlist download is running at the moment
+ */
+static int stat_download_in_progress;
+
+/**
+ * Value saying if a preconfigured bootstrap server is used
+ */
+static unsigned int stat_use_bootstrap;
+
+/**
+ * Set if we are allowed to learn new hostlists and use them
+ */
+static int stat_learning;
+
+/**
+ * Value saying if hostlist download was successful
+ */
+static unsigned int stat_download_successful;
+
+/**
+ * Value saying how many valid HELLO messages were obtained during download
+ */
+static unsigned int stat_hellos_obtained;
+
+/**
+ * Number of active connections (according to core service).
+ */
+static unsigned int stat_connection_count;
+
+
+/**
+ * Process downloaded bits by calling callback on each HELLO.
+ *
+ * @param ptr buffer with downloaded data
+ * @param size size of a record
+ * @param nmemb number of records downloaded
+ * @param ctx unused
+ * @return number of bytes that were processed (always size*nmemb)
+ */
+static size_t
+callback_download (void *ptr,
+                   size_t size,
+                   size_t nmemb,
+                   void *ctx)
+{
+  static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
+  const char *cbuf = ptr;
+  const struct GNUNET_MessageHeader *msg;
+  size_t total;
+  size_t cpy;
+  size_t left;
+  uint16_t msize;
+
+  total = size * nmemb;
+  stat_bytes_downloaded += total;
+  if ((total == 0) || (stat_bogus_url))
+  {
+    return total;               /* ok, no data or bogus data */
+  }
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop
+                            ("# bytes downloaded from hostlist servers"),
+                            (int64_t) total, GNUNET_NO);
+  left = total;
+  while ((left > 0) || (download_pos > 0))
+  {
+    cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
+    memcpy (&download_buffer[download_pos], cbuf, cpy);
+    cbuf += cpy;
+    download_pos += cpy;
+    left -= cpy;
+    if (download_pos < sizeof (struct GNUNET_MessageHeader))
+    {
+      GNUNET_assert (0 == left);
+      break;
+    }
+    msg = (const struct GNUNET_MessageHeader *) download_buffer;
+    msize = ntohs (msg->size);
+    if (msize < sizeof (struct GNUNET_MessageHeader))
+    {
+      GNUNET_STATISTICS_update (stats,
+                                gettext_noop
+                                ("# invalid HELLOs downloaded from hostlist servers"),
+                                1, GNUNET_NO);
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  _("Invalid `%s' message received from hostlist at `%s'\n"),
+                  "HELLO", current_url);
+      stat_hellos_obtained++;
+      stat_bogus_url = 1;
+      return total;
+    }
+    if (download_pos < msize)
+    {
+      GNUNET_assert (left == 0);
+      break;
+    }
+    if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Received valid `%s' message from hostlist server.\n",
+                  "HELLO");
+      GNUNET_STATISTICS_update (stats,
+                                gettext_noop
+                                ("# valid HELLOs downloaded from hostlist servers"),
+                                1, GNUNET_NO);
+      stat_hellos_obtained++;
+      GNUNET_TRANSPORT_offer_hello (transport, msg, NULL, NULL);
+    }
+    else
+    {
+      GNUNET_STATISTICS_update (stats,
+                                gettext_noop
+                                ("# invalid HELLOs downloaded from hostlist servers"),
+                                1, GNUNET_NO);
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  _("Invalid `%s' message received from hostlist at `%s'\n"),
+                  "HELLO", current_url);
+      stat_bogus_url = GNUNET_YES;
+      stat_hellos_obtained++;
+      return total;
+    }
+    memmove (download_buffer, &download_buffer[msize], download_pos - msize);
+    download_pos -= msize;
+  }
+  return total;
+}
+
+
+/**
+ * Obtain a hostlist URL that we should use.
+ *
+ * @return NULL if there is no URL available
+ */
+static char *
+get_bootstrap_server ()
+{
+  char *servers;
+  char *ret;
+  size_t urls;
+  size_t pos;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
+                                             &servers))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
+                               "hostlist", "SERVERS");
+    return NULL;
+  }
+
+  urls = 0;
+  if (strlen (servers) > 0)
+  {
+    urls++;
+    pos = strlen (servers) - 1;
+    while (pos > 0)
+    {
+      if (servers[pos] == ' ')
+        urls++;
+      pos--;
+    }
+  }
+  if (urls == 0)
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
+                               "hostlist", "SERVERS");
+    GNUNET_free (servers);
+    return NULL;
+  }
+
+  urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
+  pos = strlen (servers) - 1;
+  while (pos > 0)
+  {
+    if (servers[pos] == ' ')
+    {
+      urls--;
+      servers[pos] = '\0';
+    }
+    if (urls == 0)
+    {
+      pos++;
+      break;
+    }
+    pos--;
+  }
+  ret = GNUNET_strdup (&servers[pos]);
+  GNUNET_free (servers);
+  return ret;
+}
+
+
+/**
+ * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
+ * @return uri to use, NULL if there is no URL available
+ */
+static char *
+download_get_url ()
+{
+  uint32_t index;
+  unsigned int counter;
+  struct Hostlist *pos;
+
+  if (GNUNET_NO == stat_learning)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Using preconfigured bootstrap server\n");
+    current_hostlist = NULL;
+    return get_bootstrap_server ();
+  }
+
+  if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Testing new advertised hostlist if it is obtainable\n");
+    current_hostlist = hostlist_to_test;
+    return GNUNET_strdup (hostlist_to_test->hostlist_uri);
+  }
+
+  if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Using preconfigured bootstrap server\n");
+    current_hostlist = NULL;
+    return get_bootstrap_server ();
+  }
+  index =
+      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
+  counter = 0;
+  pos = linked_list_head;
+  while (counter < index)
+  {
+    pos = pos->next;
+    counter++;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
+              pos->hostlist_uri);
+  current_hostlist = pos;
+  return GNUNET_strdup (pos->hostlist_uri);
+}
+
+
+#define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)
+
+
+/**
+ * Method to save hostlist to a file during hostlist client shutdown
+ *
+ * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
+ */
+static void
+save_hostlist_file (int shutdown);
+
+
+/**
+ * Add val2 to val1 with overflow check
+ *
+ * @param val1 value 1
+ * @param val2 value 2
+ * @return result
+ */
+static uint64_t
+checked_add (uint64_t val1,
+             uint64_t val2)
+{
+  static uint64_t temp;
+  static uint64_t maxv;
+
+  maxv = 0;
+  maxv--;
+
+  temp = val1 + val2;
+  if (temp < val1)
+    return maxv;
+  return temp;
+}
+
+
+/**
+ * Subtract val2 from val1 with underflow check
+ *
+ * @param val1 value 1
+ * @param val2 value 2
+ * @return result
+ */
+static uint64_t
+checked_sub (uint64_t val1,
+             uint64_t val2)
+{
+  if (val1 <= val2)
+    return 0;
+  return (val1 - val2);
+}
+
+
+/**
+ * Method to check if  a URI is in hostlist linked list
+ *
+ * @param uri uri to check
+ * @return #GNUNET_YES if existing in linked list, #GNUNET_NO if not
+ */
+static int
+linked_list_contains (const char *uri)
+{
+  struct Hostlist *pos;
+
+  pos = linked_list_head;
+  while (pos != NULL)
+  {
+    if (0 == strcmp (pos->hostlist_uri, uri))
+      return GNUNET_YES;
+    pos = pos->next;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Method returning the hostlist element with the lowest quality in the datastore
+ * @return hostlist with lowest quality
+ */
+static struct Hostlist *
+linked_list_get_lowest_quality ()
+{
+  struct Hostlist *pos;
+  struct Hostlist *lowest;
+
+  if (linked_list_size == 0)
+    return NULL;
+  lowest = linked_list_head;
+  pos = linked_list_head->next;
+  while (pos != NULL)
+  {
+    if (pos->quality < lowest->quality)
+      lowest = pos;
+    pos = pos->next;
+  }
+  return lowest;
+}
+
+
+/**
+ * Method to insert a hostlist into the datastore. If datastore
+ * contains maximum number of elements, the elements with lowest
+ * quality is dismissed
+ */
+static void
+insert_hostlist ()
+{
+  struct Hostlist *lowest_quality;
+
+  if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
+  {
+    /* No free entries available, replace existing entry  */
+    lowest_quality = linked_list_get_lowest_quality ();
+    GNUNET_assert (lowest_quality != NULL);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
+                lowest_quality->hostlist_uri,
+                (unsigned long long) lowest_quality->quality);
+    GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
+                                 lowest_quality);
+    linked_list_size--;
+    GNUNET_free (lowest_quality);
+  }
+  GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
+                               hostlist_to_test);
+  linked_list_size++;
+  GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
+                         linked_list_size, GNUNET_NO);
+  stat_testing_hostlist = GNUNET_NO;
+}
+
+
+/**
+ * Method updating hostlist statistics
+ */
+static void
+update_hostlist ()
+{
+  char *stat;
+
+  if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
+      ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Updating hostlist statics for URI `%s'\n",
+                current_hostlist->hostlist_uri);
+    current_hostlist->hello_count = stat_hellos_obtained;
+    current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
+    current_hostlist->quality =
+        checked_add (current_hostlist->quality,
+                     (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
+    if (GNUNET_YES == stat_download_successful)
+    {
+      current_hostlist->times_used++;
+      current_hostlist->quality =
+          checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
+      GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
+                       current_hostlist->hostlist_uri);
+
+      GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
+      GNUNET_free (stat);
+    }
+    else
+      current_hostlist->quality =
+          checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
+  }
+  current_hostlist = NULL;
+  /* Alternating the usage of preconfigured and learned hostlists */
+
+  if (stat_testing_hostlist == GNUNET_YES)
+    return;
+
+  if (GNUNET_YES == stat_learning)
+  {
+    if (stat_use_bootstrap == GNUNET_YES)
+      stat_use_bootstrap = GNUNET_NO;
+    else
+      stat_use_bootstrap = GNUNET_YES;
+  }
+  else
+    stat_use_bootstrap = GNUNET_YES;
+}
+
+
+/**
+ * Clean up the state from the task that downloaded the
+ * hostlist and schedule the next task.
+ */
+static void
+clean_up ()
+{
+  CURLMcode mret;
+
+  if ((stat_testing_hostlist == GNUNET_YES) &&
+      (GNUNET_NO == stat_download_successful) && (NULL != hostlist_to_test))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _
+                ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
+                hostlist_to_test->hostlist_uri);
+  }
+
+  if (stat_testing_hostlist == GNUNET_YES)
+  {
+    stat_testing_hostlist = GNUNET_NO;
+  }
+  if (NULL != hostlist_to_test)
+  {
+    GNUNET_free (hostlist_to_test);
+    hostlist_to_test = NULL;
+  }
+
+  if (multi != NULL)
+  {
+    mret = curl_multi_remove_handle (multi, curl);
+    if (mret != CURLM_OK)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                  "curl_multi_remove_handle", __FILE__, __LINE__,
+                  curl_multi_strerror (mret));
+    }
+    mret = curl_multi_cleanup (multi);
+    if (mret != CURLM_OK)
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                  "curl_multi_cleanup", __FILE__, __LINE__,
+                  curl_multi_strerror (mret));
+    multi = NULL;
+  }
+  if (curl != NULL)
+  {
+    curl_easy_cleanup (curl);
+    curl = NULL;
+  }
+  GNUNET_free_non_null (current_url);
+  current_url = NULL;
+  stat_bytes_downloaded = 0;
+  stat_download_in_progress = GNUNET_NO;
+}
+
+
+/**
+ * Task that is run when we are ready to receive more data from the hostlist
+ * server.
+ *
+ * @param cls closure, unused
+ * @param tc task context, unused
+ */
+static void
+task_download (void *cls,
+               const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+/**
+ * Ask CURL for the select set and then schedule the
+ * receiving task with the scheduler.
+ */
+static void
+download_prepare ()
+{
+  CURLMcode mret;
+  fd_set rs;
+  fd_set ws;
+  fd_set es;
+  int max;
+  struct GNUNET_NETWORK_FDSet *grs;
+  struct GNUNET_NETWORK_FDSet *gws;
+  long timeout;
+  struct GNUNET_TIME_Relative rtime;
+
+  max = -1;
+  FD_ZERO (&rs);
+  FD_ZERO (&ws);
+  FD_ZERO (&es);
+  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
+  if (mret != CURLM_OK)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                "curl_multi_fdset", __FILE__, __LINE__,
+                curl_multi_strerror (mret));
+    clean_up ();
+    return;
+  }
+  mret = curl_multi_timeout (multi, &timeout);
+  if (mret != CURLM_OK)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                "curl_multi_timeout", __FILE__, __LINE__,
+                curl_multi_strerror (mret));
+    clean_up ();
+    return;
+  }
+  rtime =
+      GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
+                                GNUNET_TIME_relative_multiply
+                                (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
+  grs = GNUNET_NETWORK_fdset_create ();
+  gws = GNUNET_NETWORK_fdset_create ();
+  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
+  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Scheduling task for hostlist download using cURL\n");
+  ti_download =
+      GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
+                                   rtime, grs, gws,
+                                   &task_download, multi);
+  GNUNET_NETWORK_fdset_destroy (gws);
+  GNUNET_NETWORK_fdset_destroy (grs);
+}
+
+
+/**
+ * Task that is run when we are ready to receive more data from the hostlist
+ * server.
+ *
+ * @param cls closure, unused
+ * @param tc task context, unused
+ */
+static void
+task_download (void *cls,
+               const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  int running;
+  struct CURLMsg *msg;
+  CURLMcode mret;
+
+  ti_download = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Shutdown requested while trying to download hostlist from `%s'\n",
+                current_url);
+    update_hostlist ();
+    clean_up ();
+    return;
+  }
+  if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("Timeout trying to download hostlist from `%s'\n"),
+                current_url);
+    update_hostlist ();
+    clean_up ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Ready for processing hostlist client request\n");
+  do
+  {
+    running = 0;
+    if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  _("Download limit of %u bytes exceeded, stopping download\n"),
+                  MAX_BYTES_PER_HOSTLISTS);
+      clean_up ();
+      return;
+    }
+    mret = curl_multi_perform (multi, &running);
+    if (running == 0)
+    {
+      do
+      {
+        msg = curl_multi_info_read (multi, &running);
+        GNUNET_break (msg != NULL);
+        if (msg == NULL)
+          break;
+        switch (msg->msg)
+        {
+        case CURLMSG_DONE:
+          if ((msg->data.result != CURLE_OK) &&
+              (msg->data.result != CURLE_GOT_NOTHING))
+            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                        _("Download of hostlist from `%s' failed: `%s'\n"),
+                        current_url,
+                        curl_easy_strerror (msg->data.result));
+          else
+          {
+            GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                        _("Download of hostlist `%s' completed.\n"),
+                        current_url);
+            stat_download_successful = GNUNET_YES;
+            update_hostlist ();
+            if (GNUNET_YES == stat_testing_hostlist)
+            {
+              GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                          _
+                          ("Adding successfully tested hostlist `%s' datastore.\n"),
+                          current_url);
+              insert_hostlist ();
+              hostlist_to_test = NULL;
+              stat_testing_hostlist = GNUNET_NO;
+            }
+          }
+          clean_up ();
+          return;
+        default:
+          break;
+        }
+
+      }
+      while ((running > 0));
+    }
+  }
+  while (mret == CURLM_CALL_MULTI_PERFORM);
+
+  if (mret != CURLM_OK)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
+                "curl_multi_perform", __FILE__, __LINE__,
+                curl_multi_strerror (mret));
+    clean_up ();
+  }
+  download_prepare ();
+}
+
+
+/**
+ * Main function that will download a hostlist and process its
+ * data.
+ */
+static void
+download_hostlist ()
+{
+  CURLcode ret;
+  CURLMcode mret;
+
+
+  current_url = download_get_url ();
+  if (current_url == NULL)
+    return;
+  curl = curl_easy_init ();
+  multi = NULL;
+  if (curl == NULL)
+  {
+    GNUNET_break (0);
+    clean_up ();
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
+              _("Bootstrapping using hostlist at `%s'.\n"), current_url);
+
+  stat_download_in_progress = GNUNET_YES;
+  stat_download_successful = GNUNET_NO;
+  stat_hellos_obtained = 0;
+  stat_bytes_downloaded = 0;
+
+  GNUNET_STATISTICS_update (stats,
+                            gettext_noop ("# hostlist downloads initiated"), 1,
+                            GNUNET_NO);
+  if (proxy != NULL)
+  {
+    CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
+    CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
+    if (NULL != proxy_username)
+      CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
+    if (NULL != proxy_password)
+      CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
+  }
+  download_pos = 0;
+  stat_bogus_url = 0;
+  CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
+  if (ret != CURLE_OK)
+  {
+    clean_up ();
+    return;
+  }
+  CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
+  if (ret != CURLE_OK)
+  {
+    clean_up ();
+    return;
+  }
+  CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
+  CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+  CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+  CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
+  /* no need to abort if the above failed */
+  CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
+  if (ret != CURLE_OK)
+  {
+    clean_up ();
+    return;
+  }
+  CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
+#if 0
+  CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
+#endif
+  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  if (0 == strncmp (current_url, "http", 4))
+    CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
+  CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
+  CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
+  multi = curl_multi_init ();
+  if (multi == NULL)
+  {
+    GNUNET_break (0);
+    /* clean_up (); */
+    return;
+  }
+  mret = curl_multi_add_handle (multi, curl);
+  if (mret != CURLM_OK)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                "curl_multi_add_handle", __FILE__, __LINE__,
+                curl_multi_strerror (mret));
+    mret = curl_multi_cleanup (multi);
+    if (mret != CURLM_OK)
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
+                  "curl_multi_cleanup", __FILE__, __LINE__,
+                  curl_multi_strerror (mret));
+    multi = NULL;
+    clean_up ();
+    return;
+  }
+  end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
+  download_prepare ();
+}
+
+
+static void
+task_download_dispatcher (void *cls,
+                          const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
+  if (GNUNET_NO == stat_download_in_progress)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
+    download_hostlist ();
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Download in progess, have to wait...\n");
+    ti_download_dispatcher_task =
+        GNUNET_SCHEDULER_add_delayed (WAITING_INTERVAL,
+                                      &task_download_dispatcher, NULL);
+  }
+}
+
+
+/**
+ * Task that checks if we should try to download a hostlist.
+ * If so, we initiate the download, otherwise we schedule
+ * this task again for a later time.
+ */
+static void
+task_check (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  static int once;
+  struct GNUNET_TIME_Relative delay;
+
+  ti_check_download = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  if (stats == NULL)
+  {
+    curl_global_cleanup ();
+    return;                     /* in shutdown */
+  }
+  if ( (stat_connection_count < MIN_CONNECTIONS) &&
+       (GNUNET_SCHEDULER_NO_TASK == ti_download_dispatcher_task) )
+    ti_download_dispatcher_task =
+        GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
+
+  delay = hostlist_delay;
+  if (0 == hostlist_delay.rel_value_us)
+    hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
+  else
+    hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
+  if (hostlist_delay.rel_value_us >
+      GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
+    hostlist_delay =
+        GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
+                                       (1 + stat_connection_count));
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop
+                         ("# milliseconds between hostlist downloads"),
+                         hostlist_delay.rel_value_us / 1000LL,
+                         GNUNET_YES);
+  if (0 == once)
+  {
+    delay = GNUNET_TIME_UNIT_ZERO;
+    once = 1;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Have %u/%u connections.  Will consider downloading hostlist in %s\n"),
+              stat_connection_count, MIN_CONNECTIONS,
+              GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
+  ti_check_download = GNUNET_SCHEDULER_add_delayed (delay, &task_check, NULL);
+}
+
+
+/**
+ * This tasks sets hostlist testing to allowed after intervall between to testings is reached
+ *
+ * @param cls closure
+ * @param tc TaskContext
+ */
+static void
+task_testing_intervall_reset (void *cls,
+                              const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  stat_testing_allowed = GNUNET_OK;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testing new hostlist advertisements is allowed again\n");
+}
+
+
+/**
+ * Task that writes hostlist entries to a file on a regular base
+ *
+ * @param cls closure
+ * @param tc TaskContext
+ */
+static void
+task_hostlist_saving (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
+    return;
+  save_hostlist_file (GNUNET_NO);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hostlists will be saved to file again in %s\n",
+              GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVAL, GNUNET_YES));
+  ti_saving_task =
+      GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL, &task_hostlist_saving,
+                                    NULL);
+}
+
+
+/**
+ * Method called whenever a given peer connects.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ */
+static void
+handler_connect (void *cls, const struct GNUNET_PeerIdentity *peer)
+{
+  GNUNET_assert (stat_connection_count < UINT_MAX);
+  stat_connection_count++;
+  GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), 1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Method called whenever a given peer disconnects.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ */
+static void
+handler_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
+{
+  GNUNET_assert (stat_connection_count > 0);
+  stat_connection_count--;
+  GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), -1,
+                            GNUNET_NO);
+}
+
+
+/**
+ * Method called whenever an advertisement message arrives.
+ *
+ * @param cls closure (always NULL)
+ * @param peer the peer sending the message
+ * @param message the actual message
+ * @return #GNUNET_OK to keep the connection open,
+ *         #GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+handler_advertisement (void *cls, const struct GNUNET_PeerIdentity *peer,
+                       const struct GNUNET_MessageHeader *message)
+{
+  size_t size;
+  size_t uri_size;
+  const struct GNUNET_MessageHeader *incoming;
+  const char *uri;
+  struct Hostlist *hostlist;
+
+  GNUNET_assert (ntohs (message->type) ==
+                 GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
+  size = ntohs (message->size);
+  if (size <= sizeof (struct GNUNET_MessageHeader))
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  incoming = (const struct GNUNET_MessageHeader *) message;
+  uri = (const char *) &incoming[1];
+  uri_size = size - sizeof (struct GNUNET_MessageHeader);
+  if (uri[uri_size - 1] != '\0')
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Hostlist client recieved advertisement from `%s' containing URI `%s'\n",
+              GNUNET_i2s (peer), uri);
+  if (GNUNET_NO != linked_list_contains (uri))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "URI `%s' is already known\n", uri);
+    return GNUNET_OK;
+  }
+
+  if (GNUNET_NO == stat_testing_allowed)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_YES == stat_testing_hostlist)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Currently not accepting new advertisements: we are already testing a hostlist\n");
+    return GNUNET_SYSERR;
+  }
+
+  hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
+  hostlist->hostlist_uri = (const char *) &hostlist[1];
+  memcpy (&hostlist[1], uri, uri_size);
+  hostlist->time_creation = GNUNET_TIME_absolute_get ();
+  hostlist->quality = HOSTLIST_INITIAL;
+  hostlist_to_test = hostlist;
+
+  stat_testing_hostlist = GNUNET_YES;
+  stat_testing_allowed = GNUNET_NO;
+  ti_testing_intervall_task =
+      GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
+                                    &task_testing_intervall_reset, NULL);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testing new hostlist advertisements is locked for the next %s\n",
+              GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
+                                                      GNUNET_YES));
+
+  ti_download_dispatcher_task =
+      GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Continuation called by the statistics code once
+ * we go the stat.  Initiates hostlist download scheduling.
+ *
+ * @param cls closure
+ * @param success #GNUNET_OK if statistics were
+ *        successfully obtained, #GNUNET_SYSERR if not.
+ */
+static void
+primary_task (void *cls, int success)
+{
+  sget = NULL;
+  GNUNET_assert (stats != NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Statistics request done, scheduling hostlist download\n");
+  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
+}
+
+
+/**
+ * We've received the previous delay value from statistics.  Remember it.
+ *
+ * @param cls NULL, unused
+ * @param subsystem should be "hostlist", unused
+ * @param name will be "milliseconds between hostlist downloads", unused
+ * @param value previous delay value, in milliseconds (!)
+ * @param is_persistent unused, will be #GNUNET_YES
+ */
+static int
+process_stat (void *cls,
+              const char *subsystem,
+              const char *name,
+              uint64_t value,
+              int is_persistent)
+{
+  hostlist_delay.rel_value_us = value * 1000LL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initial time between hostlist downloads is %s\n",
+              GNUNET_STRINGS_relative_time_to_string (hostlist_delay, GNUNET_YES));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Method to load persistent hostlist file during hostlist client startup
+ */
+static void
+load_hostlist_file ()
+{
+  char *filename;
+  char *uri;
+  char *emsg;
+  struct Hostlist *hostlist;
+  uint32_t times_used;
+  uint32_t hellos_returned;
+  uint64_t quality;
+  uint64_t last_used;
+  uint64_t created;
+  uint32_t counter;
+  struct GNUNET_BIO_ReadHandle *rh;
+
+  uri = NULL;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
+                                               &filename))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
+                               "hostlist", "HOSTLISTFILE");
+    return;
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading saved hostlist entries from file `%s' \n"),
+              filename);
+  if (GNUNET_NO == GNUNET_DISK_file_test (filename))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Hostlist file `%s' does not exist\n"), filename);
+    GNUNET_free (filename);
+    return;
+  }
+
+  rh = GNUNET_BIO_read_open (filename);
+  if (NULL == rh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Could not open file `%s' for reading to load hostlists: %s\n"),
+                filename, STRERROR (errno));
+    GNUNET_free (filename);
+    return;
+  }
+
+  counter = 0;
+  while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
+         (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used))
+         && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
+         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
+  {
+    hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
+    hostlist->hello_count = hellos_returned;
+    hostlist->hostlist_uri = (const char *) &hostlist[1];
+    memcpy (&hostlist[1], uri, strlen (uri) + 1);
+    hostlist->quality = quality;
+    hostlist->time_creation.abs_value_us = created;
+    hostlist->time_last_usage.abs_value_us = last_used;
+    GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
+    linked_list_size++;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Added hostlist entry eith URI `%s' \n",
+                hostlist->hostlist_uri);
+    GNUNET_free (uri);
+    uri = NULL;
+    counter++;
+    if (counter >= MAX_NUMBER_HOSTLISTS)
+      break;
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("%u hostlist URIs loaded from file\n"),
+              counter);
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop ("# hostlist URIs read from file"),
+                         counter, GNUNET_YES);
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop ("# advertised hostlist URIs"),
+                         linked_list_size, GNUNET_NO);
+
+  GNUNET_free_non_null (uri);
+  emsg = NULL;
+  (void) GNUNET_BIO_read_close (rh, &emsg);
+  if (emsg != NULL)
+    GNUNET_free (emsg);
+  GNUNET_free (filename);
+}
+
+
+/**
+ * Method to save persistent hostlist file during hostlist client shutdown
+ *
+ * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
+ */
+static void
+save_hostlist_file (int shutdown)
+{
+  char *filename;
+  struct Hostlist *pos;
+  struct GNUNET_BIO_WriteHandle *wh;
+  int ok;
+  uint32_t counter;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
+                                               &filename))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
+                               "hostlist", "HOSTLISTFILE");
+    return;
+  }
+  if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
+  {
+    GNUNET_free (filename);
+    return;
+  }
+  wh = GNUNET_BIO_write_open (filename);
+  if (NULL == wh)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _
+                ("Could not open file `%s' for writing to save hostlists: %s\n"),
+                filename, STRERROR (errno));
+    GNUNET_free (filename);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Writing %u hostlist URIs to `%s'\n"),
+              linked_list_size, filename);
+  /* add code to write hostlists to file using bio */
+  ok = GNUNET_YES;
+  counter = 0;
+  while (NULL != (pos = linked_list_head))
+  {
+    if (GNUNET_YES == shutdown)
+    {
+      GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
+      linked_list_size--;
+    }
+    if (GNUNET_YES == ok)
+    {
+      if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
+          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
+          (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
+          (GNUNET_OK !=
+           GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
+          (GNUNET_OK !=
+           GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
+          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _("Error writing hostlist URIs to file `%s'\n"), filename);
+        ok = GNUNET_NO;
+      }
+    }
+
+    if (GNUNET_YES == shutdown)
+      GNUNET_free (pos);
+    counter++;
+    if (counter >= MAX_NUMBER_HOSTLISTS)
+      break;
+  }
+  GNUNET_STATISTICS_set (stats,
+                         gettext_noop ("# hostlist URIs written to file"),
+                         counter, GNUNET_YES);
+
+  if (GNUNET_OK != GNUNET_BIO_write_close (wh))
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                _("Error writing hostlist URIs to file `%s'\n"), filename);
+  GNUNET_free (filename);
+}
+
+
+/**
+ * Start downloading hostlists from hostlist servers as necessary.
+ *
+ * @param c configuration to use
+ * @param st statistics handle to use
+ * @param ch[OUT] set to handler for CORE connect events
+ * @param dh[OUT] set to handler for CORE disconnect events
+ * @param msgh[OUT] set to handler for CORE advertisement messages
+ * @param learn should we learn hostlist URLs from CORE
+ * @return #GNUNET_OK on success
+ */
+int
+GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
+                              struct GNUNET_STATISTICS_Handle *st,
+                              GNUNET_CORE_ConnectEventHandler *ch,
+                              GNUNET_CORE_DisconnectEventHandler *dh,
+                              GNUNET_CORE_MessageCallback *msgh,
+                              int learn)
+{
+  char *filename;
+  char *proxytype_str;
+  int result;
+
+  GNUNET_assert (NULL != st);
+  if (0 != curl_global_init (CURL_GLOBAL_WIN32))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  transport = GNUNET_TRANSPORT_connect (c, NULL, NULL, NULL, NULL, NULL);
+  if (NULL == transport)
+  {
+    curl_global_cleanup ();
+    return GNUNET_SYSERR;
+  }
+  cfg = c;
+  stats = st;
+
+  /* Read proxy configuration */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
+      "HOSTLIST", "PROXY", &proxy))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                     "Found proxy host: `%s'\n",
+                     proxy);
+    /* proxy username */
+    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
+        "HOSTLIST", "PROXY_USERNAME", &proxy_username))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                       "Found proxy username name: `%s'\n",
+                       proxy_username);
+    }
+
+    /* proxy password */
+    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
+        "HOSTLIST", "PROXY_PASSWORD", &proxy_password))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                       "Found proxy password name: `%s'\n",
+                       proxy_password);
+    }
+
+    /* proxy type */
+    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
+        "HOSTLIST", "PROXY_TYPE", &proxytype_str))
+    {
+      GNUNET_STRINGS_utf8_toupper (proxytype_str, proxytype_str);
+
+      proxy_type = CURLPROXY_HTTP;
+      if (0 == strcmp(proxytype_str, "HTTP"))
+        proxy_type = CURLPROXY_HTTP;
+      else if (0 == strcmp(proxytype_str, "HTTP_1_0"))
+        proxy_type = CURLPROXY_HTTP_1_0;
+      else if (0 == strcmp(proxytype_str, "SOCKS4"))
+        proxy_type = CURLPROXY_SOCKS4;
+      else if (0 == strcmp(proxytype_str, "SOCKS5"))
+        proxy_type = CURLPROXY_SOCKS5;
+      else if (0 == strcmp(proxytype_str, "SOCKS4A"))
+        proxy_type = CURLPROXY_SOCKS4A;
+      else if (0 == strcmp(proxytype_str, "SOCKS5_HOSTNAME"))
+        proxy_type = CURLPROXY_SOCKS5_HOSTNAME ;
+      else
+      {
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+             _("Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
+             proxytype_str);
+        GNUNET_free (proxytype_str);
+
+        GNUNET_free (proxy);
+        proxy = NULL;
+        GNUNET_free_non_null (proxy_username);
+        proxy_username = NULL;
+        GNUNET_free_non_null (proxy_password);
+        proxy_password = NULL;
+
+        return GNUNET_SYSERR;
+      }
+    }
+    GNUNET_free_non_null (proxytype_str);
+  }
+
+  stat_learning = learn;
+  *ch = &handler_connect;
+  *dh = &handler_disconnect;
+  linked_list_head = NULL;
+  linked_list_tail = NULL;
+  stat_use_bootstrap = GNUNET_YES;
+  stat_testing_hostlist = GNUNET_NO;
+  stat_testing_allowed = GNUNET_YES;
+
+  if (GNUNET_YES == stat_learning)
+  {
+    *msgh = &handler_advertisement;
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Learning is enabled on this peer\n"));
+    load_hostlist_file ();
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Hostlists will be saved to file again in %s\n",
+                GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL, GNUNET_YES));
+    ti_saving_task =
+        GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL, &task_hostlist_saving,
+                                      NULL);
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Learning is not enabled on this peer\n"));
+    *msgh = NULL;
+    if (GNUNET_OK ==
+        GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST",
+                                                 "HOSTLISTFILE", &filename))
+    {
+      if (GNUNET_YES == GNUNET_DISK_file_test (filename))
+      {
+        result = remove (filename);
+        if (result == 0)
+          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                      _
+                      ("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
+                      filename);
+        else
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      _("Hostlist file `%s' could not be removed\n"), filename);
+      }
+    }
+    GNUNET_free (filename);
+  }
+  sget = GNUNET_STATISTICS_get (stats, "hostlist",
+                                gettext_noop
+                                ("# milliseconds between hostlist downloads"),
+                                GNUNET_TIME_UNIT_MINUTES, &primary_task, &process_stat,
+                                NULL);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Stop downloading hostlists from hostlist servers as necessary.
+ */
+void
+GNUNET_HOSTLIST_client_stop ()
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Hostlist client shutdown\n");
+  if (NULL != sget)
+  {
+    GNUNET_STATISTICS_get_cancel (sget);
+    sget = NULL;
+  }
+  stats = NULL;
+  if (GNUNET_YES == stat_learning)
+    save_hostlist_file (GNUNET_YES);
+  if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
+  {
+    GNUNET_SCHEDULER_cancel (ti_saving_task);
+    ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+
+  if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+    GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
+    ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
+  {
+    GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
+    ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (ti_download != GNUNET_SCHEDULER_NO_TASK)
+  {
+    GNUNET_SCHEDULER_cancel (ti_download);
+    ti_download = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
+  {
+    GNUNET_SCHEDULER_cancel (ti_check_download);
+    ti_check_download = GNUNET_SCHEDULER_NO_TASK;
+    curl_global_cleanup ();
+  }
+  if (NULL != transport)
+  {
+    GNUNET_TRANSPORT_disconnect (transport);
+    transport = NULL;
+  }
+  GNUNET_free_non_null (proxy);
+  proxy = NULL;
+  GNUNET_free_non_null (proxy_username);
+  proxy_username = NULL;
+  GNUNET_free_non_null (proxy_password);
+  proxy_password = NULL;
+
+  cfg = NULL;
+}
+
+/* end of gnunet-daemon-hostlist_client.c */