more testcases, allow location uris in fs_download

7 files changed, 203 insertions, 58 deletions

diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index bbac950a0..c798e9947 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -135,7 +135,9 @@ check_PROGRAMS = \
  test_fs_uri \
  test_gnunet_service_fs_migration \
  test_gnunet_service_fs_p2p \
- perf_gnunet_service_fs_p2p
+ perf_gnunet_service_fs_p2p \
+ perf_gnunet_service_fs_p2p_dht \
+ perf_gnunet_service_fs_p2p_index
 
 
 if HAVE_PYTHON_PEXPECT
@@ -303,6 +305,22 @@ perf_gnunet_service_fs_p2p_LDADD = \
   $(top_builddir)/src/fs/libgnunetfs.la  \
   $(top_builddir)/src/util/libgnunetutil.la  
 
+perf_gnunet_service_fs_p2p_index_SOURCES = \
+ perf_gnunet_service_fs_p2p.c
+perf_gnunet_service_fs_p2p_index_LDADD = \
+  $(top_builddir)/src/fs/libgnunetfstest.a \
+  $(top_builddir)/src/testing/libgnunettesting.la \
+  $(top_builddir)/src/fs/libgnunetfs.la  \
+  $(top_builddir)/src/util/libgnunetutil.la  
+
+perf_gnunet_service_fs_p2p_dht_SOURCES = \
+ perf_gnunet_service_fs_p2p.c
+perf_gnunet_service_fs_p2p_dht_LDADD = \
+  $(top_builddir)/src/fs/libgnunetfstest.a \
+  $(top_builddir)/src/testing/libgnunettesting.la \
+  $(top_builddir)/src/fs/libgnunetfs.la  \
+  $(top_builddir)/src/util/libgnunetutil.la  
+
 
 do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g'
 
diff --git a/src/fs/fs.h b/src/fs/fs.h
index 9d662b267..68ed2184e 100644
--- a/src/fs/fs.h
+++ b/src/fs/fs.h
@@ -37,6 +37,18 @@
 
 /**
  * Maximum number of outgoing messages we queue per peer.
+ *
+ * Performance measurements for 2 peer setup for 50 MB file
+ * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1):
+ *
+ *   2: 1700 kb/s, 1372 kb/s
+ *   8: 2117 kb/s, 1284 kb/s, 1112 kb/s
+ *  16: 3500 kb/s, 3200 kb/s, 3388 kb/s
+ *  32: 3441 kb/s, 3163 kb/s, 3277 kb/s
+ * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s
+ *
+ * Conclusion: 16 seems to be a pretty good value (stable
+ * and high performance, no excessive memory use).
  */
 #define MAX_QUEUE_PER_PEER 16
 
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index c7596de14..2be58aa1a 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -32,7 +32,7 @@
 #include "fs.h"
 #include "fs_tree.h"
 
-#define DEBUG_DOWNLOAD GNUNET_YES
+#define DEBUG_DOWNLOAD GNUNET_NO
 
 /**
  * Determine if the given download (options and meta data) should cause
@@ -1719,9 +1719,11 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
   struct GNUNET_FS_ProgressInfo pi;
   struct GNUNET_FS_DownloadContext *dc;
 
-  GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
+  GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
+                 GNUNET_FS_uri_test_loc (uri) );
+                 
   if ( (offset + length < offset) ||
-       (offset + length > uri->data.chk.file_length) )
+       (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
     {      
       GNUNET_break (0);
       return NULL;
@@ -1762,7 +1764,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
   dc->anonymity = anonymity;
   dc->options = options;
   dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
-  dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
+  dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
   if ( (filename == NULL) &&
        (is_recursive_download (dc) ) )
     {
@@ -1787,7 +1789,9 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h,
   pi.value.download.specifics.start.meta = meta;
   GNUNET_FS_download_make_status_ (&pi, dc);
   schedule_block_download (dc, 
-                           &dc->uri->data.chk.chk,
+                           (dc->uri->type == chk) 
+                           ? &dc->uri->data.chk.chk
+                           : &dc->uri->data.loc.fi.chk,
                            0, 
                            1 /* 0 == CHK, 1 == top */); 
   GNUNET_FS_download_sync_ (dc);
diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c
index dd646eafe..f91092c4e 100644
--- a/src/fs/fs_test_lib.c
+++ b/src/fs/fs_test_lib.c
@@ -27,6 +27,7 @@
  * @author Christian Grothoff
  */
 #include "platform.h"
+#include "fs.h"
 #include "fs_test_lib.h"
 #include "gnunet_testing_lib.h"
 
@@ -106,6 +107,11 @@ struct GNUNET_FS_TestDaemon
   struct GNUNET_FS_Uri *publish_uri;
 
   /**
+   * Name of the temporary file used, or NULL for none.
+   */ 
+  char *publish_tmp_file;
+
+  /**
    * Scheduler to use (for download_cont).
    */
   struct GNUNET_SCHEDULER_Handle *download_sched;
@@ -562,6 +568,11 @@ GNUNET_FS_TEST_daemons_stop (struct GNUNET_SCHEDULER_Handle *sched,
         GNUNET_FS_stop (daemons[i]->fs);
       if (daemons[i]->cfg != NULL)
         GNUNET_CONFIGURATION_destroy (daemons[i]->cfg);
+      if (NULL != daemons[i]->publish_tmp_file)
+        {
+          GNUNET_DISK_directory_remove (daemons[i]->publish_tmp_file);
+          GNUNET_free (daemons[i]->publish_tmp_file);
+        }
       GNUNET_free (daemons[i]);
       daemons[i] = NULL;
     }  
@@ -647,25 +658,71 @@ GNUNET_FS_TEST_publish (struct GNUNET_SCHEDULER_Handle *sched,
                         GNUNET_FS_TEST_UriContinuation cont,
                         void *cont_cls)
 {
-  GNUNET_assert (daemon->publish_cont == NULL);
   struct GNUNET_FS_FileInformation *fi;
+  struct GNUNET_DISK_FileHandle *fh;
+  char *emsg;
+  uint64_t off;
+  char buf[DBLOCK_SIZE];
+  size_t bsize;
 
+  GNUNET_assert (daemon->publish_cont == NULL);
   daemon->publish_cont = cont;
   daemon->publish_cont_cls = cont_cls;
   daemon->publish_seed = seed;
   daemon->verbose = verbose;
   daemon->publish_sched = sched;
-  fi = GNUNET_FS_file_information_create_from_reader (daemon->fs,
-                                                      daemon,                                                 
-                                                      size,
-                                                      &file_generator,
-                                                      daemon,
-                                                      NULL,
-                                                      NULL,
-                                                      do_index,
-                                                      anonymity,
-                                                      42 /* priority */,
-                                                      GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+  if (GNUNET_YES == do_index)
+    {
+      GNUNET_assert (daemon->publish_tmp_file == NULL);
+      daemon->publish_tmp_file = GNUNET_DISK_mktemp ("fs-test-publish-index");
+      GNUNET_assert (daemon->publish_tmp_file != NULL);
+      fh = GNUNET_DISK_file_open (daemon->publish_tmp_file,
+                                  GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_CREATE,
+                                  GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
+      GNUNET_assert (NULL != fh);
+      off = 0;      
+      while (off < size)
+        {
+          bsize = GNUNET_MIN (sizeof (buf),
+                              size - off);
+          GNUNET_assert (bsize ==
+                         file_generator (daemon,
+                                         off,
+                                         bsize,
+                                         buf,
+                                         &emsg));
+          GNUNET_assert (emsg == NULL);
+          GNUNET_assert (bsize ==
+                         GNUNET_DISK_file_write (fh,
+                                                 buf,
+                                                 bsize));
+          off += bsize;
+        }
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_DISK_file_close (fh));
+      fi = GNUNET_FS_file_information_create_from_file (daemon->fs,
+                                                        daemon,                                               
+                                                        daemon->publish_tmp_file,
+                                                        NULL, NULL,
+                                                        do_index,
+                                                        anonymity,
+                                                        42 /* priority */,
+                                                        GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+    }
+  else
+    {
+      fi = GNUNET_FS_file_information_create_from_reader (daemon->fs,
+                                                          daemon,                                                     
+                                                          size,
+                                                          &file_generator,
+                                                          daemon,
+                                                          NULL,
+                                                          NULL,
+                                                          do_index,
+                                                          anonymity,
+                                                          42 /* priority */,
+                                                          GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+    }
   daemon->publish_context = GNUNET_FS_publish_start (daemon->fs,
                                                      fi,
                                                      NULL, NULL, NULL,
diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf
index 4e7dabd9d..25f9979bb 100644
--- a/src/fs/fs_test_lib_data.conf
+++ b/src/fs/fs_test_lib_data.conf
@@ -20,7 +20,7 @@ HOSTNAME = localhost
 DEFAULTSERVICES = fs
 
 [datastore]
-DEBUG = YES
+# DEBUG = YES
 #PREFIX = valgrind --tool=memcheck --leak-check=yes
 QUOTA = 2000000000
 
@@ -52,7 +52,7 @@ PORT = 43471
 HOSTNAME = localhost
 #OPTIONS = -L DEBUG
 ACTIVEMIGRATION = NO
-DEBUG = YES
+# DEBUG = YES
 #PREFIX = valgrind --tool=memcheck --leak-check=yes 
 #BINARY = /home/grothoff/gn9/bin/gnunet-service-fs
 #PREFIX = xterm -e gdb -x cmd --args 
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index 9ba722c3c..f59f18546 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -54,15 +54,15 @@
  * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for
  * a rather wasteful mode of operation (that might still get the highest
  * throughput overall).
+ *
+ * Performance measurements (for 50 MB file, 2 peers):
+ *
+ * - Without delays: 3300 kb/s
+ * - With    delays:  101 kb/s
  */
 #define SUPPORT_DELAYS GNUNET_NO
 
 /**
- * Currently experimental code...
- */
-#define ENABLE_LOAD_MGMT GNUNET_YES
-
-/**
  * Size for the hash map for DHT requests from the FS
  * service.  Should be about the number of concurrent
  * DHT requests we plan to make.
@@ -101,6 +101,24 @@
  * give us 5 MB/s.  OTOH, obviously re-trying the same peer can be
  * rather inefficient in larger networks, hence picking 1 is in 
  * general not the best choice.
+ *
+ * Performance measurements (for 50 MB file, 2 peers, no delays):
+ *
+ * - 1: 3300 kb/s (consistently)
+ * - 3: 2046 kb/s, 754 kb/s, 3490 kb/s
+ * - 5:  759 kb/s, 968 kb/s, 1160 kb/s
+ *
+ * Note that this does NOT mean that the value should be 1 since
+ * a 2-peer network is far from representative here (and this fails
+ * to take into consideration bandwidth wasted by repeatedly 
+ * sending queries to peers that don't have the content).  Also,
+ * it is expected that higher values lead to more inconsistent
+ * measurements since this only affects lost messages towards the
+ * end of the download.
+ *
+ * Finally, we should probably consider changing this and making
+ * it dependent on the number of connected peers or a related
+ * metric (bad magic constants...).
  */
 #define RETRY_PROBABILITY_INV 1
 
@@ -1565,7 +1583,7 @@ peer_connect_handler (void *cls,
   uint32_t trust;
   
   cp = GNUNET_malloc (sizeof (struct ConnectedPeer));
-  cp->transmission_delay = GNUNET_LOAD_value_init (GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT);
+  cp->transmission_delay = GNUNET_LOAD_value_init (latency);
   cp->pid = GNUNET_PEER_intern (peer);
 
   fn = get_trust_filename (peer);
@@ -1590,6 +1608,39 @@ peer_connect_handler (void *cls,
 
 
 /**
+ * Method called whenever a given peer has a status change.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ * @param latency reported latency of the connection with 'other'
+ * @param distance reported distance (DV) to 'other' 
+ * @param bandwidth_in available amount of inbound bandwidth
+ * @param bandwidth_out available amount of outbound bandwidth
+ * @param timeout absolute time when this peer will time out
+ *        unless we see some further activity from it
+ */
+static void
+peer_status_handler (void *cls,
+                     const struct
+                     GNUNET_PeerIdentity * peer,
+                     struct GNUNET_TIME_Relative latency,
+                     uint32_t distance,
+                     struct GNUNET_BANDWIDTH_Value32NBO bandwidth_in,
+                     struct GNUNET_BANDWIDTH_Value32NBO bandwidth_out,
+                     struct GNUNET_TIME_Absolute timeout)
+{
+  struct ConnectedPeer *cp;
+
+  cp = GNUNET_CONTAINER_multihashmap_get (connected_peers,
+                                          &peer->hashPubKey);
+  GNUNET_assert (cp != NULL);
+  GNUNET_LOAD_value_set_decline (cp->transmission_delay,
+                                 latency);  
+}
+
+
+
+/**
  * Increase the host credit by a value.
  *
  * @param host which peer to change the trust value on
@@ -3804,12 +3855,8 @@ process_local_reply (void *cls,
                                 gettext_noop ("# processing result set cut short due to load"),
                                 1,
                                 GNUNET_NO);
-      /* FIXME: if this is activated, we might stall large downloads
-         indefinitely since (presumably) the load can never go down again! */
-#if ENABLE_LOAD_MGMT
       GNUNET_DATASTORE_get_next (dsh, GNUNET_NO);
       return;
-#endif
     }
   GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
 }
@@ -4021,14 +4068,7 @@ handle_p2p_get (void *cls,
                   "Dropping query from `%s', this peer is too busy.\n",
                   GNUNET_i2s (other));
 #endif
-      GNUNET_STATISTICS_update (stats,
-                                gettext_noop ("# requests dropped due to high load"),
-                                1,
-                                GNUNET_NO);
-#if ENABLE_LOAD_MGMT
-      /* FIXME: this causes problems... */
       return GNUNET_OK;
-#endif
     }
 #if DEBUG_FS 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -4053,12 +4093,7 @@ handle_p2p_get (void *cls,
       /* don't have BW to send to peer, or would likely take longer than we have for it,
          so at best indirect the query */
       priority = 0;
-#if ENABLE_LOAD_MGMT
-      /* FIXME: if this line is enabled, the 'perf' test for larger files simply "hangs";
-         the cause seems to be that the load goes up (to the point where we do this)
-         and then never goes down again... (outch) */
       pr->forward_only = GNUNET_YES;
-#endif
     }
   pr->type = type;
   pr->mingle = ntohl (gm->filter_mutator);
@@ -4457,7 +4492,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s,
     }
   connected_peers = GNUNET_CONTAINER_multihashmap_create (enc); 
   query_request_map = GNUNET_CONTAINER_multihashmap_create (max_pending_requests);
-  rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_SECONDS);
+  rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL);
   peer_request_map = GNUNET_CONTAINER_multihashmap_create (enc);
   requests_by_expiration_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); 
   core = GNUNET_CORE_connect (sched,
@@ -4467,7 +4502,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s,
                               NULL,
                               &peer_connect_handler,
                               &peer_disconnect_handler,
-                              NULL,
+                              &peer_status_handler,
                               NULL, GNUNET_NO,
                               NULL, GNUNET_NO,
                               p2p_handlers);
diff --git a/src/fs/perf_gnunet_service_fs_p2p.c b/src/fs/perf_gnunet_service_fs_p2p.c
index ac4458994..fe57c1666 100644
--- a/src/fs/perf_gnunet_service_fs_p2p.c
+++ b/src/fs/perf_gnunet_service_fs_p2p.c
@@ -27,12 +27,12 @@
 #include "fs_test_lib.h"
 #include "gnunet_testing_lib.h"
 
-#define VERBOSE GNUNET_YES
+#define VERBOSE GNUNET_NO
 
 /**
  * File-size we use for testing.
  */
-#define FILESIZE (1024 * 1024 * 1)
+#define FILESIZE (1024 * 1024 * 50)
 
 /**
  * How long until we give up on transmitting the message?
@@ -51,6 +51,8 @@ static int ok;
 
 static struct GNUNET_TIME_Absolute start_time;
 
+static const char *progname;
+
 static void
 do_stop (void *cls,
          const struct GNUNET_SCHEDULER_TaskContext *tc)
@@ -87,21 +89,19 @@ static struct StatValues stats[] =
     { "fs", "# results found locally"},
     { "fs", "# requests forwarded due to high load"},
     { "fs", "# requests done for free (low load)"},
+    { "fs", "# requests dropped, priority insufficient"},
+    { "fs", "# requests done for a price (normal load)"},
+    { "fs", "# requests dropped by datastore (queue length limit)"},
     { "fs", "# P2P searches received"},
-    { "fs", "# replies received for local clients"},
     { "fs", "# P2P searches discarded (queue length bound)"},
-    { "fs", "# requests dropped due to high load"},
-    { "fs", "# requests dropped by datastore (queue length limit)"},
+    { "fs", "# replies received for local clients"},
     { "fs", "# queries retransmitted to same target"},
     { "fs", "cummulative artificial delay introduced (ms)"},
     { "core", "# bytes decrypted"},
     { "core", "# bytes encrypted"},
-    { "core", "# transmissions delayed due to corking"},
     { "transport", "# bytes received via TCP"},
     { "transport", "# bytes transmitted via TCP"},
     { "datacache", "# bytes stored"},
-    { "dht", "# DHT ROUTE Requests Seen"},
-    { "dht", "# DHT ROUTE Requests Forwarded"},
     { NULL, NULL}
   };
 
@@ -236,6 +236,8 @@ static void
 do_download (void *cls,
              const struct GNUNET_FS_Uri *uri)
 {
+  int anonymity;
+
   if (NULL == uri)
     {
       GNUNET_FS_TEST_daemons_stop (sched,
@@ -250,10 +252,14 @@ do_download (void *cls,
               "Downloading %llu bytes\n",
               (unsigned long long) FILESIZE);
   start_time = GNUNET_TIME_absolute_get ();
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
   GNUNET_FS_TEST_download (sched,
                            daemons[0],
                            TIMEOUT,
-                           1, SEED, uri, 
+                           anonymity, SEED, uri, 
                            VERBOSE, 
                            &do_report, NULL);
 }
@@ -263,6 +269,9 @@ static void
 do_publish (void *cls,
             const char *emsg)
 {
+  int do_index;
+  int anonymity;
+
   if (NULL != emsg)
     {
       GNUNET_FS_TEST_daemons_stop (sched,
@@ -277,10 +286,20 @@ do_publish (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Publishing %llu bytes\n",
               (unsigned long long) FILESIZE);
+  if (NULL != strstr (progname, "index"))
+    do_index = GNUNET_YES;
+  else
+    do_index = GNUNET_NO;
+  if (NULL != strstr (progname, "dht"))
+    anonymity = 0;
+  else
+    anonymity = 1;
+  
   GNUNET_FS_TEST_publish (sched,
                           daemons[NUM_DAEMONS-1],
                           TIMEOUT,
-                          1, GNUNET_NO, FILESIZE, SEED, 
+                          anonymity, 
+                          do_index, FILESIZE, SEED, 
                           VERBOSE, 
                           &do_download, NULL);
 }
@@ -343,9 +362,9 @@ main (int argc, char *argv[])
   struct GNUNET_GETOPT_CommandLineOption options[] = {
     GNUNET_GETOPT_OPTION_END
   };
-
+  progname = argv[0];
   GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/");
-  GNUNET_log_setup ("perf_gnunet_service_fs_p2p", 
+  GNUNET_log_setup ("perf_gnunet_service_fs_p2p_index", 
 #if VERBOSE
                     "DEBUG",
 #else
@@ -353,10 +372,10 @@ main (int argc, char *argv[])
 #endif
                     NULL);
   GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1,
-                      argvx, "perf-gnunet-service-fs-p2p",
+                      argvx, "perf-gnunet-service-fs-p2p-index",
                       "nohelp", options, &run, NULL);
   GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/");
   return ok;
 }
 
-/* end of perf_gnunet_service_fs_p2p.c */
+/* end of perf_gnunet_service_fs_p2p_index.c */