nse changes

5 files changed, 565 insertions, 14 deletions

diff --git a/src/nse/Makefile.am b/src/nse/Makefile.am
index a13d6d8b1..0718a9d51 100644
--- a/src/nse/Makefile.am
+++ b/src/nse/Makefile.am
@@ -23,16 +23,18 @@ libgnunetnse_la_LDFLAGS = \
 
 
 bin_PROGRAMS = \
- gnunet-service-nse
-
-# gnunet_nse_SOURCES = \
-# gnunet-nse.c         
-# gnunet_nse_LDADD = \
-#  $(top_builddir)/src/nse/libgnunetnse.la \
-#  $(top_builddir)/src/util/libgnunetutil.la \
-#  $(GN_LIBINTL)
-# gnunet_nse_DEPENDENCIES = \
-#  libgnunetnse.la                               
+ gnunet-service-nse \
+ nse-profiler
+
+nse_profiler_SOURCES = \
+ nse-profiler.c         
+nse_profiler_LDADD = \
+  $(top_builddir)/src/nse/libgnunetnse.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(top_builddir)/src/testing/libgnunettesting.la
+  $(GN_LIBINTL)
+nse_profiler_DEPENDENCIES = \
+  libgnunetnse.la                                
 
 gnunet_service_nse_SOURCES = \
  gnunet-service-nse.c         
diff --git a/src/nse/gnunet-service-nse.c b/src/nse/gnunet-service-nse.c
index 874b8b6f6..eb5a72def 100644
--- a/src/nse/gnunet-service-nse.c
+++ b/src/nse/gnunet-service-nse.c
@@ -50,7 +50,7 @@
 #include "gnunet_nse_service.h"
 #include "nse.h"
 
-#define DEFAULT_HISTORY_SIZE 50
+#define DEFAULT_HISTORY_SIZE 10
 
 #define DEFAULT_CORE_QUEUE_SIZE 32
 
diff --git a/src/nse/nse-profiler.c b/src/nse/nse-profiler.c
new file mode 100644
index 000000000..58ce5fb49
--- /dev/null
+++ b/src/nse/nse-profiler.c
@@ -0,0 +1,469 @@
+/*
+     This file is part of GNUnet.
+     (C) 2009 Christian Grothoff (and other contributing authors)
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+/**
+ * @file nse/nse-profiler.c
+ *
+ * @brief Profiling driver for the network size estimation service.
+ *        Generally, the profiler starts a given number of peers,
+ *        then churns some off, waits a certain amount of time, then
+ *        churns again, and repeats.
+ */
+#include "platform.h"
+#include "gnunet_testing_lib.h"
+#include "gnunet_nse_service.h"
+
+#define VERBOSE GNUNET_NO
+
+struct NSEPeer
+{
+  struct NSEPeer *prev;
+
+  struct NSEPeer *next;
+
+  struct GNUNET_TESTING_Daemon *daemon;
+
+  struct GNUNET_NSE_Handle *nse_handle;
+};
+
+struct NSEPeer *peer_head;
+
+struct NSEPeer *peer_tail;
+
+/**
+ * How long until we give up on connecting the peers?
+ */
+#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1500)
+
+static int ok;
+
+/**
+ * Be verbose
+ */
+static int verbose;
+
+/**
+ * Total number of peers in the test.
+ */
+static unsigned long long num_peers;
+
+/**
+ * Global configuration file
+ */
+struct GNUNET_CONFIGURATION_Handle *testing_cfg;
+
+/**
+ * Total number of currently running peers.
+ */
+static unsigned long long peers_running;
+
+/**
+ * Current round we are in.
+ */
+static unsigned long long current_round;
+
+/**
+ * Peers desired in the next round.
+ */
+static unsigned long long peers_next_round;
+
+/**
+ * Total number of connections in the whole network.
+ */
+static unsigned int total_connections;
+
+/**
+ * The currently running peer group.
+ */
+static struct GNUNET_TESTING_PeerGroup *pg;
+
+/**
+ * File to report results to.
+ */
+static struct GNUNET_DISK_FileHandle *output_file;
+
+/**
+ * How many data points to capture before triggering next round?
+ */
+static struct GNUNET_TIME_Relative wait_time;
+
+/**
+ * Task called to disconnect peers.
+ */
+static GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
+
+/**
+ * Task called to shutdown test.
+ */
+static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
+
+/**
+ * Task used to churn the network.
+ */
+static GNUNET_SCHEDULER_TaskIdentifier churn_task;
+
+/**
+ * Check whether peers successfully shut down.
+ */
+void
+shutdown_callback (void *cls, const char *emsg)
+{
+  if (emsg != NULL)
+    {
+#if VERBOSE
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutdown of peers failed!\n");
+#endif
+      if (ok == 0)
+        ok = 666;
+    }
+  else
+    {
+#if VERBOSE
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "All peers successfully shut down!\n");
+#endif
+      ok = 0;
+    }
+}
+
+static void
+shutdown_task (void *cls,
+               const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct NSEPeer *pos;
+#if VERBOSE
+  fprintf(stderr, "Ending test.\n");
+#endif
+
+  if (disconnect_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel(disconnect_task);
+      disconnect_task = GNUNET_SCHEDULER_NO_TASK;
+    }
+  while (NULL != (pos = peer_head))
+    {
+      if (pos->nse_handle != NULL)
+        GNUNET_NSE_disconnect(pos->nse_handle);
+      GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
+      GNUNET_free(pos);
+    }
+
+  GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
+}
+
+/**
+ * Callback to call when network size estimate is updated.
+ *
+ * @param cls closure
+ * @param estimate the value of the current network size estimate
+ * @param std_dev standard deviation (rounded down to nearest integer)
+ *                of the size estimation values seen
+ *
+ */
+static void
+handle_estimate (void *cls, double estimate, double std_dev)
+{
+  struct NSEPeer *peer = cls;
+  char *output_buffer;
+  int size;
+  //fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
+  if (output_file != NULL)
+    {
+      size = GNUNET_asprintf(&output_buffer, "%s %u %f %f\n", GNUNET_i2s(&peer->daemon->id), peers_running, estimate, std_dev);
+      if (size != GNUNET_DISK_file_write(output_file, output_buffer, size))
+        GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Unable to write to file!\n", "nse-profiler");
+    }
+  else
+    fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
+
+}
+
+
+static void
+connect_nse_service (void *cls,
+                     const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct NSEPeer *current_peer;
+  unsigned int i;
+#if VERBOSE
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
+#endif
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
+  for (i = 0; i < num_peers; i++)
+    {
+      current_peer = GNUNET_malloc(sizeof(struct NSEPeer));
+      current_peer->daemon = GNUNET_TESTING_daemon_get(pg, i);
+      if (GNUNET_YES == GNUNET_TESTING_daemon_running(GNUNET_TESTING_daemon_get(pg, i)))
+        {
+          current_peer->nse_handle = GNUNET_NSE_connect (current_peer->daemon->cfg, &handle_estimate, current_peer);
+          GNUNET_assert(current_peer->nse_handle != NULL);
+        }
+      GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer);
+    }
+}
+
+static void
+churn_peers (void *cls,
+             const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+static void
+disconnect_nse_peers (void *cls,
+                      const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct NSEPeer *pos;
+  char *buf;
+  disconnect_task = GNUNET_SCHEDULER_NO_TASK;
+  pos = peer_head;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: disconnecting nse service of peers\n");
+  while (pos != NULL)
+    {
+      if (pos->nse_handle != NULL)
+        {
+          GNUNET_NSE_disconnect(pos->nse_handle);
+          pos->nse_handle = NULL;
+        }
+      pos = pos->next;
+    }
+
+  GNUNET_asprintf(&buf, "round%llu", current_round);
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", buf, &peers_next_round))
+    {
+      current_round++;
+      GNUNET_assert(churn_task == GNUNET_SCHEDULER_NO_TASK);
+      churn_task = GNUNET_SCHEDULER_add_now(&churn_peers, NULL);
+    }
+  else /* No more rounds, let's shut it down! */
+    {
+      GNUNET_SCHEDULER_cancel(shutdown_handle);
+      shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
+    }
+  GNUNET_free(buf);
+}
+
+/**
+ * Prototype of a function that will be called when a
+ * particular operation was completed the testing library.
+ *
+ * @param cls closure
+ * @param emsg NULL on success
+ */
+static void
+churn_callback (void *cls, const char *emsg)
+{
+  if (emsg == NULL) /* Everything is okay! */
+    {
+      peers_running = GNUNET_TESTING_daemons_running(pg);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Round %lu, churn finished successfully.\n", current_round);
+      GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
+      disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
+      GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
+    }
+  else
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Round %lu, churn FAILED!!\n", current_round);
+      GNUNET_SCHEDULER_cancel(shutdown_handle);
+      GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
+    }
+}
+
+static void
+churn_peers (void *cls,
+                      const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  peers_running = GNUNET_TESTING_daemons_running(pg);
+  churn_task = GNUNET_SCHEDULER_NO_TASK;
+  if (peers_next_round == peers_running)
+    {
+      /* Nothing to do... */
+      GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
+      GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
+      disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
+      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Round %lu, doing nothing!\n", current_round);
+    }
+  else
+    {
+      if (peers_next_round > num_peers)
+        {
+          GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Asked to turn on more peers than have!!\n");
+          GNUNET_SCHEDULER_cancel(shutdown_handle);
+          GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
+        }
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Round %lu, turning off %lu peers, turning on %lu peers!\n",
+                  current_round,
+                  (peers_running > peers_next_round) ? peers_running
+                      - peers_next_round : 0,
+                  (peers_next_round > peers_running) ? peers_next_round
+                      - peers_running : 0);
+      GNUNET_TESTING_daemons_churn (pg,
+                                    (peers_running > peers_next_round) ? peers_running
+                                        - peers_next_round
+                                        : 0,
+                                    (peers_next_round > peers_running) ? peers_next_round
+                                        - peers_running
+                                        : 0, wait_time, &churn_callback,
+                                    NULL);
+    }
+}
+
+
+
+static void
+my_cb (void *cls,
+       const char *emsg)
+{
+  if (emsg != NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Peergroup callback called with error, aborting test!\n");
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error from testing: `%s'\n");
+      ok = 1;
+      GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
+      return;
+    }
+#if VERBOSE
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Peer Group started successfully, connecting to NSE service for each peer!\n");
+#endif
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Have %u connections\n", total_connections);
+  peers_running = GNUNET_TESTING_daemons_running(pg);
+  GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
+  disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
+}
+
+/**
+ * Prototype of a function that will be called whenever
+ * two daemons are connected by the testing library.
+ *
+ * @param cls closure
+ * @param first peer id for first daemon
+ * @param second peer id for the second daemon
+ * @param distance distance between the connected peers
+ * @param first_cfg config for the first daemon
+ * @param second_cfg config for the second daemon
+ * @param first_daemon handle for the first daemon
+ * @param second_daemon handle for the second daemon
+ * @param emsg error message (NULL on success)
+ */
+void connect_cb (void *cls,
+                 const struct GNUNET_PeerIdentity *first,
+                 const struct GNUNET_PeerIdentity *second,
+                 uint32_t distance,
+                 const struct GNUNET_CONFIGURATION_Handle *first_cfg,
+                 const struct GNUNET_CONFIGURATION_Handle *second_cfg,
+                 struct GNUNET_TESTING_Daemon *first_daemon,
+                 struct GNUNET_TESTING_Daemon *second_daemon,
+                 const char *emsg)
+{
+  char *second_id;
+
+  second_id = GNUNET_strdup(GNUNET_i2s(second));
+  if (emsg == NULL)
+    total_connections++;
+}
+
+
+static void
+run (void *cls,
+     char *const *args,
+     const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  char *temp_str;
+  unsigned long long temp_wait;
+  ok = 1;
+  testing_cfg = GNUNET_CONFIGURATION_create();
+  GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_load(testing_cfg, cfgfile));
+#if VERBOSE
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n");
+  GNUNET_CONFIGURATION_set_value_string (testing_cfg, "testing",
+                                           "use_progressbars",
+                                           "YES");
+#endif
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "testing", "num_peers", &num_peers))
+    {
+      GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option TESTING:NUM_PEERS is required!\n");
+      return;
+    }
+
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", "wait_time", &temp_wait))
+    {
+      GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:wait_time is required!\n");
+      return;
+    }
+  wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
+
+  if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg, "nse-profiler", "output_file", &temp_str))
+    {
+      output_file = GNUNET_DISK_file_open (temp_str, GNUNET_DISK_OPEN_READWRITE
+                                                                  | GNUNET_DISK_OPEN_CREATE,
+                                                                  GNUNET_DISK_PERM_USER_READ |
+                                                                  GNUNET_DISK_PERM_USER_WRITE);
+      if (output_file == NULL)
+        GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", temp_str);
+    }
+  GNUNET_free_non_null(temp_str);
+
+  pg = GNUNET_TESTING_peergroup_start(testing_cfg,
+                                      num_peers,
+                                      TIMEOUT,
+                                      &connect_cb,
+                                      &my_cb, NULL,
+                                      NULL);
+  GNUNET_assert (pg != NULL);
+  shutdown_handle = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever(), &shutdown_task, NULL);
+}
+
+
+
+/**
+ * nse-profiler command line options
+ */
+static struct GNUNET_GETOPT_CommandLineOption options[] = {
+  {'V', "verbose", NULL,
+   gettext_noop ("be verbose (print progress information)"),
+   0, &GNUNET_GETOPT_set_one, &verbose},
+  GNUNET_GETOPT_OPTION_END
+};
+
+int
+main (int argc, char *argv[])
+{
+  int ret;
+
+  GNUNET_log_setup ("nse-profiler",
+#if VERBOSE
+                    "DEBUG",
+#else
+                    "WARNING",
+#endif
+                    NULL);
+  ret = 1;
+  GNUNET_PROGRAM_run (argc,
+                      argv, "nse-profiler", gettext_noop
+                      ("Run a test of the NSE service."),
+                      options, &run, &ok);
+
+  GNUNET_DISK_directory_remove ("/tmp/nse-profiler");
+  return ret;
+}
+
+/* end of nse-profiler.c */
diff --git a/src/nse/nse_profiler_test.conf b/src/nse/nse_profiler_test.conf
new file mode 100644
index 000000000..8275d9152
--- /dev/null
+++ b/src/nse/nse_profiler_test.conf
@@ -0,0 +1,78 @@
+[PATHS]
+SERVICEHOME = /tmp/nse-profiler/
+DEFAULTCONFIG = nse_profiler_test.conf
+
+[nse]
+PORT = 0
+UNIXPATH = /tmp/test-nse-service-nse.unix
+BINARY = gnunet-service-nse
+#BINARY = /home/mrwiggles/documents/research/gnunet/gnunet-ng/src/nse/.libs/gnunet-service-nse
+#PREFIX = valgrind --leak-check=full --log-file=valgrind_nse.%p
+AUTOSTART = YES
+DEBUG = NO
+CONFIG = $DEFAULTCONFIG
+
+[arm]
+PORT = 0
+DEFAULTSERVICES = nse core topology
+UNIXPATH = /tmp/test-nse-service-arm.unix
+#DEBUG = YES
+
+[statistics]
+AUTOSTART = NO
+
+[fs]
+AUTOSTART = NO
+
+[datastore]
+AUTOSTART = NO
+
+[dht]
+AUTOSTART = NO
+
+[transport]
+PORT = 0
+AUTOSTART = YES
+
+[transport-unix]
+PORT = 11111
+[core]
+AUTOSTART = YES
+
+[peerinfo]
+AUTOSTART = YES
+
+[dns]
+AUTOSTART = NO
+
+[topology]
+PORT = 0
+
+[transport]
+PORT = 0
+plugins = unix
+
+[testing]
+NUM_PEERS = 1000
+WEAKRANDOM = YES
+TOPOLOGY = NONE
+CONNECT_TOPOLOGY = SMALL_WORLD_RING
+PERCENTAGE = 5
+F2F = NO
+CONNECT_TIMEOUT = 60
+CONNECT_ATTEMPTS = 3
+#DEBUG = YES
+HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat
+MAX_CONCURRENT_SSH = 1
+USE_PROGRESSBARS = YES
+PEERGROUP_TIMEOUT = 1000
+MAX_OUTSTANDING_CONNECTIONS = 20
+
+[nse-profiler]
+OUTPUT_FILE = nse_output_1000_peers.dat
+ROUND0 = 1000
+ROUND1 = 750
+ROUND2 = 500
+ROUND3 = 1000
+WAIT_TIME = 800
+
diff --git a/src/nse/test_nse.conf b/src/nse/test_nse.conf
index 228963fb3..1492f1a00 100644
--- a/src/nse/test_nse.conf
+++ b/src/nse/test_nse.conf
@@ -40,9 +40,11 @@ AUTOSTART = YES
 AUTOSTART = NO
 
 [testing]
-NUM_PEERS = 20
+NUM_PEERS = 100
 WEAKRANDOM = YES
-TOPOLOGY = CLIQUE
+TOPOLOGY = NONE
+CONNECT_TOPOLOGY = SMALL_WORLD_RING
+PERCENTAGE = 3
 F2F = NO
 CONNECT_TIMEOUT = 60
 CONNECT_ATTEMPTS = 3
@@ -50,4 +52,4 @@ CONNECT_ATTEMPTS = 3
 HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat
 MAX_CONCURRENT_SSH = 20
 USE_PROGRESSBARS = YES
-PEERGROUP_TIMEOUT = 180
+PEERGROUP_TIMEOUT = 1000