nse changes

6 files changed, 156 insertions, 34 deletions

diff --git a/src/nse/gnunet-service-nse.c b/src/nse/gnunet-service-nse.c
index eb5a72def..41b13a879 100644
--- a/src/nse/gnunet-service-nse.c
+++ b/src/nse/gnunet-service-nse.c
@@ -275,7 +275,7 @@ transmit_ready(void *cls, size_t size, void *buf)
 
   size_t msize;
   peer_entry->th = NULL;
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s: transmit_ready called\n",
       GNUNET_i2s (&my_identity));
 #endif
@@ -298,7 +298,7 @@ transmit_ready(void *cls, size_t size, void *buf)
   msize = ntohs (peer_entry->pending_message->size);
   if (msize <= size)
     memcpy (cbuf, peer_entry->pending_message, msize);
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
       "%s: transmit_ready called (transmit %d bytes)\n",
       GNUNET_i2s (&my_identity), msize);
@@ -441,7 +441,7 @@ schedule_flood_message(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       * GNUNET_NSE_INTERVAL;
   /* Find the next interval start time */
   next_timestamp.abs_value = previous_timestamp.abs_value + GNUNET_NSE_INTERVAL;
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
       "%s: curr_time %lu, prev timestamp %lu, next timestamp %lu\n",
       GNUNET_i2s (&my_identity), curr_time.abs_value,
@@ -474,7 +474,7 @@ schedule_flood_message(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   millisecond_offset = ((double) GNUNET_NSE_INTERVAL / (double) 2)
       - ((GNUNET_NSE_INTERVAL / M_PI) * atan (matching_bits
           - current_size_estimate));
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
       "%s: id matches %d bits, offset is %lu\n\n",
       GNUNET_i2s (&my_identity), matching_bits,
@@ -495,8 +495,8 @@ schedule_flood_message(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 #if DEBUG_NSE
   GNUNET_log (
       GNUNET_ERROR_TYPE_WARNING,
-      "%s: milliseconds until next timestamp %lu, sending flood in %lu\n",
-      GNUNET_i2s (&my_identity),
+      "%s: %u bits match, %lu milliseconds to timestamp , sending flood in %lu\n",
+      GNUNET_i2s (&my_identity), matching_bits,
       GNUNET_TIME_absolute_get_remaining (next_timestamp).rel_value,
       offset.rel_value);
 #endif
@@ -618,7 +618,7 @@ handle_p2p_size_estimate(void *cls, const struct GNUNET_PeerIdentity *peer,
   struct GNUNET_TIME_Absolute curr_time;
   uint64_t drift;
 
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s: received flood message!\n",
       GNUNET_i2s (&my_identity));
 #endif
@@ -748,7 +748,7 @@ send_flood_message(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
       if (peer_entry->th == NULL)
         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                     "%s: transmit handle is null!\n", GNUNET_i2s (&my_identity));
-#if DEBUG_NSE
+#if DEBUG_NSE > 1
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
           "%s: Sending flood message (distance %d) to %s!\n",
           GNUNET_i2s (&my_identity), ntohl (to_send->distance),
@@ -820,6 +820,7 @@ handle_core_disconnect(void *cls, const struct GNUNET_PeerIdentity *peer)
   while ((NULL != pos) && (0 != memcmp (&pos->id, peer,
                                         sizeof(struct GNUNET_PeerIdentity))))
     pos = pos->next;
+
   if (pos == NULL)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -828,12 +829,6 @@ handle_core_disconnect(void *cls, const struct GNUNET_PeerIdentity *peer)
       return;
     }
 
-  /* TODO: decide whether to copy the message, or always use the static pointer */
-#if TODO
-  if (pos->pending_message != NULL)
-  GNUNET_free(pos->pending_message);
-#endif
-
   if (pos->th != NULL)
     GNUNET_CORE_notify_transmit_ready_cancel (pos->th);
   GNUNET_CONTAINER_DLL_remove(peers_head, peers_tail, pos);
diff --git a/src/nse/nse-profiler.c b/src/nse/nse-profiler.c
index 7c3c7a5ec..70ea1a689 100644
--- a/src/nse/nse-profiler.c
+++ b/src/nse/nse-profiler.c
@@ -42,6 +42,14 @@ struct NSEPeer
   struct GNUNET_NSE_Handle *nse_handle;
 };
 
+struct StatsContext
+{
+  GNUNET_SCHEDULER_Task task;
+  GNUNET_SCHEDULER_TaskIdentifier *task_id;
+  void *task_cls;
+  unsigned long long total_nse_bytes;
+};
+
 struct NSEPeer *peer_head;
 
 struct NSEPeer *peer_tail;
@@ -99,6 +107,11 @@ static struct GNUNET_TESTING_PeerGroup *pg;
 static struct GNUNET_DISK_FileHandle *output_file;
 
 /**
+ * File to log connection info, statistics to.
+ */
+static struct GNUNET_DISK_FileHandle *data_file;
+
+/**
  * How many data points to capture before triggering next round?
  */
 static struct GNUNET_TIME_Relative wait_time;
@@ -118,7 +131,9 @@ static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
  */
 static GNUNET_SCHEDULER_TaskIdentifier churn_task;
 
-char *topology_file;
+static char *topology_file;
+
+static char *data_filename;
 
 /**
  * Check whether peers successfully shut down.
@@ -166,6 +181,8 @@ shutdown_task (void *cls,
       GNUNET_free(pos);
     }
 
+  if (data_file != NULL)
+    GNUNET_DISK_file_close(data_file);
   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 }
 
@@ -196,7 +213,6 @@ handle_estimate (void *cls, double estimate, double std_dev)
 
 }
 
-
 static void
 connect_nse_service (void *cls,
                      const struct GNUNET_SCHEDULER_TaskContext *tc)
@@ -224,12 +240,85 @@ static void
 churn_peers (void *cls,
              const struct GNUNET_SCHEDULER_TaskContext *tc);
 
+/**
+ * Continuation called by the "get_all" and "get" functions.
+ *
+ * @param cls struct StatsContext
+ * @param success GNUNET_OK if statistics were
+ *        successfully obtained, GNUNET_SYSERR if not.
+ */
+static void stats_finished_callback (void *cls, int success)
+{
+  struct StatsContext *stats_context = (struct StatsContext *)cls;
+  char *buf;
+  int buf_len;
+
+  if ((GNUNET_OK == success) && (data_file != NULL)) /* Stats lookup successful, write out data */
+    {
+      buf = NULL;
+      buf_len = GNUNET_asprintf(&buf, "TOTAL_NSE_BYTES: %u\n", stats_context->total_nse_bytes);
+      if (buf_len > 0)
+        {
+          GNUNET_DISK_file_write(data_file, buf, buf_len);
+        }
+      GNUNET_free_non_null(buf);
+    }
+
+  if (stats_context->task != NULL)
+    (*stats_context->task_id) = GNUNET_SCHEDULER_add_now(stats_context->task, stats_context->task_cls);
+
+  GNUNET_free(stats_context);
+}
+
+/**
+ * Callback function to process statistic values.
+ *
+ * @param cls struct StatsContext
+ * @param peer the peer the statistics belong to
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int statistics_iterator (void *cls,
+                                const struct GNUNET_PeerIdentity *peer,
+                                const char *subsystem,
+                                const char *name,
+                                uint64_t value,
+                                int is_persistent)
+{
+  struct StatsContext *stats_context = (struct StatsContext *)cls;
+  char *buf;
+
+  GNUNET_assert(0 < GNUNET_asprintf(&buf, "bytes of messages of type %d received", GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD));
+  if ((0 == strstr(subsystem, "core")) && (0 == strstr(name, buf)))
+    {
+      stats_context->total_nse_bytes += value;
+    }
+  GNUNET_free(buf);
+  return GNUNET_OK;
+}
+
+/**
+ * @param cls struct StatsContext
+ * @param tc task context
+ */
+static void
+get_statistics (void *cls,
+                const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct StatsContext *stats_context = (struct StatsContext *)cls;
+  GNUNET_TESTING_get_statistics(pg, &stats_finished_callback, &statistics_iterator, stats_context);
+}
+
 static void
 disconnect_nse_peers (void *cls,
                       const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct NSEPeer *pos;
   char *buf;
+  struct StatsContext *stats_context;
   disconnect_task = GNUNET_SCHEDULER_NO_TASK;
   pos = peer_head;
 
@@ -253,8 +342,14 @@ disconnect_nse_peers (void *cls,
     }
   else /* No more rounds, let's shut it down! */
     {
+      stats_context = GNUNET_malloc(sizeof(struct StatsContext));
       GNUNET_SCHEDULER_cancel(shutdown_handle);
-      shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
+      shutdown_handle = GNUNET_SCHEDULER_NO_TASK;
+      stats_context->task = &shutdown_task;
+      stats_context->task_cls = NULL;
+      stats_context->task_id = &shutdown_handle;
+      GNUNET_SCHEDULER_add_now(&get_statistics, stats_context);
+      //shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
     }
   GNUNET_free(buf);
 }
@@ -268,6 +363,9 @@ disconnect_nse_peers (void *cls,
  */
 void topology_output_callback (void *cls, const char *emsg)
 {
+  struct StatsContext *stats_context;
+  stats_context = GNUNET_malloc(sizeof(struct StatsContext));
+
   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
 }
@@ -353,6 +451,8 @@ static void
 my_cb (void *cls,
        const char *emsg)
 {
+  char *buf;
+  int buf_len;
   if (emsg != NULL)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -367,6 +467,14 @@ my_cb (void *cls,
               "Peer Group started successfully, connecting to NSE service for each peer!\n");
 #endif
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Have %u connections\n", total_connections);
+  if (data_file != NULL)
+    {
+      buf = NULL;
+      buf_len = GNUNET_asprintf(&buf, "CONNECTIONS_0: %u\n", total_connections);
+      if (buf_len > 0)
+        GNUNET_DISK_file_write(data_file, buf, buf_len);
+      GNUNET_free_non_null(buf);
+    }
   peers_running = GNUNET_TESTING_daemons_running(pg);
   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
@@ -438,6 +546,21 @@ run (void *cls,
       return;
     }
 
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "data_output_file", &data_filename))
+    {
+      GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:data_output_file is required!\n");
+      return;
+    }
+
+
+  data_file = GNUNET_DISK_file_open (data_filename, GNUNET_DISK_OPEN_READWRITE
+                                                  | GNUNET_DISK_OPEN_CREATE,
+                                                  GNUNET_DISK_PERM_USER_READ |
+                                                  GNUNET_DISK_PERM_USER_WRITE);
+  if (data_file == NULL)
+    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", data_filename);
+  GNUNET_free(data_filename);
+
   wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
 
   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg, "nse-profiler", "output_file", &temp_str))
diff --git a/src/nse/nse.h b/src/nse/nse.h
index f0a914488..a1bf2b1dd 100644
--- a/src/nse/nse.h
+++ b/src/nse/nse.h
@@ -30,7 +30,7 @@
 
 #include "gnunet_common.h"
 
-#define DEBUG_NSE GNUNET_NO
+#define DEBUG_NSE GNUNET_YES
 
 #define VERIFY_CRYPTO GNUNET_NO
 
diff --git a/src/nse/nse_profiler_test.conf b/src/nse/nse_profiler_test.conf
index 5e05cf422..7da26b232 100644
--- a/src/nse/nse_profiler_test.conf
+++ b/src/nse/nse_profiler_test.conf
@@ -14,7 +14,7 @@ CONFIG = $DEFAULTCONFIG
 
 [arm]
 PORT = 0
-DEFAULTSERVICES = nse core topology
+DEFAULTSERVICES = core
 UNIXPATH = /tmp/test-nse-service-arm.unix
 #DEBUG = YES
 
@@ -36,6 +36,7 @@ AUTOSTART = YES
 
 [transport-unix]
 PORT = 11111
+
 [core]
 AUTOSTART = YES
 
@@ -47,6 +48,7 @@ AUTOSTART = NO
 
 [topology]
 PORT = 0
+AUTOSTART = NO
 
 [transport]
 PORT = 0
@@ -57,25 +59,27 @@ NUM_PEERS = 1000
 WEAKRANDOM = YES
 TOPOLOGY = NONE
 CONNECT_TOPOLOGY = SMALL_WORLD_RING
-PERCENTAGE = 5
+PERCENTAGE = 4
 F2F = NO
 CONNECT_TIMEOUT = 60
 CONNECT_ATTEMPTS = 3
 #DEBUG = YES
 HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat
-MAX_CONCURRENT_SSH = 1
+MAX_CONCURRENT_SSH = 20
 USE_PROGRESSBARS = YES
 PEERGROUP_TIMEOUT = 1000
-MAX_OUTSTANDING_CONNECTIONS = 20
+TOPOLOGY_OUTPUT_FILE = nse_topo_1000_peers_initial
+MAX_OUTSTANDING_CONNECTIONS = 200
+#SINGLE_PEERINFO_PER_HOST = YES
+#NUM_PEERINFO_PER_HOST = 10
+#SINGLE_STATISTICS_PER_HOST = YES
+#NUM_STATISTICS_PER_HOST = 10
 
 [nse-profiler]
-OUTPUT_FILE = nse_output_1000_peers.dat
-ROUND0 = 1000
-ROUND1 = 750
-ROUND2 = 500
-ROUND3 = 1000
-WAIT_TIME = 800
-
-
-
-
+#OUTPUT_FILE = nse_output_1000_peers.dat
+#TOPOLOGY_OUTPUT_FILE = nse_topo_1000_peers
+#ROUND0 = 75
+#ROUND1 = 750
+#ROUND2 = 500
+#ROUND3 = 1000
+WAIT_TIME = 7200
diff --git a/src/nse/test_nse.conf b/src/nse/test_nse.conf
index 6a87e1bb7..17574a748 100644
--- a/src/nse/test_nse.conf
+++ b/src/nse/test_nse.conf
@@ -40,7 +40,7 @@ AUTOSTART = YES
 AUTOSTART = NO
 
 [testing]
-NUM_PEERS = 100
+NUM_PEERS = 20
 WEAKRANDOM = YES
 TOPOLOGY = NONE
 CONNECT_TOPOLOGY = SMALL_WORLD_RING
diff --git a/src/nse/test_nse_multipeer.c b/src/nse/test_nse_multipeer.c
index 4edf53f20..2f538b62c 100644
--- a/src/nse/test_nse_multipeer.c
+++ b/src/nse/test_nse_multipeer.c
@@ -193,7 +193,7 @@ void connect_cb (void *cls,
   second_id = GNUNET_strdup(GNUNET_i2s(second));
   if (emsg == NULL)
     {
-      fprintf(stderr, "Connected %s -> %s\n", GNUNET_i2s(first), second_id);
+      //fprintf(stderr, "Connected %s -> %s\n", GNUNET_i2s(first), second_id);
       total_connections++;
     }
 }