From 383814672c031e49c4dffca0a69af3b0973d2248 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 7 Oct 2015 22:01:03 +0000 Subject: statistics in set profiler, temp. fix for salt --- src/set/Makefile.am | 1 + src/set/gnunet-service-set.c | 7 +++-- src/set/gnunet-service-set_union.c | 13 ++++++-- src/set/gnunet-set-profiler.c | 61 +++++++++++++++++++++++++++++++------- src/set/ibf.c | 8 ++--- src/set/test_set.conf | 2 +- 6 files changed, 71 insertions(+), 21 deletions(-) diff --git a/src/set/Makefile.am b/src/set/Makefile.am index 5da0c564a..b4595d0e7 100644 --- a/src/set/Makefile.am +++ b/src/set/Makefile.am @@ -29,6 +29,7 @@ gnunet_set_profiler_SOURCES = \ gnunet-set-profiler.c gnunet_set_profiler_LDADD = \ $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ libgnunetset.la \ $(top_builddir)/src/testing/libgnunettesting.la \ $(GN_LIBINTL) diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c index cc172974e..cfddef6fb 100644 --- a/src/set/gnunet-service-set.c +++ b/src/set/gnunet-service-set.c @@ -827,12 +827,13 @@ execute_add (struct Set *set, msg = (const struct GNUNET_SET_ElementMessage *) m; el.size = ntohs (m->size) - sizeof *msg; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Client inserts element of size %u\n", - el.size); el.data = &msg[1]; el.element_type = ntohs (msg->element_type); GNUNET_SET_element_hash (&el, &hash); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Client inserts element %s of size %u\n", + GNUNET_h2s (&hash), + el.size); ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements, &hash); diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c index 04fbba348..1c5dfe3fb 100644 --- a/src/set/gnunet-service-set_union.c +++ b/src/set/gnunet-service-set_union.c @@ -341,6 +341,10 @@ get_ibf_key (const struct GNUNET_HashCode *src, { struct IBF_Key key; + /* FIXME: Ensure that the salt is handled correctly. + This is a quick fix so that consensus works for now. */ + salt = 0; + GNUNET_CRYPTO_kdf (&key, sizeof (key), src, sizeof *src, &salt, sizeof (salt), @@ -499,7 +503,7 @@ init_key_to_element_iterator (void *cls, */ static void prepare_ibf (struct Operation *op, - uint16_t size) + uint32_t size) { if (NULL == op->state->key_to_element) { @@ -540,6 +544,12 @@ send_ibf (struct Operation *op, "sending ibf of size %u\n", 1<state->local_ibf; while (buckets_sent < (1 << ibf_order)) @@ -1116,7 +1126,6 @@ handle_p2p_elements (void *cls, "# repeated elements", 1, GNUNET_NO); - GNUNET_break (0); GNUNET_free (ee); } else diff --git a/src/set/gnunet-set-profiler.c b/src/set/gnunet-set-profiler.c index 2ee1b762d..4b6a1e202 100644 --- a/src/set/gnunet-set-profiler.c +++ b/src/set/gnunet-set-profiler.c @@ -25,6 +25,7 @@ */ #include "platform.h" #include "gnunet_util_lib.h" +#include "gnunet_statistics_service.h" #include "gnunet_set_service.h" #include "gnunet_testbed_service.h" @@ -35,6 +36,8 @@ static unsigned int num_a = 5; static unsigned int num_b = 5; static unsigned int num_c = 20; +static unsigned int dump_statistics; + static char *op_str = "union"; const static struct GNUNET_CONFIGURATION_Handle *config; @@ -57,6 +60,8 @@ static struct GNUNET_PeerIdentity local_peer; static struct GNUNET_SET_ListenHandle *set_listener; +static struct GNUNET_STATISTICS_Handle *statistics; + static int map_remove_iterator (void *cls, @@ -76,6 +81,37 @@ map_remove_iterator (void *cls, } +/** + * Callback function to process statistic values. + * + * @param cls closure + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration + */ +static int +statistics_result (void *cls, + const char *subsystem, + const char *name, + uint64_t value, + int is_persistent) +{ + printf ("stat %s/%s=%lu\n", subsystem, name, (unsigned long) value); + return GNUNET_OK; +} + + +static void +statistics_done (void *cls, + int success) +{ + GNUNET_assert (GNUNET_YES == success); + printf("dumped statistics\n"); + GNUNET_SCHEDULER_shutdown (); +} + static void check_all_done (void) { @@ -88,7 +124,14 @@ check_all_done (void) printf ("set a: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (info1.sent)); printf ("set b: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (info2.sent)); - GNUNET_SCHEDULER_shutdown (); + if (0 == dump_statistics) + { + GNUNET_SCHEDULER_shutdown (); + return; + } + + GNUNET_STATISTICS_get (statistics, NULL, NULL, GNUNET_TIME_UNIT_FOREVER_REL, + statistics_done, statistics_result, NULL); } @@ -215,6 +258,7 @@ handle_shutdown (void *cls, GNUNET_SET_destroy (info2.set); info2.set = NULL; } + GNUNET_STATISTICS_destroy (statistics, GNUNET_NO); } @@ -225,7 +269,6 @@ run (void *cls, { unsigned int i; struct GNUNET_HashCode hash; - struct GNUNET_HashCode hashhash; config = cfg; @@ -236,6 +279,8 @@ run (void *cls, return; } + statistics = GNUNET_STATISTICS_create ("set-profiler", cfg); + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, handle_shutdown, NULL); info1.id = "a"; @@ -251,9 +296,6 @@ run (void *cls, for (i = 0; i < num_a; i++) { GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash); - GNUNET_CRYPTO_hash (&hash, sizeof (struct GNUNET_HashCode), &hashhash); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Set a: Created element %s\n", - GNUNET_h2s (&hashhash)); GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE); } @@ -261,9 +303,6 @@ run (void *cls, for (i = 0; i < num_b; i++) { GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash); - GNUNET_CRYPTO_hash (&hash, sizeof (struct GNUNET_HashCode), &hashhash); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Set b: Created element %s\n", - GNUNET_h2s (&hashhash)); GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE); } @@ -271,9 +310,6 @@ run (void *cls, for (i = 0; i < num_c; i++) { GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash); - GNUNET_CRYPTO_hash (&hash, sizeof (struct GNUNET_HashCode), &hashhash); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Set c: Created element %s\n", - GNUNET_h2s (&hashhash)); GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE); } @@ -328,6 +364,9 @@ main (int argc, char **argv) { 'x', "operation", NULL, gettext_noop ("operation to execute"), GNUNET_YES, &GNUNET_GETOPT_set_string, &op_str }, + { 's', "statistics", NULL, + gettext_noop ("dump statistics to stdout after completion"), + GNUNET_NO, &GNUNET_GETOPT_set_one, &dump_statistics }, GNUNET_GETOPT_OPTION_END }; GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler", diff --git a/src/set/ibf.c b/src/set/ibf.c index 83cbafc65..19d15a5b2 100644 --- a/src/set/ibf.c +++ b/src/set/ibf.c @@ -76,12 +76,12 @@ ibf_create (uint32_t size, uint8_t hash_num) { struct InvertibleBloomFilter *ibf; - /* TODO: use malloc_large */ + GNUNET_assert (0 != size); ibf = GNUNET_new (struct InvertibleBloomFilter); - ibf->count = GNUNET_malloc (size * sizeof (uint8_t)); - ibf->key_sum = GNUNET_malloc (size * sizeof (struct IBF_Key)); - ibf->key_hash_sum = GNUNET_malloc (size * sizeof (struct IBF_KeyHash)); + ibf->count = GNUNET_malloc_large (size * sizeof (uint8_t)); + ibf->key_sum = GNUNET_malloc_large (size * sizeof (struct IBF_Key)); + ibf->key_hash_sum = GNUNET_malloc_large (size * sizeof (struct IBF_KeyHash)); ibf->size = size; ibf->hash_num = hash_num; diff --git a/src/set/test_set.conf b/src/set/test_set.conf index ae7708c2d..30ccbde55 100644 --- a/src/set/test_set.conf +++ b/src/set/test_set.conf @@ -5,7 +5,7 @@ GNUNET_TEST_HOME = /tmp/test-gnunet-set/ [set] AUTOSTART = YES -#PREFIX = valgrind +PREFIX = valgrind #PREFIX = valgrind --leak-check=full #PREFIX = gdbserver :1234 OPTIONS = -L INFO -- cgit v1.2.3