From 722a93c626d4033abbb221f44238fd7e25f99b0d Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Mon, 3 Aug 2015 10:29:09 +0000 Subject: -reduce autostarting for tests, reduce timeouts --- src/datastore/datastore_api.c | 52 ++++++--- src/datastore/gnunet-service-datastore.c | 128 ++++++++++++++------- src/datastore/perf_datastore_api.c | 13 ++- src/datastore/test_datastore_api.c | 2 +- src/datastore/test_datastore_api_data_mysql.conf | 10 -- .../test_datastore_api_data_postgres.conf | 11 -- src/datastore/test_datastore_api_data_sqlite.conf | 12 +- src/datastore/test_defaults.conf | 6 +- 8 files changed, 138 insertions(+), 96 deletions(-) (limited to 'src') diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c index d7e933fc6..a5b3e9088 100644 --- a/src/datastore/datastore_api.c +++ b/src/datastore/datastore_api.c @@ -328,7 +328,8 @@ transmit_drop (void *cls, size_t size, void *buf) * @param drop set to #GNUNET_YES to delete all data in datastore (!) */ void -GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h, int drop) +GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h, + int drop) { struct GNUNET_DATASTORE_QueueEntry *qe; @@ -338,12 +339,12 @@ GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h, int drop) GNUNET_CLIENT_notify_transmit_ready_cancel (h->th); h->th = NULL; } - if (h->client != NULL) + if (NULL != h->client) { GNUNET_CLIENT_disconnect (h->client); h->client = NULL; } - if (h->reconnect_task != NULL) + if (NULL != h->reconnect_task) { GNUNET_SCHEDULER_cancel (h->reconnect_task); h->reconnect_task = NULL; @@ -356,21 +357,23 @@ GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h, int drop) if (GNUNET_YES == drop) { h->client = GNUNET_CLIENT_connect ("datastore", h->cfg); - if (h->client != NULL) + if (NULL != h->client) { if (NULL != GNUNET_CLIENT_notify_transmit_ready (h->client, sizeof (struct GNUNET_MessageHeader), - GNUNET_TIME_UNIT_MINUTES, - GNUNET_YES, &transmit_drop, h)) + GNUNET_TIME_UNIT_SECONDS, + GNUNET_YES, + &transmit_drop, h)) return; GNUNET_CLIENT_disconnect (h->client); h->client = NULL; } GNUNET_break (0); } - GNUNET_STATISTICS_destroy (h->stats, GNUNET_NO); + GNUNET_STATISTICS_destroy (h->stats, + GNUNET_NO); h->stats = NULL; GNUNET_free (h); } @@ -396,8 +399,12 @@ timeout_queue_entry (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) LOG (GNUNET_ERROR_TYPE_DEBUG, "Timeout of request in datastore queue\n"); /* response_proc's expect request at the head of the queue! */ - GNUNET_CONTAINER_DLL_remove (h->queue_head, h->queue_tail, qe); - GNUNET_CONTAINER_DLL_insert (h->queue_head, h->queue_tail, qe); + GNUNET_CONTAINER_DLL_remove (h->queue_head, + h->queue_tail, + qe); + GNUNET_CONTAINER_DLL_insert (h->queue_head, + h->queue_tail, + qe); GNUNET_assert (h->queue_head == qe); qe->response_proc (qe->h, NULL); } @@ -418,8 +425,10 @@ timeout_queue_entry (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) * @return NULL if the queue is full */ static struct GNUNET_DATASTORE_QueueEntry * -make_queue_entry (struct GNUNET_DATASTORE_Handle *h, size_t msize, - unsigned int queue_priority, unsigned int max_queue_size, +make_queue_entry (struct GNUNET_DATASTORE_Handle *h, + size_t msize, + unsigned int queue_priority, + unsigned int max_queue_size, struct GNUNET_TIME_Relative timeout, GNUNET_CLIENT_MessageHandler response_proc, const union QueueContext *qc) @@ -856,13 +865,18 @@ process_status_message (void *cls, * (or rather, will already have been invoked) */ struct GNUNET_DATASTORE_QueueEntry * -GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, uint32_t rid, - const struct GNUNET_HashCode * key, size_t size, - const void *data, enum GNUNET_BLOCK_Type type, - uint32_t priority, uint32_t anonymity, +GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, + uint32_t rid, + const struct GNUNET_HashCode *key, + size_t size, + const void *data, + enum GNUNET_BLOCK_Type type, + uint32_t priority, + uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, - unsigned int queue_priority, unsigned int max_queue_size, + unsigned int queue_priority, + unsigned int max_queue_size, struct GNUNET_TIME_Relative timeout, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls) @@ -881,7 +895,11 @@ GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, uint32_t rid, GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); qc.sc.cont = cont; qc.sc.cont_cls = cont_cls; - qe = make_queue_entry (h, msize, queue_priority, max_queue_size, timeout, + qe = make_queue_entry (h, + msize, + queue_priority, + max_queue_size, + timeout, &process_status_message, &qc); if (qe == NULL) { diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index 77e96c94f..3561fb406 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c @@ -224,8 +224,14 @@ static int stats_worked; static void sync_stats () { - GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES); - GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload, GNUNET_NO); + GNUNET_STATISTICS_set (stats, + quota_stat_name, + payload, + GNUNET_YES); + GNUNET_STATISTICS_set (stats, + "# utilization by current datastore", + payload, + GNUNET_NO); last_sync = 0; } @@ -362,7 +368,9 @@ expired_processor (void *cls, now), GNUNET_YES)); min_expiration = now; - GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes expired"), size, + GNUNET_STATISTICS_update (stats, + gettext_noop ("# bytes expired"), + size, GNUNET_YES); GNUNET_CONTAINER_bloomfilter_remove (filter, key); expired_kill_task = @@ -640,7 +648,9 @@ transmit_item (void *cls, const struct GNUNET_HashCode * key, uint32_t size, GNUNET_STRINGS_absolute_time_to_string (expiration), GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (expiration), GNUNET_YES)); - GNUNET_STATISTICS_update (stats, gettext_noop ("# results found"), 1, + GNUNET_STATISTICS_update (stats, + gettext_noop ("# results found"), + 1, GNUNET_NO); transmit (client, &dm->header); GNUNET_SERVER_client_drop (client); @@ -709,7 +719,9 @@ handle_reserve (void *cls, struct GNUNET_SERVER_Client *client, return; } reserved += req; - GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_STATISTICS_set (stats, + gettext_noop ("# reserved"), + reserved, GNUNET_NO); e = GNUNET_new (struct ReservationList); e->next = reservations; @@ -764,7 +776,8 @@ handle_release_reserve (void *cls, GNUNET_assert (reserved >= rem); reserved -= rem; GNUNET_STATISTICS_set (stats, - gettext_noop ("# reserved"), reserved, + gettext_noop ("# reserved"), + reserved, GNUNET_NO); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Returning %llu remaining reserved bytes to storage pool\n", @@ -839,8 +852,8 @@ struct PutContext * @param msg error message on error */ static void -put_continuation (void *cls, - const struct GNUNET_HashCode *key, +put_continuation (void *cls, + const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg) @@ -850,7 +863,8 @@ put_continuation (void *cls, if (GNUNET_OK == status) { GNUNET_STATISTICS_update (stats, - gettext_noop ("# bytes stored"), size, + gettext_noop ("# bytes stored"), + size, GNUNET_YES); GNUNET_CONTAINER_bloomfilter_add (filter, key); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -870,6 +884,7 @@ put_continuation (void *cls, } } + /** * Actually put the data message. * @@ -877,7 +892,8 @@ put_continuation (void *cls, * @param dm message with the data to store */ static void -execute_put (struct GNUNET_SERVER_Client *client, const struct DataMessage *dm) +execute_put (struct GNUNET_SERVER_Client *client, + const struct DataMessage *dm) { GNUNET_SERVER_client_keep (client); plugin->api->put (plugin->api->cls, &dm->key, ntohl (dm->size), &dm[1], @@ -889,8 +905,8 @@ execute_put (struct GNUNET_SERVER_Client *client, const struct DataMessage *dm) static void -check_present_continuation (void *cls, - int status, +check_present_continuation (void *cls, + int status, const char *msg) { struct GNUNET_SERVER_Client *client = cls; @@ -918,13 +934,13 @@ check_present_continuation (void *cls, * #GNUNET_NO to delete the item */ static int -check_present (void *cls, +check_present (void *cls, const struct GNUNET_HashCode *key, uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, + const void *data, + enum GNUNET_BLOCK_Type type, uint32_t priority, - uint32_t anonymity, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, uint64_t uid) { @@ -950,7 +966,7 @@ check_present (void *cls, if ((ntohl (dm->priority) > 0) || (GNUNET_TIME_absolute_ntoh (dm->expiration).abs_value_us > expiration.abs_value_us)) - plugin->api->update (plugin->api->cls, + plugin->api->update (plugin->api->cls, uid, (int32_t) ntohl (dm->priority), GNUNET_TIME_absolute_ntoh (dm->expiration), @@ -1016,7 +1032,8 @@ handle_put (void *cls, struct GNUNET_SERVER_Client *client, pos->amount -= size; reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); GNUNET_STATISTICS_set (stats, - gettext_noop ("# reserved"), reserved, + gettext_noop ("# reserved"), + reserved, GNUNET_NO); } } @@ -1029,11 +1046,11 @@ handle_put (void *cls, struct GNUNET_SERVER_Client *client, GNUNET_SERVER_client_keep (client); memcpy (&pc[1], dm, size + sizeof (struct DataMessage)); plugin->api->get_key (plugin->api->cls, - 0, - &dm->key, + 0, + &dm->key, &vhash, ntohl (dm->type), - &check_present, + &check_present, pc); return; } @@ -1068,7 +1085,8 @@ handle_get (void *cls, struct GNUNET_SERVER_Client *client, "Processing `%s' request for `%s' of type %u\n", "GET", GNUNET_h2s (&msg->key), ntohl (msg->type)); GNUNET_STATISTICS_update (stats, - gettext_noop ("# GET requests received"), 1, + gettext_noop ("# GET requests received"), + 1, GNUNET_NO); GNUNET_SERVER_client_keep (client); if ((size == sizeof (struct GetMessage)) && @@ -1080,7 +1098,8 @@ handle_get (void *cls, struct GNUNET_SERVER_Client *client, "GET", GNUNET_h2s (&msg->key)); GNUNET_STATISTICS_update (stats, gettext_noop - ("# requests filtered by bloomfilter"), 1, + ("# requests filtered by bloomfilter"), + 1, GNUNET_NO); transmit_item (client, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); @@ -1094,8 +1113,8 @@ handle_get (void *cls, struct GNUNET_SERVER_Client *client, static void -update_continuation (void *cls, - int status, +update_continuation (void *cls, + int status, const char *msg) { struct GNUNET_SERVER_Client *client = cls; @@ -1118,8 +1137,10 @@ handle_update (void *cls, struct GNUNET_SERVER_Client *client, { const struct UpdateMessage *msg; - GNUNET_STATISTICS_update (stats, gettext_noop ("# UPDATE requests received"), - 1, GNUNET_NO); + GNUNET_STATISTICS_update (stats, + gettext_noop ("# UPDATE requests received"), + 1, + GNUNET_NO); msg = (const struct UpdateMessage *) message; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request for %llu\n", "UPDATE", (unsigned long long) GNUNET_ntohll (msg->uid)); @@ -1142,11 +1163,12 @@ static void handle_get_replication (void *cls, struct GNUNET_SERVER_Client *client, const struct GNUNET_MessageHeader *message) { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' request\n", "GET_REPLICATION"); GNUNET_STATISTICS_update (stats, - gettext_noop - ("# GET REPLICATION requests received"), 1, + gettext_noop ("# GET REPLICATION requests received"), + 1, GNUNET_NO); GNUNET_SERVER_client_keep (client); plugin->api->get_replication (plugin->api->cls, &transmit_item, client); @@ -1175,11 +1197,12 @@ handle_get_zero_anonymity (void *cls, struct GNUNET_SERVER_Client *client, GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); return; } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' request\n", "GET_ZERO_ANONYMITY"); GNUNET_STATISTICS_update (stats, - gettext_noop - ("# GET ZERO ANONYMITY requests received"), 1, + gettext_noop ("# GET ZERO ANONYMITY requests received"), + 1, GNUNET_NO); GNUNET_SERVER_client_keep (client); plugin->api->get_zero_anonymity (plugin->api->cls, @@ -1203,17 +1226,24 @@ remove_callback (void *cls, const struct GNUNET_HashCode * key, uint32_t size, if (key == NULL) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No further matches for `%s' request.\n", "REMOVE"); - transmit_status (client, GNUNET_NO, _("Content not found")); + "No further matches for `%s' request.\n", + "REMOVE"); + transmit_status (client, + GNUNET_NO, + _("Content not found")); GNUNET_SERVER_client_drop (client); return GNUNET_OK; /* last item */ } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Item %llu matches `%s' request for key `%s' and type %u.\n", - (unsigned long long) uid, "REMOVE", GNUNET_h2s (key), type); + (unsigned long long) uid, + "REMOVE", + GNUNET_h2s (key), + type); GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes removed (explicit request)"), - size, GNUNET_YES); + size, + GNUNET_YES); GNUNET_CONTAINER_bloomfilter_remove (filter, key); transmit_status (client, GNUNET_OK, NULL); GNUNET_SERVER_client_drop (client); @@ -1449,7 +1479,8 @@ add_key_to_bloomfilter (void *cls, * @param success #GNUNET_NO if we failed to read the stat */ static void -process_stat_done (void *cls, int success) +process_stat_done (void *cls, + int success) { stat_get = NULL; @@ -1469,7 +1500,8 @@ process_stat_done (void *cls, int success) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to obtain value from statistics service, recomputing it\n"); - plugin->api->estimate_size (plugin->api->cls, &payload); + plugin->api->estimate_size (plugin->api->cls, + &payload); } if (GNUNET_YES == refresh_bf) { @@ -1594,7 +1626,9 @@ cleanup_reservations (void *cls, } pos = next; } - GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_STATISTICS_set (stats, + gettext_noop ("# reserved"), + reserved, GNUNET_NO); } @@ -1618,11 +1652,14 @@ run (void *cls, server = serv; cfg = c; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + GNUNET_CONFIGURATION_get_value_string (cfg, + "DATASTORE", + "DATABASE", &plugin_name)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + _("No `%s' specified for `%s' in configuration!\n"), + "DATABASE", "DATASTORE"); return; } @@ -1633,7 +1670,8 @@ run (void *cls, GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", "a)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), "QUOTA", + _("No `%s' specified for `%s' in configuration!\n"), + "QUOTA", "DATASTORE"); return; } @@ -1648,7 +1686,9 @@ run (void *cls, bf_size = quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */ fn = NULL; if ((GNUNET_OK != - GNUNET_CONFIGURATION_get_value_filename (cfg, "DATASTORE", "BLOOMFILTER", + GNUNET_CONFIGURATION_get_value_filename (cfg, + "DATASTORE", + "BLOOMFILTER", &fn)) || (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn))) { diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c index 082b583c6..c3dbad00a 100644 --- a/src/datastore/perf_datastore_api.c +++ b/src/datastore/perf_datastore_api.c @@ -407,11 +407,22 @@ run_continuation (void *cls, "DATASTORE-%s", plugin_name); if ((crc->i == ITERATIONS) && (stored_ops > 0)) + { GAUGER (gstr, "PUT operation duration", GNUNET_TIME_absolute_get_duration (start_time).rel_value_us / 1000LL / stored_ops, "ms/operation"); + fprintf (stdout, + "\nPUT performance: %s for %llu operations\n", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (start_time), + GNUNET_YES), + stored_ops); + fprintf (stdout, + "PUT performance: %llu ms/operation\n", + GNUNET_TIME_absolute_get_duration (start_time).rel_value_us / 1000LL / + stored_ops); + } GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); GNUNET_free (crc); @@ -493,7 +504,7 @@ run (void *cls, 0, 0, 0, GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_SECONDS), 0, 1, - GNUNET_TIME_UNIT_MINUTES, + TIMEOUT, &run_tests, crc)) { FPRINTF (stderr, diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c index 36e637845..a8011bd53 100644 --- a/src/datastore/test_datastore_api.c +++ b/src/datastore/test_datastore_api.c @@ -639,7 +639,7 @@ run (void *cls, GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_SECONDS), 0, 1, - GNUNET_TIME_UNIT_MINUTES, + TIMEOUT, &run_tests, crc)) { FPRINTF (stderr, diff --git a/src/datastore/test_datastore_api_data_mysql.conf b/src/datastore/test_datastore_api_data_mysql.conf index f79c62d17..de6fff7ab 100644 --- a/src/datastore/test_datastore_api_data_mysql.conf +++ b/src/datastore/test_datastore_api_data_mysql.conf @@ -2,19 +2,9 @@ [PATHS] GNUNET_TEST_HOME = /tmp/test-gnunet-datastore-mysql/ -[arm] -PORT = 42466 - -[statistics] -PORT = 22667 - -[resolver] -PORT = 42464 - [datastore] QUOTA = 10 MB DATABASE = mysql [datastore-mysql] DATABASE = gnunetcheck - diff --git a/src/datastore/test_datastore_api_data_postgres.conf b/src/datastore/test_datastore_api_data_postgres.conf index c30f1f10e..f436530d2 100644 --- a/src/datastore/test_datastore_api_data_postgres.conf +++ b/src/datastore/test_datastore_api_data_postgres.conf @@ -2,20 +2,9 @@ [PATHS] GNUNET_TEST_HOME = /tmp/test-gnunet-datastore-postgres/ -[arm] -PORT = 42466 - -[statistics] -PORT = 22667 - -[resolver] -PORT = 42464 - [datastore] QUOTA = 10 MB DATABASE = postgres [datastore-postgres] CONFIG = dbname=gnunetcheck - - diff --git a/src/datastore/test_datastore_api_data_sqlite.conf b/src/datastore/test_datastore_api_data_sqlite.conf index 6a6e00787..894945586 100644 --- a/src/datastore/test_datastore_api_data_sqlite.conf +++ b/src/datastore/test_datastore_api_data_sqlite.conf @@ -2,16 +2,6 @@ [PATHS] GNUNET_TEST_HOME = /tmp/test-gnunet-datastore-sqlite/ -[arm] -PORT = 42466 - -[statistics] -PORT = 22667 - -[resolver] -PORT = 42464 - [datastore] QUOTA = 10 MB - - +DATABASE = sqlite diff --git a/src/datastore/test_defaults.conf b/src/datastore/test_defaults.conf index 1edde7c5a..67d782f3a 100644 --- a/src/datastore/test_defaults.conf +++ b/src/datastore/test_defaults.conf @@ -1,6 +1,10 @@ +@inline@ ../../contrib/no_autostart_above_core.conf +@inline@ ../../contrib/no_forcestart.conf + [datastore] PORT = 22654 QUOTA = 1 MB +AUTOSTART = YES [nse] -WORKBITS = 1 \ No newline at end of file +WORKBITS = 1 -- cgit v1.2.3