/*
This file is part of GNUnet.
Copyright (C) 2011 - 2017 GNUnet e.V.
GNUnet is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.
GNUnet is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
SPDX-License-Identifier: AGPL3.0-or-later
*/
/**
* @file regex/gnunet-regex-profiler.c
* @brief Regex profiler for testing distributed regex use.
* @author Bartlomiej Polot
* @author Maximilian Szengel
*
*/
#include
#include "platform.h"
#include "gnunet_applications.h"
#include "gnunet_util_lib.h"
#include "regex_internal_lib.h"
#include "gnunet_arm_service.h"
#include "gnunet_dht_service.h"
#include "gnunet_testbed_service.h"
#define FIND_TIMEOUT \
GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
/**
* DLL of operations
*/
struct DLLOperation
{
/**
* The testbed operation handle
*/
struct GNUNET_TESTBED_Operation *op;
/**
* Closure
*/
void *cls;
/**
* The next pointer for DLL
*/
struct DLLOperation *next;
/**
* The prev pointer for DLL
*/
struct DLLOperation *prev;
};
/**
* Available states during profiling
*/
enum State
{
/**
* Initial state
*/
STATE_INIT = 0,
/**
* Starting slaves
*/
STATE_SLAVES_STARTING,
/**
* Creating peers
*/
STATE_PEERS_CREATING,
/**
* Starting peers
*/
STATE_PEERS_STARTING,
/**
* Linking peers
*/
STATE_PEERS_LINKING,
/**
* Matching strings against announced regexes
*/
STATE_SEARCH_REGEX,
/**
* Destroying peers; we can do this as the controller takes care of stopping a
* peer if it is running
*/
STATE_PEERS_DESTROYING
};
/**
* Peer handles.
*/
struct RegexPeer
{
/**
* Peer id.
*/
unsigned int id;
/**
* Peer configuration handle.
*/
struct GNUNET_CONFIGURATION_Handle *cfg;
/**
* The actual testbed peer handle.
*/
struct GNUNET_TESTBED_Peer *peer_handle;
/**
* Peer's search string.
*/
const char *search_str;
/**
* Set to GNUNET_YES if the peer successfully matched the above
* search string. GNUNET_NO if the string could not be matched
* during the profiler run. GNUNET_SYSERR if the string matching
* timed out. Undefined if search_str is NULL
*/
int search_str_matched;
/**
* Peer's DHT handle.
*/
struct GNUNET_DHT_Handle *dht_handle;
/**
* Handle to a running regex search.
*/
struct REGEX_INTERNAL_Search *search_handle;
/**
* Testbed operation handle for DHT.
*/
struct GNUNET_TESTBED_Operation *op_handle;
/**
* Peers's statistics handle.
*/
struct GNUNET_STATISTICS_Handle *stats_handle;
/**
* The starting time of a profiling step.
*/
struct GNUNET_TIME_Absolute prof_start_time;
/**
* Operation timeout
*/
struct GNUNET_SCHEDULER_Task *timeout;
/**
* Deamon start
*/
struct GNUNET_TESTBED_Operation *daemon_op;
};
/**
* Set when shutting down to avoid making more queries.
*/
static int in_shutdown;
/**
* The array of peers; we fill this as the peers are given to us by the testbed
*/
static struct RegexPeer *peers;
/**
* Host registration handle
*/
static struct GNUNET_TESTBED_HostRegistrationHandle *reg_handle;
/**
* Handle to the master controller process
*/
static struct GNUNET_TESTBED_ControllerProc *mc_proc;
/**
* Handle to the master controller
*/
static struct GNUNET_TESTBED_Controller *mc;
/**
* Handle to global configuration
*/
static struct GNUNET_CONFIGURATION_Handle *cfg;
/**
* Abort task identifier
*/
static struct GNUNET_SCHEDULER_Task *abort_task;
/**
* Host registration task identifier
*/
static struct GNUNET_SCHEDULER_Task *register_hosts_task;
/**
* Global event mask for all testbed events
*/
static uint64_t event_mask;
/**
* The starting time of a profiling step
*/
static struct GNUNET_TIME_Absolute prof_start_time;
/**
* Duration profiling step has taken
*/
static struct GNUNET_TIME_Relative prof_time;
/**
* Number of peers to be started by the profiler
*/
static unsigned int num_peers;
/**
* Global testing status
*/
static int result;
/**
* current state of profiling
*/
enum State state;
/**
* Folder where policy files are stored.
*/
static char *policy_dir;
/**
* File with hostnames where to execute the test.
*/
static char *hosts_file;
/**
* File with the strings to look for.
*/
static char *strings_file;
/**
* Search strings (num_peers of them).
*/
static char **search_strings;
/**
* How many searches are we going to start in parallel
*/
static long long unsigned int init_parallel_searches;
/**
* How many searches are running in parallel
*/
static unsigned int parallel_searches;
/**
* Number of strings found in the published regexes.
*/
static unsigned int strings_found;
/**
* Index of peer to start next announce/search.
*/
static unsigned int next_search;
/**
* Search timeout task identifier.
*/
static struct GNUNET_SCHEDULER_Task *search_timeout_task;
/**
* Search timeout in seconds.
*/
static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
/**
* File to log statistics to.
*/
static struct GNUNET_DISK_FileHandle *data_file;
/**
* Filename to log statistics to.
*/
static char *data_filename;
/**
* Prefix used for regex announcing. We need to prefix the search
* strings with it, in order to find something.
*/
static char *regex_prefix;
/**
* What's the maximum regex reannounce period.
*/
static struct GNUNET_TIME_Relative reannounce_period_max;
/******************************************************************************/
/****************************** DECLARATIONS ********************************/
/******************************************************************************/
/**
* DHT connect callback.
*
* @param cls internal peer id.
* @param op operation handle.
* @param ca_result connect adapter result.
* @param emsg error message.
*/
static void
dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
void *ca_result, const char *emsg);
/**
* DHT connect adapter.
*
* @param cls not used.
* @param cfg configuration handle.
*
* @return
*/
static void *
dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
/**
* Adapter function called to destroy a connection to
* the DHT service
*
* @param cls closure
* @param op_result service handle returned from the connect adapter
*/
static void
dht_da (void *cls, void *op_result);
/**
* Function called by testbed once we are connected to stats
* service. Get the statistics for the services of interest.
*
* @param cls the 'struct RegexPeer' for which we connected to stats
* @param op connect operation handle
* @param ca_result handle to stats service
* @param emsg error message on failure
*/
static void
stats_connect_cb (void *cls,
struct GNUNET_TESTBED_Operation *op,
void *ca_result,
const char *emsg);
/**
* Start announcing the next regex in the DHT.
*
* @param cls Index of the next peer in the peers array.
*/
static void
announce_next_regex (void *cls);
/******************************************************************************/
/******************************** SHUTDOWN **********************************/
/******************************************************************************/
/**
* Shutdown nicely
*
* @param cls NULL
*/
static void
do_shutdown (void *cls)
{
struct RegexPeer *peer;
unsigned int peer_cnt;
unsigned int search_str_cnt;
char output_buffer[512];
size_t size;
if (NULL != abort_task)
{
GNUNET_SCHEDULER_cancel (abort_task);
abort_task = NULL;
}
if (NULL != register_hosts_task)
{
GNUNET_SCHEDULER_cancel (register_hosts_task);
register_hosts_task = NULL;
}
for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
{
peer = &peers[peer_cnt];
if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
{
prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
size =
GNUNET_snprintf (output_buffer,
sizeof(output_buffer),
"%p Search string not found: %s (%d)\n"
"%p On peer: %u (%p)\n"
"%p After: %s\n",
peer, peer->search_str, peer->search_str_matched,
peer, peer->id, peer,
peer,
GNUNET_STRINGS_relative_time_to_string (prof_time,
GNUNET_NO));
if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
}
if (NULL != peers[peer_cnt].op_handle)
GNUNET_TESTBED_operation_done (peers[peer_cnt].op_handle);
}
if (NULL != data_file)
{
GNUNET_DISK_file_close (data_file);
data_file = NULL;
}
for (search_str_cnt = 0;
search_str_cnt < num_peers && NULL != search_strings;
search_str_cnt++)
{
GNUNET_free_non_null (search_strings[search_str_cnt]);
}
GNUNET_free_non_null (search_strings);
search_strings = NULL;
if (NULL != reg_handle)
{
GNUNET_TESTBED_cancel_registration (reg_handle);
reg_handle = NULL;
}
if (NULL != mc)
{
GNUNET_TESTBED_controller_disconnect (mc);
mc = NULL;
}
if (NULL != mc_proc)
{
GNUNET_TESTBED_controller_stop (mc_proc);
mc_proc = NULL;
}
if (NULL != cfg)
{
GNUNET_CONFIGURATION_destroy (cfg);
cfg = NULL;
}
}
/**
* abort task to run on test timed out
*
* @param cls NULL
*/
static void
do_abort (void *cls)
{
unsigned long i = (unsigned long) cls;
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Aborting from line %lu...\n", i);
abort_task = NULL;
result = GNUNET_SYSERR;
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
}
/******************************************************************************/
/********************* STATISTICS SERVICE CONNECTIONS ***********************/
/******************************************************************************/
/**
* Adapter function called to establish a connection to
* statistics service.
*
* @param cls closure
* @param cfg configuration of the peer to connect to; will be available until
* GNUNET_TESTBED_operation_done() is called on the operation returned
* from GNUNET_TESTBED_service_connect()
* @return service handle to return in 'op_result', NULL on error
*/
static void *
stats_ca (void *cls,
const struct GNUNET_CONFIGURATION_Handle *cfg)
{
return GNUNET_STATISTICS_create ("", cfg);
}
/**
* Adapter function called to destroy a connection to
* statistics service.
*
* @param cls closure
* @param op_result service handle returned from the connect adapter
*/
static void
stats_da (void *cls, void *op_result)
{
struct RegexPeer *peer = cls;
GNUNET_assert (op_result == peer->stats_handle);
GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
peer->stats_handle = NULL;
}
/**
* Process statistic values. Write all values to global 'data_file', if present.
*
* @param cls closure
* @param subsystem name of subsystem that created the statistic
* @param name the name of the datum
* @param value the current value
* @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
* @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
*/
static int
stats_iterator (void *cls,
const char *subsystem,
const char *name,
uint64_t value, int is_persistent)
{
struct RegexPeer *peer = cls;
char output_buffer[512];
size_t size;
if (NULL == data_file)
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"%p -> %s [%s]: %llu\n",
peer,
subsystem,
name,
(unsigned long long) value);
return GNUNET_OK;
}
size =
GNUNET_snprintf (output_buffer,
sizeof(output_buffer),
"%p [%s] %llu %s\n",
peer,
subsystem, value, name);
if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Unable to write to file!\n");
return GNUNET_OK;
}
/**
* Stats callback. Finish the stats testbed operation and when all stats have
* been iterated, shutdown the profiler.
*
* @param cls closure
* @param success GNUNET_OK if statistics were
* successfully obtained, GNUNET_SYSERR if not.
*/
static void
stats_cb (void *cls,
int success)
{
static unsigned int peer_cnt;
struct RegexPeer *peer = cls;
if (GNUNET_OK != success)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Getting statistics for peer %u failed!\n",
peer->id);
return;
}
GNUNET_assert (NULL != peer->op_handle);
GNUNET_TESTBED_operation_done (peer->op_handle);
peer->op_handle = NULL;
peer_cnt++;
peer = &peers[peer_cnt];
fprintf (stderr, "s");
if (peer_cnt == num_peers)
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"\nCollecting stats finished. Shutting down.\n");
GNUNET_SCHEDULER_shutdown ();
result = GNUNET_OK;
}
else
{
peer->op_handle =
GNUNET_TESTBED_service_connect (NULL,
peer->peer_handle,
"statistics",
&stats_connect_cb,
peer,
&stats_ca,
&stats_da,
peer);
}
}
/**
* Function called by testbed once we are connected to stats
* service. Get the statistics for the services of interest.
*
* @param cls the 'struct RegexPeer' for which we connected to stats
* @param op connect operation handle
* @param ca_result handle to stats service
* @param emsg error message on failure
*/
static void
stats_connect_cb (void *cls,
struct GNUNET_TESTBED_Operation *op,
void *ca_result,
const char *emsg)
{
struct RegexPeer *peer = cls;
if ((NULL == ca_result) || (NULL != emsg))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Failed to connect to statistics service on peer %u: %s\n",
peer->id, emsg);
peer->stats_handle = NULL;
return;
}
peer->stats_handle = ca_result;
if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
&stats_cb,
&stats_iterator, peer))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Could not get statistics of peer %u!\n", peer->id);
}
}
/**
* Task to collect all statistics from all peers, will shutdown the
* profiler, when done.
*
* @param cls NULL
*/
static void
do_collect_stats (void *cls)
{
struct RegexPeer *peer = &peers[0];
GNUNET_assert (NULL != peer->peer_handle);
peer->op_handle =
GNUNET_TESTBED_service_connect (NULL,
peer->peer_handle,
"statistics",
&stats_connect_cb,
peer,
&stats_ca,
&stats_da,
peer);
}
/******************************************************************************/
/************************ REGEX FIND CONNECTIONS **************************/
/******************************************************************************/
/**
* Start searching for the next string in the DHT.
*
* @param cls Index of the next peer in the peers array.
*/
static void
find_string (void *cls);
/**
* Method called when we've found a peer that announced a regex
* that matches our search string. Now get the statistics.
*
* @param cls Closure provided in REGEX_INTERNAL_search.
* @param id Peer providing a regex that matches the string.
* @param get_path Path of the get request.
* @param get_path_length Lenght of get_path.
* @param put_path Path of the put request.
* @param put_path_length Length of the put_path.
*/
static void
regex_found_handler (void *cls,
const struct GNUNET_PeerIdentity *id,
const struct GNUNET_PeerIdentity *get_path,
unsigned int get_path_length,
const struct GNUNET_PeerIdentity *put_path,
unsigned int put_path_length)
{
struct RegexPeer *peer = cls;
char output_buffer[512];
size_t size;
if (GNUNET_YES == peer->search_str_matched)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"String %s on peer %u already matched!\n",
peer->search_str, peer->id);
return;
}
strings_found++;
parallel_searches--;
if (NULL != peer->timeout)
{
GNUNET_SCHEDULER_cancel (peer->timeout);
peer->timeout = NULL;
if (GNUNET_NO == in_shutdown)
GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
}
if (NULL == id)
{
// FIXME not possible right now
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"String matching timed out for string %s on peer %u (%i/%i)\n",
peer->search_str, peer->id, strings_found, num_peers);
peer->search_str_matched = GNUNET_SYSERR;
}
else
{
prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"String %s found on peer %u after %s (%i/%i) (%u||)\n",
peer->search_str, peer->id,
GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO),
strings_found, num_peers, parallel_searches);
peer->search_str_matched = GNUNET_YES;
if (NULL != data_file)
{
size =
GNUNET_snprintf (output_buffer,
sizeof(output_buffer),
"%p Peer: %u\n"
"%p Search string: %s\n"
"%p Search duration: %s\n\n",
peer, peer->id,
peer, peer->search_str,
peer,
GNUNET_STRINGS_relative_time_to_string (prof_time,
GNUNET_NO));
if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
}
}
GNUNET_TESTBED_operation_done (peer->op_handle);
peer->op_handle = NULL;
if (strings_found == num_peers)
{
prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"All strings successfully matched in %s\n",
GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO));
if (NULL != search_timeout_task)
{
GNUNET_SCHEDULER_cancel (search_timeout_task);
search_timeout_task = NULL;
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
}
}
/**
* Connect by string timeout task. This will cancel the profiler after the
* specified timeout 'search_timeout'.
*
* @param cls NULL
*/
static void
search_timed_out (void *cls)
{
unsigned int i;
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Finding matches to all strings did not succeed after %s.\n",
GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
GNUNET_NO));
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Found %i of %i strings\n", strings_found, num_peers);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Search timed out after %s."
"Collecting stats and shutting down.\n",
GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
GNUNET_NO));
in_shutdown = GNUNET_YES;
for (i = 0; i < num_peers; i++)
{
if (NULL != peers[i].op_handle)
{
GNUNET_TESTBED_operation_done (peers[i].op_handle);
peers[i].op_handle = NULL;
}
}
GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL);
}
/**
* Search timed out. It might still complete in the future,
* but we should start another one.
*
* @param cls Index of the next peer in the peers array.
*/
static void
find_timed_out (void *cls)
{
struct RegexPeer *p = cls;
p->timeout = NULL;
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Searching for string \"%s\" on peer %d timed out.\n",
p->search_str,
p->id);
if (GNUNET_NO == in_shutdown)
GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
}
/**
* Start searching for a string in the DHT.
*
* @param cls Index of the next peer in the peers array.
*/
static void
find_string (void *cls)
{
unsigned int search_peer = (unsigned int) (long) cls;
if ((search_peer >= num_peers) ||
(GNUNET_YES == in_shutdown))
return;
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Searching for string \"%s\" on peer %d (%u||)\n",
peers[search_peer].search_str,
search_peer,
parallel_searches);
peers[search_peer].op_handle =
GNUNET_TESTBED_service_connect (NULL,
peers[search_peer].peer_handle,
"dht",
&dht_connect_cb,
&peers[search_peer],
&dht_ca,
&dht_da,
&peers[search_peer]);
GNUNET_assert (NULL != peers[search_peer].op_handle);
peers[search_peer].timeout
= GNUNET_SCHEDULER_add_delayed (FIND_TIMEOUT,
&find_timed_out,
&peers[search_peer]);
}
/**
* Callback called when testbed has started the daemon we asked for.
*
* @param cls NULL
* @param op the operation handle
* @param emsg NULL on success; otherwise an error description
*/
static void
daemon_started (void *cls,
struct GNUNET_TESTBED_Operation *op,
const char *emsg)
{
struct RegexPeer *peer = (struct RegexPeer *) cls;
unsigned long search_peer;
unsigned int i;
GNUNET_TESTBED_operation_done (peer->daemon_op);
peer->daemon_op = NULL;
if (NULL != emsg)
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
GNUNET_assert (0);
}
else
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Deamon %u started successfully\n", peer->id);
}
/* Find a peer to look for a string matching the regex announced */
search_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
num_peers);
for (i = 0; peers[search_peer].search_str != NULL; i++)
{
search_peer = (search_peer + 1) % num_peers;
if (i > num_peers)
GNUNET_assert (0); /* we ran out of peers, must be a bug */
}
peers[search_peer].search_str = search_strings[peer->id];
peers[search_peer].search_str_matched = GNUNET_NO;
GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_saturating_multiply (
reannounce_period_max,
2),
&find_string,
(void *) search_peer);
}
/**
* Task to start the daemons on each peer so that the regexes are announced
* into the DHT.
*
* @param cls NULL
* @param tc the task context
*/
static void
do_announce (void *cls)
{
unsigned int i;
if (GNUNET_YES == in_shutdown)
return;
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Starting announce.\n");
for (i = 0; i < init_parallel_searches; i++)
{
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
" scheduling announce %u\n",
i);
(void) GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL);
}
}
/**
* Start announcing the next regex in the DHT.
*
* @param cls Closure (unused).
*/
static void
announce_next_regex (void *cls)
{
struct RegexPeer *peer;
if (GNUNET_YES == in_shutdown)
return;
if (next_search >= num_peers)
{
if (strings_found != num_peers)
{
struct GNUNET_TIME_Relative new_delay;
if (NULL != search_timeout_task)
GNUNET_SCHEDULER_cancel (search_timeout_task);
new_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15);
search_timeout_task = GNUNET_SCHEDULER_add_delayed (new_delay,
&search_timed_out,
NULL);
}
return;
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
peer = &peers[next_search];
peer->daemon_op =
GNUNET_TESTBED_peer_manage_service (NULL,
peer->peer_handle,
"regexprofiler",
&daemon_started,
peer,
1);
next_search++;
parallel_searches++;
}
/**
* DHT connect callback. Called when we are connected to the dht service for
* the peer in 'cls'. If successfull we connect to the stats service of this
* peer and then try to match the search string of this peer.
*
* @param cls internal peer id.
* @param op operation handle.
* @param ca_result connect adapter result.
* @param emsg error message.
*/
static void
dht_connect_cb (void *cls,
struct GNUNET_TESTBED_Operation *op,
void *ca_result,
const char *emsg)
{
struct RegexPeer *peer = (struct RegexPeer *) cls;
if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
GNUNET_assert (0);
}
GNUNET_assert (NULL != peer->dht_handle);
GNUNET_assert (peer->op_handle == op);
GNUNET_assert (peer->dht_handle == ca_result);
peer->search_str_matched = GNUNET_NO;
peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
peer->search_str,
®ex_found_handler, peer,
NULL);
peer->prof_start_time = GNUNET_TIME_absolute_get ();
}
/**
* DHT connect adapter. Opens a connection to the dht service.
*
* @param cls Closure (peer).
* @param cfg Configuration handle.
*
* @return
*/
static void *
dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
{
struct RegexPeer *peer = cls;
peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
return peer->dht_handle;
}
/**
* Adapter function called to destroy a connection to the dht service.
*
* @param cls Closure (peer).
* @param op_result Service handle returned from the connect adapter.
*/
static void
dht_da (void *cls, void *op_result)
{
struct RegexPeer *peer = (struct RegexPeer *) cls;
GNUNET_assert (peer->dht_handle == op_result);
if (NULL != peer->search_handle)
{
REGEX_INTERNAL_search_cancel (peer->search_handle);
peer->search_handle = NULL;
}
if (NULL != peer->dht_handle)
{
GNUNET_DHT_disconnect (peer->dht_handle);
peer->dht_handle = NULL;
}
}
/**
* Signature of a main function for a testcase.
*
* @param cls NULL
* @param h the run handle
* @param num_peers_ number of peers in 'peers'
* @param testbed_peers handle to peers run in the testbed. NULL upon timeout (see
* GNUNET_TESTBED_test_run()).
* @param links_succeeded the number of overlay link connection attempts that
* succeeded
* @param links_failed the number of overlay link connection attempts that
* failed
*/
static void
test_master (void *cls,
struct GNUNET_TESTBED_RunHandle *h,
unsigned int num_peers_,
struct GNUNET_TESTBED_Peer **testbed_peers,
unsigned int links_succeeded,
unsigned int links_failed)
{
unsigned int i;
GNUNET_assert (num_peers_ == num_peers);
prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Testbed started in %s\n",
GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO));
if (NULL != abort_task)
{
GNUNET_SCHEDULER_cancel (abort_task);
abort_task = NULL;
}
for (i = 0; i < num_peers; i++)
{
peers[i].peer_handle = testbed_peers[i];
}
if (GNUNET_NO ==
GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
{
struct GNUNET_TIME_Relative settle_time;
settle_time =
GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
10 * num_peers);
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Waiting for DHT for %s to settle new connections.\n\n",
GNUNET_STRINGS_relative_time_to_string (settle_time,
GNUNET_NO));
GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
}
else
{
GNUNET_SCHEDULER_add_now (&do_announce, NULL);
}
search_timeout_task =
GNUNET_SCHEDULER_add_delayed (search_timeout_time, &search_timed_out, NULL);
}
/**
* Function that will be called whenever something in the testbed changes.
*
* @param cls closure, NULL
* @param event information on what is happening
*/
static void
master_controller_cb (void *cls,
const struct GNUNET_TESTBED_EventInformation *event)
{
switch (event->type)
{
case GNUNET_TESTBED_ET_CONNECT:
printf (".");
break;
case GNUNET_TESTBED_ET_PEER_START:
printf ("#");
break;
default:
break;
}
fflush (stdout);
}
/******************************************************************************/
/*************************** TESTBED PEER SETUP *****************************/
/******************************************************************************/
/**
* Process the text buffer counting the non-empty lines and separating them
* with NULL characters, for later ease of copy using (as)printf.
*
* @param data Memory buffer with strings.
* @param data_size Size of the @a data buffer in bytes.
* @param str_max Maximum number of strings to return.
* @return Positive number of lines found in the buffer,
* #GNUNET_SYSERR otherwise.
*/
static int
count_and_separate_strings (char *data,
uint64_t data_size,
unsigned int str_max)
{
char *buf; // Keep track of last string to skip blank lines
unsigned int offset;
unsigned int str_cnt;
buf = data;
offset = 0;
str_cnt = 0;
while ((offset < (data_size - 1)) && (str_cnt < str_max))
{
offset++;
if (((data[offset] == '\n')) &&
(buf != &data[offset]))
{
data[offset] = '\0';
str_cnt++;
buf = &data[offset + 1];
}
else if ((data[offset] == '\n') ||
(data[offset] == '\0'))
buf = &data[offset + 1];
}
return str_cnt;
}
/**
* Allocate a string array and fill it with the prefixed strings
* from a pre-processed, NULL-separated memory region.
*
* @param data Preprocessed memory with strings
* @param data_size Size of the @a data buffer in bytes.
* @param strings Address of the string array to be created.
* Must be freed by caller if function end in success.
* @param str_cnt String count. The @a data buffer should contain
* at least this many NULL-separated strings.
* @return #GNUNET_OK in ase of success, #GNUNET_SYSERR otherwise.
* In case of error @a strings must not be freed.
*/
static int
create_string_array (char *data, uint64_t data_size,
char ***strings, unsigned int str_cnt)
{
uint64_t offset;
uint64_t len;
unsigned int i;
*strings = GNUNET_malloc (sizeof(char *) * str_cnt);
offset = 0;
for (i = 0; i < str_cnt; i++)
{
len = strlen (&data[offset]);
if (offset + len >= data_size)
{
GNUNET_free (*strings);
*strings = NULL;
return GNUNET_SYSERR;
}
if (0 == len) // empty line
{
offset++;
i--;
continue;
}
GNUNET_asprintf (&(*strings)[i],
"%s%s",
regex_prefix,
&data[offset]);
offset += len + 1;
}
return GNUNET_OK;
}
/**
* Load search strings from given filename. One search string per line.
*
* @param filename filename of the file containing the search strings.
* @param strings set of strings loaded from file. Caller needs to free this
* if number returned is greater than zero.
* @param limit upper limit on the number of strings read from the file
* @return number of strings found in the file. #GNUNET_SYSERR on error.
*/
static int
load_search_strings (const char *filename,
char ***strings,
unsigned int limit)
{
char *data;
uint64_t filesize;
int str_cnt;
/* Sanity checks */
if (NULL == filename)
{
return GNUNET_SYSERR;
}
if (GNUNET_YES != GNUNET_DISK_file_test (filename))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Could not find search strings file %s\n", filename);
return GNUNET_SYSERR;
}
if (GNUNET_OK !=
GNUNET_DISK_file_size (filename,
&filesize,
GNUNET_YES,
GNUNET_YES))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Search strings file %s cannot be read.\n",
filename);
return GNUNET_SYSERR;
}
if (0 == filesize)
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Search strings file %s is empty.\n",
filename);
return GNUNET_SYSERR;
}
/* Read data into memory */
data = GNUNET_malloc (filesize + 1);
if (filesize != GNUNET_DISK_fn_read (filename,
data,
filesize))
{
GNUNET_free (data);
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Could not read search strings file %s.\n",
filename);
return GNUNET_SYSERR;
}
/* Process buffer and build array */
str_cnt = count_and_separate_strings (data, filesize, limit);
if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
{
str_cnt = GNUNET_SYSERR;
}
GNUNET_free (data);
return str_cnt;
}
/**
* Main function that will be run by the scheduler.
*
* @param cls closure
* @param args remaining command-line arguments
* @param cfgfile name of the configuration file used (for saving, can be NULL!)
* @param config configuration
*/
static void
run (void *cls,
char *const *args,
const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *config)
{
unsigned int nsearchstrs;
unsigned int i;
struct GNUNET_TIME_Relative abort_time;
in_shutdown = GNUNET_NO;
/* Check config */
if (NULL == config)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ ("No configuration file given. Exiting\n"));
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
cfg = GNUNET_CONFIGURATION_dup (config);
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
"REGEX_PREFIX",
®ex_prefix))
{
GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
"regexprofiler",
"regex_prefix");
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
"PARALLEL_SEARCHES",
&init_parallel_searches))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Configuration option \"PARALLEL_SEARCHES\" missing."
" Using default (%d)\n", 10);
init_parallel_searches = 10;
}
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
"REANNOUNCE_PERIOD_MAX",
&reannounce_period_max))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"reannounce_period_max not given. Using 10 minutes.\n");
reannounce_period_max =
GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 10);
}
/* Check arguments */
if (NULL == policy_dir)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ (
"No policy directory specified on command line. Exiting.\n"));
return;
}
if (GNUNET_YES != GNUNET_DISK_directory_test (policy_dir, GNUNET_YES))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ ("Specified policies directory does not exist. Exiting.\n"));
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
NULL)))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ ("No files found in `%s'\n"),
policy_dir);
return;
}
GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
"POLICY_DIR", policy_dir);
if (GNUNET_YES != GNUNET_DISK_file_test (strings_file))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ ("No search strings file given. Exiting.\n"));
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
nsearchstrs = load_search_strings (strings_file,
&search_strings,
num_peers);
if (num_peers != nsearchstrs)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Error loading search strings.\n");
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"File (%s) does not contain enough strings (%u/%u).\n",
strings_file, nsearchstrs, num_peers);
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
if ((0 == num_peers) || (NULL == search_strings))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_ ("Error loading search strings. Exiting.\n"));
GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
return;
}
for (i = 0; i < num_peers; i++)
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"search string: %s\n",
search_strings[i]);
/* Check logfile */
if ((NULL != data_filename) &&
(NULL == (data_file =
GNUNET_DISK_file_open (data_filename,
GNUNET_DISK_OPEN_READWRITE
| GNUNET_DISK_OPEN_TRUNCATE
| GNUNET_DISK_OPEN_CREATE,
GNUNET_DISK_PERM_USER_READ
| GNUNET_DISK_PERM_USER_WRITE))))
{
GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
"open",
data_filename);
return;
}
/* Initialize peers */
peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
for (i = 0; i < num_peers; i++)
peers[i].id = i;
GNUNET_CONFIGURATION_set_value_number (cfg,
"TESTBED", "OVERLAY_RANDOM_LINKS",
num_peers * 20);
GNUNET_CONFIGURATION_set_value_number (cfg,
"DHT", "FORCE_NSE",
(long long unsigned)
(log (num_peers) / log (2.0)));
event_mask = 0LL;
/* For feedback about the start process activate these and pass master_cb */
event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_START);
// event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT);
// event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
prof_start_time = GNUNET_TIME_absolute_get ();
GNUNET_TESTBED_run (hosts_file,
cfg,
num_peers,
event_mask,
&master_controller_cb,
NULL, /* master_controller_cb cls */
&test_master,
NULL); /* test_master cls */
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED",
"SETUP_TIMEOUT",
&abort_time))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"SETUP_TIMEOUT not given. Using 15 minutes.\n");
abort_time =
GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15);
}
abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
abort_task =
GNUNET_SCHEDULER_add_delayed (abort_time,
&do_abort,
(void *) __LINE__);
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"setup_timeout: %s\n",
GNUNET_STRINGS_relative_time_to_string (abort_time, GNUNET_YES));
}
/**
* Main function.
*
* @param argc argument count
* @param argv argument values
* @return 0 on success
*/
int
main (int argc, char *const *argv)
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
GNUNET_GETOPT_option_filename ('o',
"output-file",
"FILENAME",
gettext_noop (
"name of the file for writing statistics"),
&data_filename),
GNUNET_GETOPT_option_relative_time ('t',
"matching-timeout",
"TIMEOUT",
gettext_noop (
"wait TIMEOUT before ending the experiment"),
&search_timeout_time),
GNUNET_GETOPT_option_filename ('p',
"policy-dir",
"DIRECTORY",
gettext_noop ("directory with policy files"),
&policy_dir),
GNUNET_GETOPT_option_filename ('s',
"strings-file",
"FILENAME",
gettext_noop (
"name of file with input strings"),
&strings_file),
GNUNET_GETOPT_option_filename ('H',
"hosts-file",
"FILENAME",
gettext_noop (
"name of file with hosts' names"),
&hosts_file),
GNUNET_GETOPT_OPTION_END
};
int ret;
if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
return 2;
result = GNUNET_SYSERR;
ret =
GNUNET_PROGRAM_run (argc, argv,
"gnunet-regex-profiler",
_ ("Profiler for regex"),
options, &run, NULL);
if (GNUNET_OK != ret)
return ret;
if (GNUNET_OK != result)
return 1;
return 0;
}