From e3a7cd0f53f7a91b27b95e9b7fc0786f52c9e9ac Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 15 May 2010 21:39:13 +0000 Subject: work on migration --- TODO | 13 - configure.ac | 1 - src/Makefile.am | 4 +- src/fs/fs.h | 7 + src/fs/gnunet-service-fs.c | 40 ++- src/migration/Makefile.am | 34 -- src/migration/gnunet-daemon-migration.c | 378 --------------------- src/migration/test_gnunet_daemon_migration.c | 184 ---------- .../test_gnunet_daemon_migration_data.conf | 37 -- src/util/scheduler.c | 3 +- 10 files changed, 48 insertions(+), 653 deletions(-) delete mode 100644 src/migration/Makefile.am delete mode 100644 src/migration/gnunet-daemon-migration.c delete mode 100644 src/migration/test_gnunet_daemon_migration.c delete mode 100644 src/migration/test_gnunet_daemon_migration_data.conf diff --git a/TODO b/TODO index 11a33e8e8..867eb27a1 100644 --- a/TODO +++ b/TODO @@ -1,25 +1,12 @@ 0.9.0pre1: -* FS-LIB: - - _start functions should return void * MIGRATION [CG] - on-demand encoding => move logic to block-library!? - peer selection => how to consider latency/bw/etc? - content transmission => how often the same block? - how to select delay before next migration? - - migration to us - testing - - integrate with FS or not? (peer list, index/on-demand encoding, block code, - inbound priority assignment; all would be easier with tight integration!) * FS: [CG] - gnunet-service-fs (hot-path routing, load-based routing, nitpicks) - - [gnunet-service-fs.c:208]: member 'LocalGetContext::results_bf_size' is never used - - [gnunet-service-fs.c:501]: member 'PendingRequest::used_pids_size' is never used - - [gnunet-service-fs.c:654]: member 'ConnectedPeer::last_client_replies' is never used - - [gnunet-service-fs.c:669]: member 'ConnectedPeer::avg_delay' is never used - - [gnunet-service-fs.c:675]: member 'ConnectedPeer::avg_priority' is never used - - [gnunet-service-fs.c:688]: member 'ConnectedPeer::pending_requests' is never used - - [gnunet-service-fs.c:694]: member 'ConnectedPeer::last_p2p_replies_woff' is never used - - [gnunet-service-fs.c:700]: member 'ConnectedPeer::last_client_replies_woff' is never used - GAP improvements: + active reply route caching design & implementation of service; gap extension! * TBENCH: [MW] diff --git a/configure.ac b/configure.ac index 0fd4d43a1..f3e2135bd 100644 --- a/configure.ac +++ b/configure.ac @@ -663,7 +663,6 @@ src/hello/Makefile src/include/Makefile src/include/gnunet_directories.h src/hostlist/Makefile -src/migration/Makefile src/nat/Makefile src/nat/libnatpmp/Makefile src/nat/miniupnp/Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 9381020f4..9bd75d892 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,5 +23,5 @@ SUBDIRS = \ testing \ hostlist \ topology \ - $(NAT_DIR) - + $(NAT_DIR) \ + fs diff --git a/src/fs/fs.h b/src/fs/fs.h index 3277ea340..aff90d4fe 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -41,6 +41,13 @@ */ #define MAX_MIGRATION_QUEUE 32 +/** + * Ratio for moving average delay calculation. The previous + * average goes in with a factor of (n-1) into the calculation. + * Must be > 0. + */ +#define RUNAVG_DELAY_N 16 + /** * Size of the individual blocks used for file-sharing. */ diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index a08a041da..418ed459a 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -27,10 +27,9 @@ * - TTL/priority calculations are absent! * TODO: * - have non-zero preference / priority for requests we initiate! - * - track stats for hot-path routing * - implement hot-path routing decision procedure * - implement: bound_priority, test_load_too_high, validate_nblock - * - add content migration support (store locally) [or create new service] + * - add content migration support (forward from migration list) * - statistics */ #include "platform.h" @@ -482,6 +481,7 @@ struct PendingRequest struct GNUNET_DATASTORE_QueueEntry *qe; /** + * Size of the 'bf' (in bytes). */ size_t bf_size; @@ -2005,7 +2005,10 @@ struct ProcessReplyClosure */ const void *data; - // FIXME: add 'struct ConnectedPeer' to track 'last_xxx_replies' here! + /** + * Who gave us this reply? NULL for local host. + */ + struct ConnectedPeer *sender; /** * When the reply expires. @@ -2055,6 +2058,7 @@ process_reply (void *cls, struct ClientList *cl; struct PutMessage *pm; struct ConnectedPeer *cp; + struct GNUNET_TIME_Relative cur_delay; GNUNET_HashCode chash; GNUNET_HashCode mhash; size_t msize; @@ -2069,6 +2073,33 @@ process_reply (void *cls, gettext_noop ("# replies received and matched"), 1, GNUNET_NO); + if (prq->sender != NULL) + { + /* FIXME: should we be more precise here and not use + "start_time" but a peer-specific time stamp? */ + cur_delay = GNUNET_TIME_absolute_get_duration (pr->start_time); + prq->sender->avg_delay.value + = (prq->sender->avg_delay.value * + (RUNAVG_DELAY_N - 1) + cur_delay.value) / RUNAVG_DELAY_N; + prq->sender->avg_priority + = (prq->sender->avg_priority * + (RUNAVG_DELAY_N - 1) + pr->priority) / (double) RUNAVG_DELAY_N; + if (pr->cp != NULL) + { + GNUNET_PEER_change_rc (prq->sender->last_p2p_replies + [prq->sender->last_p2p_replies_woff % P2P_SUCCESS_LIST_SIZE], -1); + GNUNET_PEER_change_rc (pr->cp->pid, 1); + prq->sender->last_p2p_replies + [(prq->sender->last_p2p_replies_woff++) % P2P_SUCCESS_LIST_SIZE] + = pr->cp->pid; + } + else + { + prq->sender->last_client_replies + [(prq->sender->last_client_replies_woff++) % CS2P_SUCCESS_LIST_SIZE] + = pr->client_request_list->client_list->client; + } + } GNUNET_CRYPTO_hash (prq->data, prq->size, &chash); @@ -2317,6 +2348,9 @@ handle_p2p_put (void *cls, GNUNET_NO); /* now, lookup 'query' */ prq.data = (const void*) &put[1]; + if (other != NULL) + prq.sender = GNUNET_CONTAINER_multihashmap_get (connected_peers, + &other->hashPubKey); prq.size = dsize; prq.type = type; prq.expiration = expiration; diff --git a/src/migration/Makefile.am b/src/migration/Makefile.am deleted file mode 100644 index 2bc4a1d2d..000000000 --- a/src/migration/Makefile.am +++ /dev/null @@ -1,34 +0,0 @@ -INCLUDES = -I$(top_srcdir)/src/include - -if MINGW - WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols -endif - -if USE_COVERAGE - AM_CFLAGS = --coverage -O0 - XLIBS = -lgcov -endif - -bin_PROGRAMS = \ - gnunet-daemon-migration - -gnunet_daemon_migration_SOURCES = \ - gnunet-daemon-migration.c -gnunet_daemon_migration_LDADD = \ - $(top_builddir)/src/datastore/libgnunetdatastore.la \ - $(top_builddir)/src/statistics/libgnunetstatistics.la \ - $(top_builddir)/src/core/libgnunetcore.la \ - $(top_builddir)/src/util/libgnunetutil.la \ - $(GN_LIBINTL) - -check_PROGRAMS = test_gnunet_daemon_migration -#TESTS = $(check_PROGRAMS) - -test_gnunet_daemon_migration_SOURCES = \ - test_gnunet_daemon_migration.c -test_gnunet_daemon_migration_LDADD = \ - $(top_builddir)/src/testing/libgnunettesting.la \ - $(top_builddir)/src/util/libgnunetutil.la - -EXTRA_DIST = \ - test_gnunet_daemon_migration_data.conf diff --git a/src/migration/gnunet-daemon-migration.c b/src/migration/gnunet-daemon-migration.c deleted file mode 100644 index 0f0dbaebc..000000000 --- a/src/migration/gnunet-daemon-migration.c +++ /dev/null @@ -1,378 +0,0 @@ -/* - This file is part of GNUnet. - (C) 2010 Christian Grothoff (and other contributing authors) - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. -*/ - -/** - * @file migration/gnunet-daemon-migration.c - * @brief migrating (file-sharing) content through the network; this - * daemon is only responsible for pushing content out (not for - * processing inbound messages) - * @author Christian Grothoff - */ -#include -#include "platform.h" -#include "../fs/fs.h" -#include "gnunet_constants.h" -#include "gnunet_core_service.h" -#include "gnunet_datastore_service.h" -#include "gnunet_protocols.h" -#include "gnunet_statistics_service.h" -#include "gnunet_util_lib.h" - - -#define DEBUG_MIGRATION GNUNET_YES - -/** - * Information we keep per peer. - */ -struct Peer -{ - /** - * Last time we migrated data to this peer. - */ - struct GNUNET_TIME_Absolute last_migration; - -}; - - -/** - * Our scheduler. - */ -static struct GNUNET_SCHEDULER_Handle *sched; - -/** - * Our configuration. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Handle to the core API. - */ -static struct GNUNET_CORE_Handle *handle; - -/** - * Handle for reporting statistics. - */ -static struct GNUNET_STATISTICS_Handle *stats; - -/** - * Handle for the core service. -*/ -static struct GNUNET_CORE_Handle *handle; - -/** - * Handle to the datastore. - */ -static struct GNUNET_DATASTORE_Handle *datastore; - -/** - * Anonymity level for the current block. - */ -static unsigned int current_anonymity; - -/** - * Type of the current block. - */ -static enum GNUNET_BLOCK_Type current_type; - -/** - * Data of the current block (already encrypted). - */ -static char current_block[GNUNET_SERVER_MAX_MESSAGE_SIZE]; - -/** - * Size of the current block. - */ -static size_t current_block_size; - -/** - * Key of the current block. - */ -static GNUNET_HashCode current_key; - -/** - * Task scheduled to receive content from the datastore (with some delay). - */ -static GNUNET_SCHEDULER_TaskIdentifier get_task; - - -/** - * Select a peer for transmitting the current block to. - */ -static void -select_peer () -{ - /* FIXME: select a peer for transmission... */ -} - - -/** - * Method called whenever a peer connects. - * - * @param cls closure - * @param peer peer identity this notification is about - * @param latency reported latency of the connection with 'other' - * @param distance reported distance (DV) to 'other' - */ -static void -connect_notify (void *cls, - const struct - GNUNET_PeerIdentity * peer, - struct GNUNET_TIME_Relative latency, - uint32_t distance) -{ - /* FIXME: track peer */ -} - - -/** - * Method called whenever a peer disconnects. - * - * @param cls closure - * @param peer peer identity this notification is about - */ -static void -disconnect_notify (void *cls, - const struct - GNUNET_PeerIdentity * peer) -{ - /* FIXME: untrack peer */ -} - - -/** - * Ask datastore for more content. - * @param cls closure - * @param tc scheduler context - */ -static void -get_content (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); - - -/** - * An iterator over a set of items stored in the datastore. - * - * @param cls closure - * @param key key for the content - * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum; - * maybe 0 if no unique identifier is available - */ -static void -content_processor (void *cls, - const GNUNET_HashCode * key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, uint64_t uid) -{ - if (key != NULL) - { - memcpy (current_block, data, size); - current_block_size = size; - current_type = type; - current_anonymity = anonymity; - current_key = *key; - return; - } - if (current_block_size == 0) - { - get_task = GNUNET_SCHEDULER_add_delayed (sched, - GNUNET_TIME_UNIT_MINUTES, - &get_content, - NULL); - return; - } - if (current_type == GNUNET_BLOCK_TYPE_ONDEMAND) - { - /* FIXME: do on-demand encoding... */ - return; - } - select_peer (); -} - - -/** - * Ask datastore for more content. - * @param cls closure - * @param tc scheduler context - */ -static void -get_content (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - get_task = GNUNET_SCHEDULER_NO_TASK; - GNUNET_DATASTORE_get_random (datastore, - 0, 1, - GNUNET_CONSTANTS_SERVICE_TIMEOUT, - &content_processor, - NULL); -} - - -/** - * Function called after GNUNET_CORE_connect has succeeded - * (or failed for good). - * - * @param cls closure - * @param server handle to the server, NULL if we failed - * @param my_id ID of this peer, NULL if we failed - * @param publicKey public key of this peer, NULL if we failed - */ -static void -core_init (void *cls, - struct GNUNET_CORE_Handle * server, - const struct GNUNET_PeerIdentity * - my_id, - const struct - GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded * - publicKey) -{ - handle = server; - if (datastore != NULL) - get_task = GNUNET_SCHEDULER_add_now (sched, - &get_content, - NULL); -} - - -/** - * Last task run during shutdown. Disconnects us from - * the core. - * - * @param cls unused, NULL - * @param tc scheduler context - */ -static void -cleaning_task (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - if (get_task != GNUNET_SCHEDULER_NO_TASK) - { - GNUNET_SCHEDULER_cancel (sched, - get_task); - get_task = GNUNET_SCHEDULER_NO_TASK; - } - if (handle != NULL) - { - GNUNET_CORE_disconnect (handle); - handle = NULL; - } - if (datastore != NULL) - { - GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO); - datastore = NULL; - } - if (stats != NULL) - { - GNUNET_STATISTICS_destroy (stats, GNUNET_NO); - stats = NULL; - } -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param s the scheduler to use - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - struct GNUNET_SCHEDULER_Handle * s, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle * c) -{ - struct GNUNET_CORE_MessageHandler handlers[] = - { - { NULL, 0, 0 } - }; - sched = s; - cfg = c; - stats = GNUNET_STATISTICS_create (sched, "topology", cfg); - handle = GNUNET_CORE_connect (sched, - cfg, - GNUNET_TIME_UNIT_FOREVER_REL, - NULL, - &core_init, - &connect_notify, - &disconnect_notify, - NULL, GNUNET_NO, - NULL, GNUNET_NO, - handlers); - datastore = GNUNET_DATASTORE_connect (cfg, sched); - GNUNET_SCHEDULER_add_delayed (sched, - GNUNET_TIME_UNIT_FOREVER_REL, - &cleaning_task, NULL); - if ( (NULL == handle) || - (NULL == datastore) ) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("Failed to connect to `%s' service.\n"), - (NULL == handle) ? "core" : "datastore"); - GNUNET_SCHEDULER_shutdown (sched); - return; - } -} - - -/** - * gnunet-daemon-topology command line options. - */ -static struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END -}; - - -/** - * The main function for the topology daemon. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, char *const *argv) -{ - int ret; - - ret = (GNUNET_OK == - GNUNET_PROGRAM_run (argc, - argv, - "migration", - _("Content migration for anonymous file-sharing"), - options, - &run, NULL)) ? 0 : 1; - return ret; -} - -/* end of gnunet-daemon-migration.c */ diff --git a/src/migration/test_gnunet_daemon_migration.c b/src/migration/test_gnunet_daemon_migration.c deleted file mode 100644 index 33cf5bb04..000000000 --- a/src/migration/test_gnunet_daemon_migration.c +++ /dev/null @@ -1,184 +0,0 @@ -/* - This file is part of GNUnet. - (C) 2010 Christian Grothoff (and other contributing authors) - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. -*/ -/** - * @file topology/test_gnunet_daemon_migration.c - * @brief testcase for content migration code - */ -#include "platform.h" -#include "gnunet_testing_lib.h" - -#define VERBOSE GNUNET_YES - -#define NUM_PEERS 2 - -/** - * How long until we give up on connecting the peers? - */ -#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - -#define CONNECT_ATTEMPTS 3 - - -static int ok; - -static int peers_left; - -static int connect_left; - -static struct GNUNET_TESTING_PeerGroup *pg; - -static struct GNUNET_TESTING_Daemon *first; - -static struct GNUNET_TESTING_Daemon *last; - -static struct GNUNET_SCHEDULER_Handle *sched; - - -static void -clean_up_task (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - GNUNET_TESTING_daemons_stop (pg, TIMEOUT); - ok = 0; -} - - -static void -notify_connect_complete(void *cls, - const struct GNUNET_PeerIdentity *first, - const struct GNUNET_PeerIdentity *second, - const struct GNUNET_CONFIGURATION_Handle *first_cfg, - const struct GNUNET_CONFIGURATION_Handle *second_cfg, - struct GNUNET_TESTING_Daemon *first_daemon, - struct GNUNET_TESTING_Daemon *second_daemon, - const char *emsg) -{ - if (NULL != emsg) - { - fprintf (stderr, - "Failed to connect two peers: %s\n", - emsg); - GNUNET_TESTING_daemons_stop (pg, TIMEOUT); - GNUNET_assert (0); - return; - } - connect_left--; - if (connect_left == 0) - { - /* FIXME: check that topology adds a few more links - in addition to those that were seeded */ - /* For now, sleep so we can have the daemon do some work */ - GNUNET_SCHEDULER_add_delayed (sched, - GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5), - &clean_up_task, - NULL); - } -} - - -static void my_cb(void *cls, - const struct GNUNET_PeerIdentity *id, - const struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_TESTING_Daemon *d, - const char *emsg) -{ - GNUNET_assert (id != NULL); - peers_left--; - if (first == NULL) - { - connect_left = NUM_PEERS; - first = d; - last = d; - return; - } - GNUNET_TESTING_daemons_connect (last, d, TIMEOUT, CONNECT_ATTEMPTS, - ¬ify_connect_complete, - NULL); - if (peers_left == 0) - { - /* close circle */ - GNUNET_TESTING_daemons_connect (d, first, TIMEOUT, CONNECT_ATTEMPTS, - ¬ify_connect_complete, - NULL); - } -} - - -static void -run (void *cls, - struct GNUNET_SCHEDULER_Handle *s, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) -{ - sched = s; - ok = 1; -#if VERBOSE - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting daemons.\n"); -#endif - peers_left = NUM_PEERS; - pg = GNUNET_TESTING_daemons_start (sched, cfg, - peers_left, - TIMEOUT, - NULL, NULL, - &my_cb, NULL, NULL, NULL, NULL); - GNUNET_assert (pg != NULL); -} - -static int -check () -{ - char *const argv[] = { "test-testing", - "-c", - "test_gnunet_service_topology_data.conf", -#if VERBOSE - "-L", "DEBUG", -#endif - NULL - }; - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END - }; - GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, - argv, "test-gnunet-service-topology", "nohelp", - options, &run, &ok); - return ok; -} - -int -main (int argc, char *argv[]) -{ - int ret; - - GNUNET_log_setup ("test-gnunet-daemon-migration", -#if VERBOSE - "DEBUG", -#else - "WARNING", -#endif - NULL); - ret = check (); - sleep (1); /* FIXME: needed? */ - GNUNET_DISK_directory_remove ("/tmp/test-gnunet-topology"); - return ret; -} - -/* end of test_gnunet_daemon_migration.c */ diff --git a/src/migration/test_gnunet_daemon_migration_data.conf b/src/migration/test_gnunet_daemon_migration_data.conf deleted file mode 100644 index d92f9eabf..000000000 --- a/src/migration/test_gnunet_daemon_migration_data.conf +++ /dev/null @@ -1,37 +0,0 @@ -[PATHS] -SERVICEHOME = /tmp/test-gnunet-topology/ -# DEFAULTCONFIG = test_gnunet_service_topology_data.conf - -[resolver] -PORT = 2664 - -[transport] -PORT = 2665 -PLUGINS = tcp -# DEBUG = YES -#PREFIX = xterm -e xterm -T transport -e gdb -x cmd --args -#PREFIX = valgrind --tool=memcheck --log-file=logs%p - -[arm] -PORT = 2666 -DEFAULTSERVICES = topology - -[statistics] -PORT = 2667 - -[transport-tcp] -PORT = 2668 - -[peerinfo] -PORT = 2669 - -[core] -PORT = 2670 -# DEBUG = YES - -[topology] -# DEBUG = YES -#PREFIX = valgrind --tool=memcheck - -[testing] -WEAKRANDOM = YES diff --git a/src/util/scheduler.c b/src/util/scheduler.c index c46c89500..3a646b8b7 100644 --- a/src/util/scheduler.c +++ b/src/util/scheduler.c @@ -613,7 +613,8 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_Task task, void *task_cls) if (errno == EINTR) continue; GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "select"); - break; + abort (); + break; } if (GNUNET_NETWORK_fdset_handle_isset (rs, pr)) { -- cgit v1.2.3