From 08f1114108442793fb00dc443ed6b94b81c3b443 Mon Sep 17 00:00:00 2001 From: "Nathan S. Evans" Date: Mon, 25 Jul 2011 14:27:33 +0000 Subject: churn a service --- src/dht/gnunet-dht-driver.c | 2 +- src/include/gnunet_testing_lib.h | 55 +++++++ src/nse/nse-profiler.c | 2 +- src/testing/test_testing_topology_churn.c | 8 +- src/testing/testing.c | 262 ++++++++++++++++++++++++++++++ src/testing/testing_group.c | 116 ++++++++++--- 6 files changed, 412 insertions(+), 33 deletions(-) diff --git a/src/dht/gnunet-dht-driver.c b/src/dht/gnunet-dht-driver.c index 882a77b65..7c602cfca 100644 --- a/src/dht/gnunet-dht-driver.c +++ b/src/dht/gnunet-dht-driver.c @@ -1901,7 +1901,7 @@ churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) "churn_peers: want %u total, %u running, starting %u, stopping %u\n", churn_array[current_churn_round], count_running, churn_up, churn_down); - GNUNET_TESTING_daemons_churn (pg, churn_down, churn_up, timeout, + GNUNET_TESTING_daemons_churn (pg, NULL, churn_down, churn_up, timeout, &churn_complete, find_peer_context); current_churn_round++; } diff --git a/src/include/gnunet_testing_lib.h b/src/include/gnunet_testing_lib.h index 60b0256a0..34920bbab 100644 --- a/src/include/gnunet_testing_lib.h +++ b/src/include/gnunet_testing_lib.h @@ -175,6 +175,18 @@ enum GNUNET_TESTING_StartPhase */ SP_SHUTDOWN_START, + /** + * We should shutdown a *single* service via gnunet-arm. Call the dead_cb + * upon notification from gnunet-arm that the service has been stopped. + */ + SP_SERVICE_SHUTDOWN_START, + + /** + * We should start a *single* service via gnunet-arm. Call the daemon cb + * upon notification from gnunet-arm that the service has been started. + */ + SP_SERVICE_START, + /** * We've received a configuration update and are currently waiting for * the copy process for the update to complete. Once it is, we will @@ -362,6 +374,14 @@ struct GNUNET_TESTING_Daemon * (if it's going to be restarted later) */ int churn; + + /** + * Currently, a single char * pointing to a service + * that has been churned off. + * + * FIXME: make this a linked list of services that have been churned off!!! + */ + char *churned_services; }; @@ -493,6 +513,21 @@ GNUNET_TESTING_daemon_start_stopped (struct GNUNET_TESTING_Daemon *daemon, GNUNET_TESTING_NotifyDaemonRunning cb, void *cb_cls); +/** + * Stops a GNUnet daemon. + * + * @param d the daemon for which the service should be started + * @param service the name of the service to start + * @param timeout how long to wait for process for shutdown to complete + * @param cb function called once the daemon was stopped + * @param cb_cls closure for cb + */ +void +GNUNET_TESTING_daemon_start_stopped_service (struct GNUNET_TESTING_Daemon *d, + char *service, + struct GNUNET_TIME_Relative timeout, + GNUNET_TESTING_NotifyDaemonRunning cb, void *cb_cls); + /** * Get a certain testing daemon handle. * @@ -549,6 +584,24 @@ void GNUNET_TESTING_daemon_reconfigure (struct GNUNET_TESTING_Daemon *d, GNUNET_TESTING_NotifyCompletion cb, void * cb_cls); +/** + * Stops a single service of a GNUnet daemon. Used like daemon_stop, + * only doesn't stop the entire peer in any case. If the service + * is not currently running, this call is likely to fail after + * timeout! + * + * @param d the daemon that should be stopped + * @param service the name of the service to stop + * @param timeout how long to wait for process for shutdown to complete + * @param cb function called once the service was stopped + * @param cb_cls closure for cb + */ +void +GNUNET_TESTING_daemon_stop_service (struct GNUNET_TESTING_Daemon *d, + char *service, + struct GNUNET_TIME_Relative timeout, + GNUNET_TESTING_NotifyCompletion cb, void *cb_cls); + /** * Establish a connection between two GNUnet daemons. @@ -677,6 +730,7 @@ GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg); * running even though the "peer" is being varied offline. * * @param pg handle for the peer group + * @param service the service to churn on/off, NULL for all * @param voff number of peers that should go offline * @param von number of peers that should come back online; * must be zero on first call (since "testbed_start" @@ -688,6 +742,7 @@ GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg); */ void GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg, + char *service, unsigned int voff, unsigned int von, struct GNUNET_TIME_Relative timeout, diff --git a/src/nse/nse-profiler.c b/src/nse/nse-profiler.c index 498f058d2..538cd1f50 100644 --- a/src/nse/nse-profiler.c +++ b/src/nse/nse-profiler.c @@ -496,7 +496,7 @@ churn_peers (void *cls, - peers_next_round : 0, (peers_next_round > peers_running) ? peers_next_round - peers_running : 0); - GNUNET_TESTING_daemons_churn (pg, + GNUNET_TESTING_daemons_churn (pg, "nse", (peers_running > peers_next_round) ? peers_running - peers_next_round : 0, diff --git a/src/testing/test_testing_topology_churn.c b/src/testing/test_testing_topology_churn.c index f1d93fa46..b92288199 100644 --- a/src/testing/test_testing_topology_churn.c +++ b/src/testing/test_testing_topology_churn.c @@ -173,28 +173,28 @@ static void churn_peers_both () { churn_ctx.next_task = &finish_testing; - GNUNET_TESTING_daemons_churn (pg, 1, 1, TIMEOUT, &churn_callback, NULL); + GNUNET_TESTING_daemons_churn (pg, NULL, 1, 1, TIMEOUT, &churn_callback, NULL); } static void churn_peers_off_again () { churn_ctx.next_task = &churn_peers_both; - GNUNET_TESTING_daemons_churn (pg, 2, 0, TIMEOUT, &churn_callback, NULL); + GNUNET_TESTING_daemons_churn (pg, NULL, 2, 0, TIMEOUT, &churn_callback, NULL); } static void churn_peers_on () { churn_ctx.next_task = &churn_peers_off_again; - GNUNET_TESTING_daemons_churn (pg, 0, 2, TIMEOUT, &churn_callback, NULL); + GNUNET_TESTING_daemons_churn (pg, NULL, 0, 2, TIMEOUT, &churn_callback, NULL); } static void churn_peers_off () { churn_ctx.next_task = &churn_peers_on; - GNUNET_TESTING_daemons_churn (pg, 2, 0, TIMEOUT, &churn_callback, NULL); + GNUNET_TESTING_daemons_churn (pg, NULL, 2, 0, TIMEOUT, &churn_callback, NULL); } static void diff --git a/src/testing/testing.c b/src/testing/testing.c index b9f0fb538..c35500c35 100644 --- a/src/testing/testing.c +++ b/src/testing/testing.c @@ -784,6 +784,87 @@ start_fsm (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) case SP_START_DONE: GNUNET_break (0); break; + case SP_SERVICE_START: + /* confirm gnunet-arm exited */ + if (GNUNET_OK != GNUNET_OS_process_status (d->proc, &type, &code)) + { + if (GNUNET_TIME_absolute_get_remaining (d->max_timeout).rel_value == + 0) + { + cb = d->cb; + d->cb = NULL; + if (NULL != cb) + cb (d->cb_cls, + NULL, + d->cfg, + d, + (NULL == d->hostname) + ? _("`gnunet-arm' does not seem to terminate.\n") + : _("`ssh' does not seem to terminate.\n")); + return; + } + /* wait some more */ + d->task + = GNUNET_SCHEDULER_add_delayed (GNUNET_CONSTANTS_EXEC_WAIT, + &start_fsm, d); + return; + } + if ((type != GNUNET_OS_PROCESS_EXITED) || (code != 0)) + { + cb = d->cb; + d->cb = NULL; + if (NULL != cb) + cb (d->cb_cls, + NULL, + d->cfg, + d, + (NULL == d->hostname) + ? _("`gnunet-arm' does not seem to terminate.\n") + : _("`ssh' does not seem to terminate.\n")); + return; + } +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Service startup complete!\n"); +#endif + cb = d->cb; + d->cb = NULL; + d->phase = SP_START_DONE; + if (NULL != cb) + cb (d->cb_cls, &d->id, d->cfg, d, NULL); + break; + case SP_SERVICE_SHUTDOWN_START: + /* confirm copying complete */ + if (GNUNET_OK != GNUNET_OS_process_status (d->proc, &type, &code)) + { + if (GNUNET_TIME_absolute_get_remaining (d->max_timeout).rel_value == + 0) + { + if (NULL != d->dead_cb) + d->dead_cb (d->dead_cb_cls, + _ + ("either `gnunet-arm' or `ssh' does not seem to terminate.\n")); + return; + } + /* wait some more */ + d->task + = GNUNET_SCHEDULER_add_delayed (GNUNET_CONSTANTS_EXEC_WAIT, + &start_fsm, d); + return; + } + if ((type != GNUNET_OS_PROCESS_EXITED) || (code != 0)) + { + if (NULL != d->dead_cb) + d->dead_cb (d->dead_cb_cls, + _ + ("shutdown (either `gnunet-arm' or `ssh') did not complete cleanly.\n")); + return; + } +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Service shutdown complete.\n"); +#endif + if (NULL != d->dead_cb) + d->dead_cb (d->dead_cb_cls, NULL); + break; case SP_SHUTDOWN_START: /* confirm copying complete */ if (GNUNET_OK != GNUNET_OS_process_status (d->proc, &type, &code)) @@ -957,6 +1038,94 @@ GNUNET_TESTING_daemon_running (struct GNUNET_TESTING_Daemon *daemon) } +/** + * Stops a GNUnet daemon. + * + * @param d the daemon for which the service should be started + * @param service the name of the service to start + * @param timeout how long to wait for process for shutdown to complete + * @param cb function called once the daemon was stopped + * @param cb_cls closure for cb + */ +void +GNUNET_TESTING_daemon_start_stopped_service (struct GNUNET_TESTING_Daemon *d, + char *service, + struct GNUNET_TIME_Relative timeout, + GNUNET_TESTING_NotifyDaemonRunning cb, void *cb_cls) +{ + char *arg; + d->cb = cb; + d->cb_cls = cb_cls; + + GNUNET_assert(d->running == GNUNET_YES); + + if (d->phase == SP_CONFIG_UPDATE) + { + GNUNET_SCHEDULER_cancel (d->task); + d->phase = SP_START_DONE; + } + +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + _("Terminating peer `%4s'\n"), GNUNET_i2s (&d->id)); +#endif + if (d->churned_services == NULL) + { + d->dead_cb(d->dead_cb_cls, "No service has been churned off yet!!"); + return; + } + d->phase = SP_SERVICE_START; + GNUNET_free(d->churned_services); + + /* Check if this is a local or remote process */ + if (NULL != d->hostname) + { +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting gnunet-arm with config `%s' on host `%s'.\n", + d->cfgfile, d->hostname); +#endif + + if (d->username != NULL) + GNUNET_asprintf (&arg, "%s@%s", d->username, d->hostname); + else + arg = GNUNET_strdup (d->hostname); + + d->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", +#if !DEBUG_TESTING + "-q", +#endif + arg, "gnunet-arm", +#if DEBUG_TESTING + "-L", "DEBUG", +#endif + "-c", d->cfgfile, "-i", service, "-q", + NULL); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting gnunet-arm with command ssh %s gnunet-arm -c %s -i %s -q\n", + arg, "gnunet-arm", d->cfgfile, service); + GNUNET_free (arg); + } + else + { +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting gnunet-arm with config `%s' locally.\n", + d->cfgfile); +#endif + d->proc = GNUNET_OS_start_process (NULL, NULL, "gnunet-arm", + "gnunet-arm", +#if DEBUG_TESTING + "-L", "DEBUG", +#endif + "-c", d->cfgfile, "-i", service, "-q", + NULL); + } + + d->max_timeout = GNUNET_TIME_relative_to_absolute (timeout); + d->task = GNUNET_SCHEDULER_add_now (&start_fsm, d); +} + /** * Start a peer that has previously been stopped using the daemon_stop * call (and files weren't deleted and the allow restart flag) @@ -1329,6 +1498,99 @@ GNUNET_TESTING_daemon_restart (struct GNUNET_TESTING_Daemon *d, } +/** + * Stops a GNUnet daemon. + * + * @param d the daemon that should be stopped + * @param service the name of the service to stop + * @param timeout how long to wait for process for shutdown to complete + * @param cb function called once the daemon was stopped + * @param cb_cls closure for cb + * @param delete_files GNUNET_YES to remove files, GNUNET_NO + * to leave them + * @param allow_restart GNUNET_YES to restart peer later (using this API) + * GNUNET_NO to kill off and clean up for good + */ +void +GNUNET_TESTING_daemon_stop_service (struct GNUNET_TESTING_Daemon *d, + char *service, + struct GNUNET_TIME_Relative timeout, + GNUNET_TESTING_NotifyCompletion cb, void *cb_cls) +{ + char *arg; + d->dead_cb = cb; + d->dead_cb_cls = cb_cls; + + GNUNET_assert(d->running == GNUNET_YES); + + if (d->phase == SP_CONFIG_UPDATE) + { + GNUNET_SCHEDULER_cancel (d->task); + d->phase = SP_START_DONE; + } + +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + _("Terminating peer `%4s'\n"), GNUNET_i2s (&d->id)); +#endif + if (d->churned_services != NULL) + { + d->dead_cb(d->dead_cb_cls, "A service has already been turned off!!"); + return; + } + d->phase = SP_SERVICE_SHUTDOWN_START; + d->churned_services = GNUNET_strdup(service); + + /* Check if this is a local or remote process */ + if (NULL != d->hostname) + { +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Stopping gnunet-arm with config `%s' on host `%s'.\n", + d->cfgfile, d->hostname); +#endif + + if (d->username != NULL) + GNUNET_asprintf (&arg, "%s@%s", d->username, d->hostname); + else + arg = GNUNET_strdup (d->hostname); + + d->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", +#if !DEBUG_TESTING + "-q", +#endif + arg, "gnunet-arm", +#if DEBUG_TESTING + "-L", "DEBUG", +#endif + "-c", d->cfgfile, "-k", service, "-q", + NULL); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Stopping gnunet-arm with command ssh %s gnunet-arm -c %s -k %s -q\n", + arg, "gnunet-arm", d->cfgfile, service); + GNUNET_free (arg); + } + else + { +#if DEBUG_TESTING + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Stopping gnunet-arm with config `%s' locally.\n", + d->cfgfile); +#endif + d->proc = GNUNET_OS_start_process (NULL, NULL, "gnunet-arm", + "gnunet-arm", +#if DEBUG_TESTING + "-L", "DEBUG", +#endif + "-c", d->cfgfile, "-k", service, "-q", + NULL); + } + + d->max_timeout = GNUNET_TIME_relative_to_absolute (timeout); + d->task = GNUNET_SCHEDULER_add_now (&start_fsm, d); +} + + /** * Stops a GNUnet daemon. * diff --git a/src/testing/testing_group.c b/src/testing/testing_group.c index e7be22a8a..b9ccf69f2 100644 --- a/src/testing/testing_group.c +++ b/src/testing/testing_group.c @@ -102,6 +102,12 @@ struct ChurnContext */ struct GNUNET_TESTING_PeerGroup *pg; + /** + * Name of the service to churn on/off, NULL + * to churn entire peer. + */ + char *service; + /** * Callback used to notify of churning finished */ @@ -5576,9 +5582,15 @@ schedule_churn_restart(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) &schedule_churn_restart, peer_restart_ctx); else { - GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon, - startup_ctx->timeout, - &churn_start_callback, startup_ctx); + if (startup_ctx->churn_ctx->service != NULL) + GNUNET_TESTING_daemon_start_stopped_service (peer_restart_ctx->daemon, + startup_ctx->churn_ctx->service, + startup_ctx->timeout, + &churn_start_callback, startup_ctx); + else + GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon, + startup_ctx->timeout, + &churn_start_callback, startup_ctx); GNUNET_free (peer_restart_ctx); } } @@ -6336,9 +6348,15 @@ schedule_churn_shutdown_task(void *cls, else { shutdown_ctx->outstanding++; - GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, - shutdown_ctx->timeout, shutdown_ctx->cb, - shutdown_ctx, GNUNET_NO, GNUNET_YES); + if (churn_ctx->service != NULL) + GNUNET_TESTING_daemon_stop_service (peer_shutdown_ctx->daemon, + churn_ctx->service, + shutdown_ctx->timeout, shutdown_ctx->cb, + shutdown_ctx); + else + GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, + shutdown_ctx->timeout, shutdown_ctx->cb, + shutdown_ctx, GNUNET_NO, GNUNET_YES); GNUNET_free (peer_shutdown_ctx); } } @@ -6355,6 +6373,7 @@ schedule_churn_shutdown_task(void *cls, * completion. * * @param pg handle for the peer group + * @param service the service to churn off/on, NULL to churn peer * @param voff number of peers that should go offline * @param von number of peers that should come back online; * must be zero on first call (since "testbed_start" @@ -6366,6 +6385,7 @@ schedule_churn_shutdown_task(void *cls, */ void GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, + char *service, unsigned int voff, unsigned int von, struct GNUNET_TIME_Relative timeout, GNUNET_TESTING_NotifyCompletion cb, void *cb_cls) @@ -6385,6 +6405,7 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, unsigned int *stopped_arr; unsigned int *running_permute; unsigned int *stopped_permute; + char *pos; shutdown_ctx = NULL; peer_shutdown_ctx = NULL; @@ -6402,15 +6423,39 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, for (i = 0; i < pg->total; i++) { - if (pg->peers[i].daemon->running == GNUNET_YES) + if (service == NULL) { - GNUNET_assert (running != -1); - running++; + if (pg->peers[i].daemon->running == GNUNET_YES) + { + GNUNET_assert (running != -1); + running++; + } + else + { + GNUNET_assert (stopped != -1); + stopped++; + } } else { - GNUNET_assert (stopped != -1); - stopped++; + /* FIXME: make churned services a list! */ + pos = pg->peers[i].daemon->churned_services; + /* FIXME: while (pos != NULL) */ + if (pos != NULL) + { + if (0 == strcasecmp(pos, service)) + { + GNUNET_assert (stopped != -1); + stopped++; + break; + } + /* FIXME: pos = pos->next; */ + } + if (pos == NULL) + { + GNUNET_assert (running != -1); + running++; + } } } @@ -6463,17 +6508,43 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, for (i = 0; i < pg->total; i++) { - if (pg->peers[i].daemon->running == GNUNET_YES) + if (service == NULL) { - GNUNET_assert ((running_arr != NULL) && (total_running > running)); - running_arr[running] = i; - running++; + if (pg->peers[i].daemon->running == GNUNET_YES) + { + GNUNET_assert ((running_arr != NULL) && (total_running > running)); + running_arr[running] = i; + running++; + } + else + { + GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped)); + stopped_arr[stopped] = i; + stopped++; + } } else { - GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped)); - stopped_arr[stopped] = i; - stopped++; + /* FIXME: make churned services a list! */ + pos = pg->peers[i].daemon->churned_services; + /* FIXME: while (pos != NULL) */ + if (pos != NULL) + { + if (0 == strcasecmp(pos, service)) + { + GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped)); + stopped_arr[stopped] = i; + stopped++; + break; + } + /* FIXME: pos = pos->next; */ + } + if (pos == NULL) + { + GNUNET_assert ((running_arr != NULL) && (total_running > running)); + running_arr[running] = i; + running++; + } } } @@ -6500,12 +6571,6 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, peer_shutdown_ctx->shutdown_ctx = shutdown_ctx; GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task, peer_shutdown_ctx); - - /* - GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon, - timeout, - &churn_stop_callback, churn_ctx, - GNUNET_NO, GNUNET_YES); */ } GNUNET_assert (stopped >= von); @@ -6528,9 +6593,6 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg, peer_restart_ctx->daemon = pg->peers[stopped_arr[stopped_permute[i]]].daemon; GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx); - /* - GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon, - timeout, &churn_start_callback, churn_ctx); */ } GNUNET_free_non_null (running_arr); -- cgit v1.2.3