introducing soft shutdown concept for services; during soft shutdown, services that are still managing non-monitor clients continue to run until those clients disconnect; however, the services do stop to accept new connections (will stop listening); soft shutdown is now used by ats, transport, peerinfo, namestore and most importantly statistics; this should fix #2197

13 files changed, 218 insertions, 127 deletions

diff --git a/src/arm/gnunet-service-arm.c b/src/arm/gnunet-service-arm.c
index 459171ffb..ab9e7f281 100644
--- a/src/arm/gnunet-service-arm.c
+++ b/src/arm/gnunet-service-arm.c
@@ -1301,7 +1301,8 @@ main (int argc, char *const *argv)
     GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
   ret =
     (GNUNET_OK ==
-     GNUNET_SERVICE_run (argc, argv, "arm", GNUNET_YES, &run, NULL)) ? 0 : 1;
+     GNUNET_SERVICE_run (argc, argv, "arm", 
+                         GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN, &run, NULL)) ? 0 : 1;
   GNUNET_SIGNAL_handler_uninstall (shc_chld);
   shc_chld = NULL;
   GNUNET_DISK_pipe_close (sigpipe);
diff --git a/src/ats/gnunet-service-ats.c b/src/ats/gnunet-service-ats.c
index 7deca0b62..a68ec416b 100644
--- a/src/ats/gnunet-service-ats.c
+++ b/src/ats/gnunet-service-ats.c
@@ -175,7 +175,7 @@ int
 main (int argc, char *const *argv)
 {
   return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "ats", GNUNET_SERVICE_OPTION_NONE,
+          GNUNET_SERVICE_run (argc, argv, "ats", GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN,
                               &run, NULL)) ? 0 : 1;
 }
 
diff --git a/src/include/gnunet_server_lib.h b/src/include/gnunet_server_lib.h
index 87c3a8f6c..c0a0a5cba 100644
--- a/src/include/gnunet_server_lib.h
+++ b/src/include/gnunet_server_lib.h
@@ -155,6 +155,16 @@ GNUNET_SERVER_create (GNUNET_CONNECTION_AccessCheck access, void *access_cls,
 
 
 /**
+ * Stop the listen socket and get ready to shutdown the server
+ * once only 'monitor' clients are left.
+ *
+ * @param server server to stop listening on
+ */
+void
+GNUNET_SERVER_stop_listening (struct GNUNET_SERVER_Handle *server);
+
+
+/**
  * Free resources held by this server.
  *
  * @param server server to destroy
@@ -215,6 +225,22 @@ GNUNET_SERVER_notify_transmit_ready_cancel (struct GNUNET_SERVER_TransmitHandle
 
 
 /**
+ * Set the 'monitor' flag on this client.  Clients which have been
+ * marked as 'monitors' won't prevent the server from shutting down
+ * once 'GNUNET_SERVER_stop_listening' has been invoked.  The idea is
+ * that for "normal" clients we likely want to allow them to process
+ * their requests; however, monitor-clients are likely to 'never'
+ * disconnect during shutdown and thus will not be considered when
+ * determining if the server should continue to exist after
+ * 'GNUNET_SERVER_destroy' has been called.
+ *
+ * @param client the client to set the 'monitor' flag on
+ */
+void
+GNUNET_SERVER_client_mark_monitor (struct GNUNET_SERVER_Client *client);
+
+
+/**
  * Set the persistent flag on this client, used to setup client connection
  * to only be killed when the service it's connected to is actually dead.
  *
diff --git a/src/include/gnunet_service_lib.h b/src/include/gnunet_service_lib.h
index 1641e0f93..ec0c15e1b 100644
--- a/src/include/gnunet_service_lib.h
+++ b/src/include/gnunet_service_lib.h
@@ -85,16 +85,22 @@ typedef void (*GNUNET_SERVICE_Main) (void *cls,
  */
 enum GNUNET_SERVICE_Options
 {
-    /**
-     * Use defaults.
-     */
+  /**
+   * Use defaults.
+   */
   GNUNET_SERVICE_OPTION_NONE = 0,
 
-    /**
-     * Do not trigger server shutdown on signals, allow for the user
-     * to terminate the server explicitly when needed.
-     */
-  GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN = 1
+  /**
+   * Do not trigger server shutdown on signals, allow for the user
+   * to terminate the server explicitly when needed.
+   */
+  GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN = 1,
+
+  /**
+   * Trigger a SOFT server shutdown on signals, allowing active
+   * non-monitor clients to complete their transactions.
+   */
+  GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN = 2
 };
 
 
@@ -104,16 +110,16 @@ enum GNUNET_SERVICE_Options
  *
  * @param argc number of command line arguments
  * @param argv command line arguments
- * @param serviceName our service name
- * @param opt service options
+ * @param service_name our service name
+ * @param options service options
  * @param task main task of the service
  * @param task_cls closure for task
  * @return GNUNET_SYSERR on error, GNUNET_OK
  *         if we shutdown nicely
  */
 int
-GNUNET_SERVICE_run (int argc, char *const *argv, const char *serviceName,
-                    enum GNUNET_SERVICE_Options opt, GNUNET_SERVICE_Main task,
+GNUNET_SERVICE_run (int argc, char *const *argv, const char *service_name,
+                    enum GNUNET_SERVICE_Options options, GNUNET_SERVICE_Main task,
                     void *task_cls);
 
 
@@ -123,13 +129,15 @@ struct GNUNET_SERVICE_Context;
  * Run a service startup sequence within an existing
  * initialized system.
  *
- * @param serviceName our service name
+ * @param service_name our service name
  * @param cfg configuration to use
+ * @param options service options
  * @return NULL on error, service handle
  */
 struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *serviceName,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg);
+GNUNET_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      enum GNUNET_SERVICE_Options options);
 
 
 /**
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 7facd5e39..aeeb9e589 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -1975,7 +1975,7 @@ main (int argc, char *const *argv)
 {
   return (GNUNET_OK ==
           GNUNET_SERVICE_run (argc, argv, "namestore",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+                              GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN, &run, NULL)) ? 0 : 1;
 }
 
 /* end of gnunet-service-namestore.c */
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c
index 2dbdc9f83..4155b1ebe 100644
--- a/src/peerinfo/gnunet-service-peerinfo.c
+++ b/src/peerinfo/gnunet-service-peerinfo.c
@@ -620,6 +620,7 @@ handle_notify (void *cls, struct GNUNET_SERVER_Client *client,
                const struct GNUNET_MessageHeader *message)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "`%s' message received\n", "NOTIFY");
+  GNUNET_SERVER_client_mark_monitor (client);
   GNUNET_SERVER_notification_context_add (notify_list, client);
   GNUNET_CONTAINER_multihashmap_iterate (hostmap, &do_notify_entry, client);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
@@ -728,7 +729,7 @@ main (int argc, char *const *argv)
 
   ret =
       (GNUNET_OK ==
-       GNUNET_SERVICE_run (argc, argv, "peerinfo", GNUNET_SERVICE_OPTION_NONE,
+       GNUNET_SERVICE_run (argc, argv, "peerinfo", GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN,
                            &run, NULL)) ? 0 : 1;
   GNUNET_free_non_null (networkIdDirectory);
   return ret;
diff --git a/src/statistics/gnunet-service-statistics.c b/src/statistics/gnunet-service-statistics.c
index 2b60d760b..5cced1426 100644
--- a/src/statistics/gnunet-service-statistics.c
+++ b/src/statistics/gnunet-service-statistics.c
@@ -651,6 +651,7 @@ handle_watch (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
+  GNUNET_SERVER_client_mark_monitor (client);
   ce = make_client_entry (client);
   msize = ntohs (message->size);
   if (msize < sizeof (struct GNUNET_MessageHeader))
@@ -844,7 +845,7 @@ main (int argc, char *const *argv)
 {
   return (GNUNET_OK ==
           GNUNET_SERVICE_run (argc, argv, "statistics",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+                              GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN, &run, NULL)) ? 0 : 1;
 }
 
 /* end of gnunet-service-statistics.c */
diff --git a/src/transport/gnunet-service-transport.c b/src/transport/gnunet-service-transport.c
index d518a3bf7..33c92d6e5 100644
--- a/src/transport/gnunet-service-transport.c
+++ b/src/transport/gnunet-service-transport.c
@@ -642,7 +642,7 @@ main (int argc, char *const *argv)
 {
   return (GNUNET_OK ==
           GNUNET_SERVICE_run (argc, argv, "transport",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+                              GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN, &run, NULL)) ? 0 : 1;
 }
 
-/* end of file gnunet-service-transport-new.c */
+/* end of file gnunet-service-transport.c */
diff --git a/src/transport/gnunet-service-transport_blacklist.c b/src/transport/gnunet-service-transport_blacklist.c
index 44a029474..8c368886b 100644
--- a/src/transport/gnunet-service-transport_blacklist.c
+++ b/src/transport/gnunet-service-transport_blacklist.c
@@ -230,11 +230,9 @@ read_blacklist_file ()
       GNUNET_CONFIGURATION_get_value_filename (GST_cfg, "TRANSPORT",
                                                "BLACKLIST_FILE", &fn))
   {
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Option `%s' in section `%s' not specified!\n",
                 "BLACKLIST_FILE", "TRANSPORT");
-#endif
     return;
   }
   if (GNUNET_OK != GNUNET_DISK_file_test (fn))
@@ -251,10 +249,8 @@ read_blacklist_file ()
   }
   if (fsize == 0)
   {
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklist file `%s' is empty.\n"),
                 fn);
-#endif
     GNUNET_free (fn);
     return;
   }
@@ -322,11 +318,9 @@ read_blacklist_file ()
     transport_name = GNUNET_malloc (tsize + 1);
     memcpy (transport_name, &data[pos], tsize);
     pos = colon_pos + 1;
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Read transport name `%s' in blacklist file.\n",
                 transport_name);
-#endif
     memcpy (&enc, &data[pos], sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded));
     if (!isspace
         ((unsigned char)
@@ -451,11 +445,9 @@ transmit_blacklist_message (void *cls, size_t size, void *buf)
                 GNUNET_i2s (&bc->peer));
     return 0;
   }
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Sending blacklist test for peer `%s' to client\n",
               GNUNET_i2s (&bc->peer));
-#endif
   bl = bc->bl_pos;
   bm.header.size = htons (sizeof (struct BlacklistMessage));
   bm.header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_BLACKLIST_QUERY);
@@ -484,11 +476,9 @@ do_blacklist_check (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   bl = bc->bl_pos;
   if (bl == NULL)
   {
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "No other blacklist clients active, will allow neighbour `%s'\n",
                 GNUNET_i2s (&bc->peer));
-#endif
     bc->cont (bc->cont_cls, &bc->peer, GNUNET_OK);
     GNUNET_CONTAINER_DLL_remove(bc_head, bc_tail, bc);
     GNUNET_free (bc);
@@ -579,7 +569,6 @@ test_connection_ok (void *cls, const struct GNUNET_PeerIdentity *neighbour,
 }
 
 
-
 /**
  * Initialize a blacklisting client.  We got a blacklist-init
  * message from this client, add him to the list of clients
@@ -607,6 +596,7 @@ GST_blacklist_handle_init (void *cls, struct GNUNET_SERVER_Client *client,
     }
     bl = bl->next;
   }
+  GNUNET_SERVER_client_mark_monitor (client);
   bl = GNUNET_malloc (sizeof (struct Blacklisters));
   bl->client = client;
   GNUNET_SERVER_client_keep (client);
@@ -640,9 +630,7 @@ GST_blacklist_handle_reply (void *cls, struct GNUNET_SERVER_Client *client,
     bl = bl->next;
   if (bl == NULL)
   {
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Blacklist client disconnected\n");
-#endif
     /* FIXME: other error handling here!? */
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
@@ -656,20 +644,16 @@ GST_blacklist_handle_reply (void *cls, struct GNUNET_SERVER_Client *client,
      * cancelled in the meantime... */
     if (ntohl (msg->is_allowed) == GNUNET_SYSERR)
     {
-#if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Blacklist check failed, peer not allowed\n");
-#endif
       bc->cont (bc->cont_cls, &bc->peer, GNUNET_NO);
       GNUNET_CONTAINER_DLL_remove (bc_head, bc_tail, bc);
       GNUNET_free (bc);
     }
     else
     {
-#if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Blacklist check succeeded, continuing with checks\n");
-#endif
       bc->bl_pos = bc->bl_pos->next;
       bc->task = GNUNET_SCHEDULER_add_now (&do_blacklist_check, bc);
     }
@@ -695,11 +679,9 @@ void
 GST_blacklist_add_peer (const struct GNUNET_PeerIdentity *peer,
                         const char *transport_name)
 {
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Adding peer `%s' with plugin `%s' to blacklist\n",
               GNUNET_i2s (peer), transport_name);
-#endif
   if (blacklist == NULL)
     blacklist =
         GNUNET_CONTAINER_multihashmap_create (TRANSPORT_BLACKLIST_HT_SIZE);
diff --git a/src/transport/gnunet-service-transport_clients.c b/src/transport/gnunet-service-transport_clients.c
index f430ef91a..939f57b10 100644
--- a/src/transport/gnunet-service-transport_clients.c
+++ b/src/transport/gnunet-service-transport_clients.c
@@ -208,10 +208,7 @@ setup_client (struct GNUNET_SERVER_Client *client)
   GNUNET_assert (lookup_client (client) == NULL);
   tc = GNUNET_malloc (sizeof (struct TransportClient));
   tc->client = client;
-
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Client %p connected\n", tc);
-#endif
   return tc;
 }
 
@@ -293,10 +290,8 @@ transmit_to_client_callback (void *cls, size_t size, void *buf)
   tc->th = NULL;
   if (buf == NULL)
   {
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Transmission to client failed, closing connection.\n");
-#endif
     return 0;
   }
   cbuf = buf;
@@ -307,11 +302,9 @@ transmit_to_client_callback (void *cls, size_t size, void *buf)
     msize = ntohs (msg->size);
     if (msize + tsize > size)
       break;
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Transmitting message of type %u to client %p.\n",
                 ntohs (msg->type), tc);
-#endif
     GNUNET_CONTAINER_DLL_remove (tc->message_queue_head, tc->message_queue_tail,
                                  q);
     tc->message_count--;
@@ -403,10 +396,8 @@ client_disconnect_notification (void *cls, struct GNUNET_SERVER_Client *client)
   tc = lookup_client (client);
   if (tc == NULL)
     return;
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
               "Client %p disconnected, cleaning up.\n", tc);
-#endif
   while (NULL != (mqe = tc->message_queue_head))
   {
     GNUNET_CONTAINER_DLL_remove (tc->message_queue_head, tc->message_queue_tail,
@@ -481,18 +472,14 @@ clients_handle_start (void *cls, struct GNUNET_SERVER_Client *client,
 
   tc = lookup_client (client);
 
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
               "Client %p sent START\n", tc);
-#endif
   if (tc != NULL)
   {
     /* got 'start' twice from the same client, not allowed */
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
                 "TransportClient %p ServerClient %p sent multiple START messages\n",
                 tc, tc->client);
-#endif
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
@@ -627,19 +614,15 @@ clients_handle_send (void *cls, struct GNUNET_SERVER_Client *client,
                             gettext_noop
                             ("# bytes payload received for other peers"), msize,
                             GNUNET_NO);
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received `%s' request from client with target `%4s' and first message of type %u and total size %u\n",
               "SEND", GNUNET_i2s (&obm->peer), ntohs (obmm->type), msize);
-#endif
   if (GNUNET_NO == GST_neighbours_test_connected (&obm->peer))
   {
     /* not connected, not allowed to send; can happen due to asynchronous operations */
-#if DEBUG_TRANSPORT
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Could not send message to peer `%s': not connected\n",
                 GNUNET_i2s (&obm->peer));
-#endif
     GNUNET_STATISTICS_update (GST_stats,
                               gettext_noop
                               ("# bytes payload dropped (other peer was not connected)"),
@@ -695,11 +678,9 @@ clients_handle_request_connect (void *cls, struct GNUNET_SERVER_Client *client,
                             gettext_noop
                             ("# REQUEST CONNECT messages received"), 1,
                             GNUNET_NO);
-#if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received a request connect message for peer `%s'\n",
               GNUNET_i2s (&trcm->peer));
-#endif
   (void) GST_blacklist_test_allowed (&trcm->peer, NULL, &try_connect_if_allowed,
                                      NULL);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
diff --git a/src/transport/plugin_transport_tcp.c b/src/transport/plugin_transport_tcp.c
index ea8b6ee08..54bcf16ef 100644
--- a/src/transport/plugin_transport_tcp.c
+++ b/src/transport/plugin_transport_tcp.c
@@ -2161,7 +2161,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
     aport = 0;
   if (bport != 0)
   {
-    service = GNUNET_SERVICE_start ("transport-tcp", env->cfg);
+    service = GNUNET_SERVICE_start ("transport-tcp", env->cfg, GNUNET_SERVICE_OPTION_NONE);
     if (service == NULL)
     {
       GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "tcp",
diff --git a/src/util/server.c b/src/util/server.c
index 6a458b35b..4c7f62c17 100644
--- a/src/util/server.c
+++ b/src/util/server.c
@@ -133,12 +133,6 @@ struct GNUNET_SERVER_Handle
   GNUNET_SCHEDULER_TaskIdentifier listen_task;
 
   /**
-   * Do we ignore messages of types that we do not understand or do we
-   * require that a handler is found (and if not kill the connection)?
-   */
-  int require_found;
-
-  /**
    * Alternative function to create a MST instance.
    */
   GNUNET_SERVER_MstCreateCallback mst_create;
@@ -157,6 +151,19 @@ struct GNUNET_SERVER_Handle
    * Closure for 'mst_'-callbacks.
    */
   void *mst_cls;
+
+  /**
+   * Do we ignore messages of types that we do not understand or do we
+   * require that a handler is found (and if not kill the connection)?
+   */
+  int require_found;
+
+  /**
+   * Set to GNUNET_YES once we are in 'soft' shutdown where we wait for
+   * all non-monitor clients to disconnect before we call
+   * GNUNET_SERVER_destroy.  See 'test_monitor_clients'.
+   */
+  int in_soft_shutdown;
 };
 
 
@@ -291,6 +298,13 @@ struct GNUNET_SERVER_Client
   int persist;
 
   /**
+   * Is this client a 'monitor' client that should not be counted
+   * when deciding on destroying the server during soft shutdown?
+   * (see also GNUNET_SERVICE_start)
+   */
+  int is_monitor;
+
+  /**
    * Type of last message processed (for warn_no_receive_done).
    */
   uint16_t warn_type;
@@ -558,6 +572,76 @@ GNUNET_SERVER_create (GNUNET_CONNECTION_AccessCheck access, void *access_cls,
 
 
 /**
+ * Set the 'monitor' flag on this client.  Clients which have been
+ * marked as 'monitors' won't prevent the server from shutting down
+ * once 'GNUNET_SERVER_stop_listening' has been invoked.  The idea is
+ * that for "normal" clients we likely want to allow them to process
+ * their requests; however, monitor-clients are likely to 'never'
+ * disconnect during shutdown and thus will not be considered when
+ * determining if the server should continue to exist after
+ * 'GNUNET_SERVER_destroy' has been called.
+ *
+ * @param client the client to set the 'monitor' flag on
+ */
+void
+GNUNET_SERVER_client_mark_monitor (struct GNUNET_SERVER_Client *client)
+{
+  client->is_monitor = GNUNET_YES;
+}
+
+
+/**
+ * Check if only 'monitor' clients are left.  If so, destroy the
+ * server completely.
+ *
+ * @param server server to test for full shutdown
+ */
+static void
+test_monitor_clients (struct GNUNET_SERVER_Handle *server)
+{
+  struct GNUNET_SERVER_Client *client;
+
+  if (GNUNET_YES != server->in_soft_shutdown)
+    return;
+  for (client = server->clients; NULL != client; client = client->next)
+    if (GNUNET_NO == client->is_monitor)
+      return; /* not done yet */
+  GNUNET_SERVER_destroy (server);
+}
+
+
+/**
+ * Stop the listen socket and get ready to shutdown the server
+ * once only 'monitor' clients are left.
+ *
+ * @param server server to stop listening on
+ */
+void
+GNUNET_SERVER_stop_listening (struct GNUNET_SERVER_Handle *server)
+{
+  unsigned int i;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Server in soft shutdown\n");
+  if (GNUNET_SCHEDULER_NO_TASK != server->listen_task)
+  {
+    GNUNET_SCHEDULER_cancel (server->listen_task);
+    server->listen_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (NULL != server->listen_sockets)
+  {
+    i = 0;
+    while (NULL != server->listen_sockets[i])
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (server->listen_sockets[i++]));
+    GNUNET_free (server->listen_sockets);
+    server->listen_sockets = NULL;
+  }
+  server->in_soft_shutdown = GNUNET_YES;
+  test_monitor_clients (server);
+}
+
+
+/**
  * Free resources held by this server.
  *
  * @param server server to destroy
@@ -1163,10 +1247,10 @@ GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client)
     GNUNET_CONNECTION_receive_cancel (client->connection);
     client->receive_pending = GNUNET_NO;
   }
+  server = client->server;
   rc = client->reference_count;
   if (GNUNET_YES != client->shutdown_now)
   {
-    server = client->server;
     client->shutdown_now = GNUNET_YES;
     prev = NULL;
     pos = server->clients;
@@ -1212,11 +1296,13 @@ GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client)
     GNUNET_SERVER_notify_transmit_ready_cancel (&client->th);
   GNUNET_CONNECTION_destroy (client->connection);
 
-  if (NULL != client->server->mst_destroy)
-    client->server->mst_destroy (client->server->mst_cls, client->mst);
+  if (NULL != server->mst_destroy)
+    server->mst_destroy (server->mst_cls, client->mst);
   else
     GNUNET_SERVER_mst_destroy (client->mst);
   GNUNET_free (client);
+  /* we might be in soft-shutdown, test if we're done */
+  test_monitor_clients (server);
 }
 
 
diff --git a/src/util/service.c b/src/util/service.c
index 4eb35d97a..3cddae69e 100644
--- a/src/util/service.c
+++ b/src/util/service.c
@@ -436,7 +436,7 @@ struct GNUNET_SERVICE_Context
   /**
    * Name of our service.
    */
-  const char *serviceName;
+  const char *service_name;
 
   /**
    * Main service-specific task to run.
@@ -697,7 +697,7 @@ check_access (void *cls, const struct GNUNET_CONNECTION_Credentials *uc,
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Access from `%s' denied to service `%s'\n"), 
          GNUNET_a2s (addr, addrlen),
-         sctx->serviceName);
+         sctx->service_name);
   }
   return ret;
 }
@@ -716,7 +716,7 @@ get_pid_file_name (struct GNUNET_SERVICE_Context *sctx)
   char *pif;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->serviceName,
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
                                                "PIDFILE", &pif))
     return NULL;
   return pif;
@@ -738,20 +738,20 @@ process_acl4 (struct IPv4NetworkSet **ret, struct GNUNET_SERVICE_Context *sctx,
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->serviceName, option))
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
   {
     *ret = NULL;    
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
                 GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->serviceName,
+                                                       sctx->service_name,
                                                        option, &opt));
   if (NULL == (*ret = parse_ipv4_specification (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->serviceName, option);
+         opt, sctx->service_name, option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -775,20 +775,20 @@ process_acl6 (struct IPv6NetworkSet **ret, struct GNUNET_SERVICE_Context *sctx,
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->serviceName, option))
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
   {
     *ret = NULL;
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
                 GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->serviceName,
+                                                       sctx->service_name,
                                                        option, &opt));
   if (NULL == (*ret = parse_ipv6_specification (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->serviceName, option);
+         opt, sctx->service_name, option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -841,7 +841,7 @@ add_unixpath (struct sockaddr **saddrs, socklen_t * saddrlens,
  * Get the list of addresses that a server for the given service
  * should bind to.
  *
- * @param serviceName name of the service
+ * @param service_name name of the service
  * @param cfg configuration (which specifies the addresses)
  * @param addrs set (call by reference) to an array of pointers to the
  *              addresses the server should bind to and listen on; the
@@ -858,7 +858,7 @@ add_unixpath (struct sockaddr **saddrs, socklen_t * saddrlens,
  *              set to NULL).
  */
 int
-GNUNET_SERVICE_get_server_addresses (const char *serviceName,
+GNUNET_SERVICE_get_server_addresses (const char *service_name,
                                      const struct GNUNET_CONFIGURATION_Handle
                                      *cfg, struct sockaddr ***addrs,
                                      socklen_t ** addr_lens)
@@ -881,11 +881,11 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
   *addrs = NULL;
   *addr_lens = NULL;
   desc = NULL;
-  if (GNUNET_CONFIGURATION_have_value (cfg, serviceName, "DISABLEV6"))
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "DISABLEV6"))
   {
     if (GNUNET_SYSERR ==
         (disablev6 =
-         GNUNET_CONFIGURATION_get_value_yesno (cfg, serviceName, "DISABLEV6")))
+         GNUNET_CONFIGURATION_get_value_yesno (cfg, service_name, "DISABLEV6")))
       return GNUNET_SYSERR;
   }
   else
@@ -906,7 +906,7 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
       LOG (GNUNET_ERROR_TYPE_INFO,
            _
            ("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
-           serviceName, STRERROR (errno));
+           service_name, STRERROR (errno));
       disablev6 = GNUNET_YES;
     }
     else
@@ -917,24 +917,24 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
   }
 
   port = 0;
-  if (GNUNET_CONFIGURATION_have_value (cfg, serviceName, "PORT"))
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "PORT"))
   {
     GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_number (cfg, serviceName,
+                  GNUNET_CONFIGURATION_get_value_number (cfg, service_name,
                                                          "PORT", &port));
     if (port > 65535)
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Require valid port number for service `%s' in configuration!\n"),
-           serviceName);
+           service_name);
       return GNUNET_SYSERR;
     }
   }
 
-  if (GNUNET_CONFIGURATION_have_value (cfg, serviceName, "BINDTO"))
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "BINDTO"))
   {
     GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_string (cfg, serviceName,
+                  GNUNET_CONFIGURATION_get_value_string (cfg, service_name,
                                                          "BINDTO", &hostname));
   }
   else
@@ -943,9 +943,9 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
   unixpath = NULL;
 #ifdef AF_UNIX
   if ((GNUNET_YES ==
-       GNUNET_CONFIGURATION_have_value (cfg, serviceName, "UNIXPATH")) &&
+       GNUNET_CONFIGURATION_have_value (cfg, service_name, "UNIXPATH")) &&
       (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_string (cfg, serviceName, "UNIXPATH",
+       GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "UNIXPATH",
                                               &unixpath)) &&
       (0 < strlen (unixpath)))
   {
@@ -976,7 +976,7 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
       LOG (GNUNET_ERROR_TYPE_INFO,
            _
            ("Disabling UNIX domain socket support for service `%s', failed to create UNIX domain socket: %s\n"),
-           serviceName, STRERROR (errno));
+           service_name, STRERROR (errno));
       GNUNET_free (unixpath);
       unixpath = NULL;
     }
@@ -993,7 +993,7 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
     LOG (GNUNET_ERROR_TYPE_ERROR,
          _
          ("Have neither PORT nor UNIXPATH for service `%s', but one is required\n"),
-         serviceName);
+         service_name);
     GNUNET_free_non_null (hostname);
     return GNUNET_SYSERR;
   }
@@ -1013,7 +1013,7 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Resolving `%s' since that is where `%s' will bind to.\n", hostname,
-         serviceName);
+         service_name);
     memset (&hints, 0, sizeof (struct addrinfo));
     if (disablev6)
       hints.ai_family = AF_INET;
@@ -1066,7 +1066,7 @@ GNUNET_SERVICE_get_server_addresses (const char *serviceName,
       if ((SOCK_STREAM != pos->ai_socktype) && (0 != pos->ai_socktype))
         continue;               /* huh? */
       LOG (GNUNET_ERROR_TYPE_DEBUG, "Service `%s' will bind to `%s'\n",
-           serviceName, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
+           service_name, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
       if (AF_INET == pos->ai_family)
       {
         GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
@@ -1269,15 +1269,15 @@ setup_service (struct GNUNET_SERVICE_Context *sctx)
   int flags;
 #endif
 
-  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->serviceName, "TIMEOUT"))
+  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, "TIMEOUT"))
   {
     if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->serviceName,
+        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->service_name,
                                              "TIMEOUT", &idleout))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TIMEOUT", sctx->serviceName);
+           "TIMEOUT", sctx->service_name);
       return GNUNET_SYSERR;
     }
     sctx->timeout = idleout;
@@ -1286,16 +1286,16 @@ setup_service (struct GNUNET_SERVICE_Context *sctx)
     sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
 
   if (GNUNET_CONFIGURATION_have_value
-      (sctx->cfg, sctx->serviceName, "TOLERANT"))
+      (sctx->cfg, sctx->service_name, "TOLERANT"))
   {
     if (GNUNET_SYSERR ==
         (tolerant =
-         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->serviceName,
+         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
                                                "TOLERANT")))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TOLERANT", sctx->serviceName);
+           "TOLERANT", sctx->service_name);
       return GNUNET_SYSERR;
     }
   }
@@ -1344,15 +1344,15 @@ setup_service (struct GNUNET_SERVICE_Context *sctx)
 
   if ((NULL == sctx->lsocks) &&
       (GNUNET_SYSERR ==
-       GNUNET_SERVICE_get_server_addresses (sctx->serviceName, sctx->cfg,
+       GNUNET_SERVICE_get_server_addresses (sctx->service_name, sctx->cfg,
                                             &sctx->addrs, &sctx->addrlens)))
     return GNUNET_SYSERR;
   sctx->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
   sctx->match_uid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->serviceName,
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
                                             "UNIX_MATCH_UID");
   sctx->match_gid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->serviceName,
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
                                             "UNIX_MATCH_GID");
   process_acl4 (&sctx->v4_denied, sctx, "REJECT_FROM");
   process_acl4 (&sctx->v4_allowed, sctx, "ACCEPT_FROM");
@@ -1376,7 +1376,7 @@ get_user_name (struct GNUNET_SERVICE_Context *sctx)
   char *un;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->serviceName,
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
                                                "USERNAME", &un))
     return NULL;
   return un;
@@ -1443,19 +1443,21 @@ write_pid_file (struct GNUNET_SERVICE_Context *sctx, pid_t pid)
 
 
 /**
- * Task run during shutdown.
+ * Task run during shutdown.  Stops the server/service.
  *
- * @param cls unused
+ * @param cls the 'struct GNUNET_SERVICE_Context'
  * @param tc unused
  */
 static void
 shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct GNUNET_SERVER_Handle *server = cls;
+  struct GNUNET_SERVICE_Context *service = cls;
+  struct GNUNET_SERVER_Handle *server = service->server;
 
-  // FIXME: we should not unconditionally destroy the server
-  // here (often only stopping 'listening' would be better)
-  GNUNET_SERVER_destroy (server);
+  if (0 != (service->options & GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN))
+    GNUNET_SERVER_stop_listening (server);
+  else
+    GNUNET_SERVER_destroy (server);
 }
 
 
@@ -1488,7 +1490,7 @@ service_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       while (NULL != sctx->addrs[i])
       {
         LOG (GNUNET_ERROR_TYPE_INFO, _("Failed to start `%s' at `%s'\n"),
-             sctx->serviceName, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+             sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
         i++;
       }
     }
@@ -1500,7 +1502,7 @@ service_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     /* install a task that will kill the server
      * process if the scheduler ever gets a shutdown signal */
     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task,
-                                  sctx->server);
+                                  sctx);
   }
   sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
   memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
@@ -1521,7 +1523,7 @@ service_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     while (NULL != sctx->addrs[i])
     {
       LOG (GNUNET_ERROR_TYPE_INFO, _("Service `%s' runs at %s\n"),
-           sctx->serviceName, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+           sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
       i++;
     }
   }
@@ -1681,16 +1683,16 @@ pid_file_delete (struct GNUNET_SERVICE_Context *sctx)
  *
  * @param argc number of command line arguments
  * @param argv command line arguments
- * @param serviceName our service name
- * @param opt service options
+ * @param service_name our service name
+ * @param options service options
  * @param task main task of the service
  * @param task_cls closure for task
  * @return GNUNET_SYSERR on error, GNUNET_OK
  *         if we shutdown nicely
  */
 int
-GNUNET_SERVICE_run (int argc, char *const *argv, const char *serviceName,
-                    enum GNUNET_SERVICE_Options opt, GNUNET_SERVICE_Main task,
+GNUNET_SERVICE_run (int argc, char *const *argv, const char *service_name,
+                    enum GNUNET_SERVICE_Options options, GNUNET_SERVICE_Main task,
                     void *task_cls)
 {
 #define HANDLE_ERROR do { GNUNET_break (0); goto shutdown; } while (0)
@@ -1724,19 +1726,19 @@ GNUNET_SERVICE_run (int argc, char *const *argv, const char *serviceName,
   loglev = NULL;
   cfg_fn = GNUNET_strdup (GNUNET_DEFAULT_USER_CONFIG_FILE);
   memset (&sctx, 0, sizeof (sctx));
-  sctx.options = opt;
+  sctx.options = options;
   sctx.ready_confirm_fd = -1;
   sctx.ret = GNUNET_OK;
   sctx.timeout = GNUNET_TIME_UNIT_FOREVER_REL;
   sctx.task = task;
   sctx.task_cls = task_cls;
-  sctx.serviceName = serviceName;
+  sctx.service_name = service_name;
   sctx.cfg = cfg = GNUNET_CONFIGURATION_create ();
   /* setup subsystems */
   if (GNUNET_SYSERR ==
-      GNUNET_GETOPT_run (serviceName, service_options, argc, argv))
+      GNUNET_GETOPT_run (service_name, service_options, argc, argv))
     goto shutdown;
-  if (GNUNET_OK != GNUNET_log_setup (serviceName, loglev, logfile))
+  if (GNUNET_OK != GNUNET_log_setup (service_name, loglev, logfile))
     HANDLE_ERROR;
   if (GNUNET_OK != GNUNET_CONFIGURATION_load (cfg, cfg_fn))
     goto shutdown;
@@ -1747,7 +1749,7 @@ GNUNET_SERVICE_run (int argc, char *const *argv, const char *serviceName,
   if (GNUNET_OK != set_user_id (&sctx))
     goto shutdown;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Service `%s' runs with configuration from `%s'\n", serviceName, cfg_fn);
+       "Service `%s' runs with configuration from `%s'\n", service_name, cfg_fn);
   if ((GNUNET_OK ==
        GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
                                               "SKEW_OFFSET", &skew_offset)) &&
@@ -1799,13 +1801,15 @@ shutdown:
  * Run a service startup sequence within an existing
  * initialized system.
  *
- * @param serviceName our service name
+ * @param service_name our service name
  * @param cfg configuration to use
+ * @param options service options
  * @return NULL on error, service handle
  */
 struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *serviceName,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg)
+GNUNET_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      enum GNUNET_SERVICE_Options options)
 {
   int i;
   struct GNUNET_SERVICE_Context *sctx;
@@ -1814,8 +1818,9 @@ GNUNET_SERVICE_start (const char *serviceName,
   sctx->ready_confirm_fd = -1;  /* no daemonizing */
   sctx->ret = GNUNET_OK;
   sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  sctx->serviceName = serviceName;
+  sctx->service_name = service_name;
   sctx->cfg = cfg;
+  sctx->options = options;
 
   /* setup subsystems */
   if (GNUNET_OK != setup_service (sctx))