Use less locking with MHD_USE_THREAD_PER_CONNECTION.

Do not maintain global list out timeouts as each thread use individual timeout.
As result - global mutex is not acquired after each single send()/recv().

3 files changed, 81 insertions, 65 deletions

diff --git a/src/microhttpd/connection.c b/src/microhttpd/connection.c
index 3aa0d16c..e1bc1f7e 100644
--- a/src/microhttpd/connection.c
+++ b/src/microhttpd/connection.c
@@ -2132,22 +2132,19 @@ update_last_activity (struct MHD_Connection *connection)
   struct MHD_Daemon *daemon = connection->daemon;
 
   connection->last_activity = MHD_monotonic_sec_counter();
+  if (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+    return; /* each connection has personal timeout */
+
   if (connection->connection_timeout != daemon->connection_timeout)
     return; /* custom timeout, no need to move it in "normal" DLL */
 
   /* move connection to head of timeout list (by remove + add operation) */
-  if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-       (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to acquire cleanup mutex\n");
   XDLL_remove (daemon->normal_timeout_head,
                daemon->normal_timeout_tail,
                connection);
   XDLL_insert (daemon->normal_timeout_head,
                daemon->normal_timeout_tail,
                connection);
-  if  ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-        (MHD_YES != MHD_mutex_unlock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to release cleanup mutex\n");
 }
 
 
@@ -2397,17 +2394,22 @@ cleanup_connection (struct MHD_Connection *connection)
       MHD_destroy_response (connection->response);
       connection->response = NULL;
     }
-  if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-       (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to acquire cleanup mutex\n");
-  if (connection->connection_timeout == daemon->connection_timeout)
-    XDLL_remove (daemon->normal_timeout_head,
-                 daemon->normal_timeout_tail,
-                 connection);
+  if (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+    {
+      if (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) 
+        MHD_PANIC ("Failed to acquire cleanup mutex\n");
+    }
   else
-    XDLL_remove (daemon->manual_timeout_head,
-                 daemon->manual_timeout_tail,
-                 connection);
+    {
+      if (connection->connection_timeout == daemon->connection_timeout)
+        XDLL_remove (daemon->normal_timeout_head,
+                     daemon->normal_timeout_tail,
+                     connection);
+      else
+        XDLL_remove (daemon->manual_timeout_head,
+                     daemon->manual_timeout_tail,
+                     connection);
+    }
   if (MHD_YES == connection->suspended)
     DLL_remove (daemon->suspended_connections_head,
                 daemon->suspended_connections_tail,
@@ -3040,37 +3042,33 @@ MHD_set_connection_option (struct MHD_Connection *connection,
   switch (option)
     {
     case MHD_CONNECTION_OPTION_TIMEOUT:
-      if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-           (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-        MHD_PANIC ("Failed to acquire cleanup mutex\n");
-      if (MHD_YES != connection->suspended)
+      if ( (0 == (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
+          (MHD_YES != connection->suspended) )
         {
           if (connection->connection_timeout == daemon->connection_timeout)
             XDLL_remove (daemon->normal_timeout_head,
-                         daemon->normal_timeout_tail,
-                         connection);
+                          daemon->normal_timeout_tail,
+                          connection);
           else
             XDLL_remove (daemon->manual_timeout_head,
-                         daemon->manual_timeout_tail,
-                         connection);
+                          daemon->manual_timeout_tail,
+                          connection);
         }
       va_start (ap, option);
       connection->connection_timeout = va_arg (ap, unsigned int);
       va_end (ap);
-      if (MHD_YES != connection->suspended)
+      if ( (0 == (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
+          (MHD_YES != connection->suspended) )
         {
           if (connection->connection_timeout == daemon->connection_timeout)
             XDLL_insert (daemon->normal_timeout_head,
-                         daemon->normal_timeout_tail,
-                         connection);
+                          daemon->normal_timeout_tail,
+                          connection);
           else
             XDLL_insert (daemon->manual_timeout_head,
-                         daemon->manual_timeout_tail,
-                         connection);
+                          daemon->manual_timeout_tail,
+                          connection);
         }
-      if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-           (MHD_YES != MHD_mutex_unlock_ (&daemon->cleanup_connection_mutex)) )
-        MHD_PANIC ("Failed to release cleanup mutex\n");
       return MHD_YES;
     default:
       return MHD_NO;
diff --git a/src/microhttpd/daemon.c b/src/microhttpd/daemon.c
index ea958204..2c5d220d 100644
--- a/src/microhttpd/daemon.c
+++ b/src/microhttpd/daemon.c
@@ -842,7 +842,6 @@ MHD_handle_connection (void *data)
   MHD_socket maxsock;
   struct timeval tv;
   struct timeval *tvp;
-  unsigned int timeout;
   time_t now;
 #if WINDOWS
   MHD_pipe spipe = con->daemon->wpipe[0];
@@ -857,10 +856,10 @@ MHD_handle_connection (void *data)
   struct pollfd p[1 + EXTRA_SLOTS];
 #endif
 
-  timeout = con->daemon->connection_timeout;
   while ( (MHD_YES != con->daemon->shutdown) &&
           (MHD_CONNECTION_CLOSED != con->state) )
     {
+      unsigned const int timeout = con->daemon->connection_timeout;
       tvp = NULL;
 #if HTTPS_SUPPORT
       if (MHD_YES == con->tls_read_ready)
@@ -1563,12 +1562,15 @@ internal_add_connection (struct MHD_Daemon *daemon,
     }
 #endif
 
-  if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-       (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to acquire cleanup mutex\n");
-  XDLL_insert (daemon->normal_timeout_head,
-               daemon->normal_timeout_tail,
-               connection);
+  if (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+  {
+    if (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex))
+      MHD_PANIC ("Failed to acquire cleanup mutex\n");
+  }
+  else
+   XDLL_insert (daemon->normal_timeout_head,
+                daemon->normal_timeout_tail,
+                connection);
   DLL_insert (daemon->connections_head,
               daemon->connections_tail,
               connection);
@@ -1655,15 +1657,18 @@ internal_add_connection (struct MHD_Daemon *daemon,
   if (0 != MHD_socket_close_ (client_socket))
     MHD_PANIC ("close failed\n");
   MHD_ip_limit_del (daemon, addr, addrlen);
-  if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-       (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to acquire cleanup mutex\n");
+  if (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+  {
+    if (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex))
+      MHD_PANIC ("Failed to acquire cleanup mutex\n");
+  }
+  else
+    XDLL_remove (daemon->normal_timeout_head,
+                 daemon->normal_timeout_tail,
+                 connection);
   DLL_remove (daemon->connections_head,
               daemon->connections_tail,
               connection);
-  XDLL_remove (daemon->normal_timeout_head,
-               daemon->normal_timeout_tail,
-               connection);
   if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
        (MHD_YES != MHD_mutex_unlock_ (&daemon->cleanup_connection_mutex)) )
     MHD_PANIC ("Failed to release cleanup mutex\n");
@@ -1712,23 +1717,28 @@ MHD_suspend_connection (struct MHD_Connection *connection)
   daemon = connection->daemon;
   if (MHD_USE_SUSPEND_RESUME != (daemon->options & MHD_USE_SUSPEND_RESUME))
     MHD_PANIC ("Cannot suspend connections without enabling MHD_USE_SUSPEND_RESUME!\n");
-  if ( (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION)) &&
-       (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex)) )
-    MHD_PANIC ("Failed to acquire cleanup mutex\n");
+  if (0 != (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+    {
+      if (MHD_YES != MHD_mutex_lock_ (&daemon->cleanup_connection_mutex))
+        MHD_PANIC ("Failed to acquire cleanup mutex\n");
+    }
+  else
+    {
+      if (connection->connection_timeout == daemon->connection_timeout)
+        XDLL_remove (daemon->normal_timeout_head,
+                     daemon->normal_timeout_tail,
+                     connection);
+      else
+        XDLL_remove (daemon->manual_timeout_head,
+                     daemon->manual_timeout_tail,
+                     connection);
+    }
   DLL_remove (daemon->connections_head,
               daemon->connections_tail,
               connection);
   DLL_insert (daemon->suspended_connections_head,
               daemon->suspended_connections_tail,
               connection);
-  if (connection->connection_timeout == daemon->connection_timeout)
-    XDLL_remove (daemon->normal_timeout_head,
-                 daemon->normal_timeout_tail,
-                 connection);
-  else
-    XDLL_remove (daemon->manual_timeout_head,
-                 daemon->manual_timeout_tail,
-                 connection);
 #if EPOLL_SUPPORT
   if (0 != (daemon->options & MHD_USE_EPOLL_LINUX_ONLY))
     {
@@ -1826,14 +1836,17 @@ resume_suspended_connections (struct MHD_Daemon *daemon)
       DLL_insert (daemon->connections_head,
                   daemon->connections_tail,
                   pos);
-      if (pos->connection_timeout == daemon->connection_timeout)
-        XDLL_insert (daemon->normal_timeout_head,
-                     daemon->normal_timeout_tail,
-                     pos);
-      else
-        XDLL_insert (daemon->manual_timeout_head,
-                     daemon->manual_timeout_tail,
-                     pos);
+      if (0 == (daemon->options & MHD_USE_THREAD_PER_CONNECTION))
+        {
+          if (pos->connection_timeout == daemon->connection_timeout)
+            XDLL_insert (daemon->normal_timeout_head,
+                         daemon->normal_timeout_tail,
+                         pos);
+          else
+            XDLL_insert (daemon->manual_timeout_head,
+                         daemon->manual_timeout_tail,
+                         pos);
+        }
 #if EPOLL_SUPPORT
       if (0 != (daemon->options & MHD_USE_EPOLL_LINUX_ONLY))
         {
diff --git a/src/microhttpd/internal.h b/src/microhttpd/internal.h
index 3856a623..58317040 100644
--- a/src/microhttpd/internal.h
+++ b/src/microhttpd/internal.h
@@ -976,12 +976,15 @@ struct MHD_Daemon
    * All connections by default start in this list; if a custom
    * timeout that does not match @e connection_timeout is set, they
    * are moved to the @e manual_timeout_head-XDLL.
+   * Not used in MHD_USE_THREAD_PER_CONNECTION mode as each thread
+   * needs only one connection-specific timeout.
    */
   struct MHD_Connection *normal_timeout_head;
 
   /**
    * Tail of the XDLL of ALL connections with a default timeout,
    * sorted by timeout (earliest timeout at the tail).
+   * Not used in MHD_USE_THREAD_PER_CONNECTION mode.
    */
   struct MHD_Connection *normal_timeout_tail;
 
@@ -989,12 +992,14 @@ struct MHD_Daemon
    * Head of the XDLL of ALL connections with a non-default/custom
    * timeout, unsorted.  MHD will do a O(n) scan over this list to
    * determine the current timeout.
+   * Not used in MHD_USE_THREAD_PER_CONNECTION mode.
    */
   struct MHD_Connection *manual_timeout_head;
 
   /**
    * Tail of the XDLL of ALL connections with a non-default/custom
    * timeout, unsorted.
+   * Not used in MHD_USE_THREAD_PER_CONNECTION mode.
    */
   struct MHD_Connection *manual_timeout_tail;