commit aad43f23f0aa5cf997d9e880e2d78eefd9c6a221
parent 1b013c78a695cee32135ec5ec93dff959bf8f8f6
Author: ng0 <ng0@n0.is>
Date: Wed, 24 Jul 2019 10:03:37 +0000
incomplete commit, adding 2 new helper functions and more.
Diffstat:
3 files changed, 333 insertions(+), 6 deletions(-)
diff --git a/src/microhttpd/connection.c b/src/microhttpd/connection.c
@@ -3350,12 +3350,20 @@ MHD_connection_handle_write (struct MHD_Connection *connection)
return;
case MHD_CONNECTION_HEADERS_SENDING:
/* TODO: Maybe use MHD_send_on_connection2_()? */
+ /*
+ ret = MHD_send_on_connection2_ (struct MHD_Connection *connection,
+ const char *header,
+ size_t header_size,
+ const char *buffer,
+ size_t buffer_size);
+ */
ret = MHD_send_on_connection_ (connection,
&connection->write_buffer
[connection->write_buffer_send_offset],
connection->write_buffer_append_offset -
connection->write_buffer_send_offset,
MHD_SSO_MAY_CORK);
+
if (ret < 0)
{
if (MHD_ERR_AGAIN_ == ret)
@@ -3392,7 +3400,8 @@ MHD_connection_handle_write (struct MHD_Connection *connection)
#if defined(_MHD_HAVE_SENDFILE)
if (MHD_resp_sender_sendfile == connection->resp_sender)
{
- ret = sendfile_adapter (connection);
+ // ret = sendfile_adapter (connection);
+ ret = MHD_sendfile_on_connection_ (connection);
}
else
#else /* ! _MHD_HAVE_SENDFILE */
diff --git a/src/microhttpd/mhd_send.c b/src/microhttpd/mhd_send.c
@@ -33,6 +33,52 @@
#include "mhd_send.h"
/**
+ * Set socket to nodelay, on or off.
+ *
+ * @param connection the MHD_Connection structure
+ * @param value the state to set, boolean
+ */
+void
+MHD_send_socket_state_nodelay_ (struct MHD_Connection *connection,
+ bool value)
+{
+#if TCP_NODELAY
+ const MHD_SCKT_OPT_BOOL_ state_val = value ? 1 : 0;
+
+ if (0 == setsockopt (connection->socket_fd,
+ IPPROTO_TCP,
+ TCP_NODELAY,
+ (const void *) &state_val,
+ sizeof (state_val)))
+ {
+ connection->sk_tcp_nodelay_on = value;
+ }
+#endif
+}
+
+void
+MHD_send_socket_state_nopush_ (struct MHD_Connection *connection,
+ bool value,
+ bool state_store)
+{
+#if TCP_NOPUSH
+ const MHD_SCKT_OPT_BOOL_ state_val = value ? 1 : 0;
+
+ if (0 == setsockopt (connection->socket_fd,
+ IPPROTO_TCP,
+ TCP_NOPUSH,
+ (const void *) &state_val,
+ sizeof (state_val)))
+ {
+ /* When TRUE above, this is usually FALSE, but
+ * not always. We can't use the negation of
+ * value for that reason. */
+ connection->sk_tcp_nodelay_on = state_store;
+ }
+#endif
+}
+
+/**
* Send buffer on connection, and remember the current state of
* the socket options; only call setsockopt when absolutely
* necessary.
@@ -145,6 +191,8 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
* exist and we can disregard TCP_NODELAY unless requested. */
if ((! using_tls) && (use_corknopush) && (have_cork && ! want_cork))
{
+ MHD_send_socket_state_nopush_ (connection, true, false);
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NOPUSH,
@@ -153,11 +201,13 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = false;
}
+ */
}
#endif
#if TCP_NODELAY
if ((! using_tls) && (! use_corknopush) && (! have_cork && want_cork))
{
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NODELAY,
@@ -166,6 +216,8 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = false;
}
+ */
+ MHD_send_socket_state_nodelay_ (connection, false);
}
#endif
@@ -274,6 +326,8 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
* for the TCP_CORK case here. */
if ((! using_tls) && (use_corknopush) && (have_cork && ! want_cork))
{
+ MHD_send_socket_state_nopush_ (connection, true, false);
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NOPUSH,
@@ -282,12 +336,15 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = false;
}
+ */
}
#endif
#if TCP_NODELAY
if ((! using_tls) && (! use_corknopush) && (have_cork && ! want_cork))
{
+ MHD_send_socket_state_nodelay_ (connection, true);
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NODELAY,
@@ -296,6 +353,7 @@ MHD_send_on_connection_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = true;
}
+ */
}
#endif
@@ -348,6 +406,8 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
#if TCP_NODELAY
if ((! use_corknopush) && (! have_cork && want_cork))
{
+ MHD_send_socket_state_nodelay_ (connection, false);
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NODELAY,
@@ -356,6 +416,7 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = false;
}
+ */
}
#endif
@@ -404,6 +465,8 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
if (ret == header_len + buffer_len)
{
/* Response complete, set NOPUSH to off */
+ MHD_send_socket_state_nopush_ (connection, false, false);
+ /*
if (0 == setsockopt (s,
IPPROTO_TCP,
TCP_NOPUSH,
@@ -412,6 +475,7 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
{
connection->sk_tcp_nodelay_on = false;
}
+ */
}
errno = eno;
}
@@ -431,11 +495,262 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
* with this seems to be to mmap the file and write(2) as
* large a chunk as possible to the socket. Alternatively,
* use madvise(..., MADV_SEQUENTIAL). */
+
+#if defined(_MHD_HAVE_SENDFILE)
+/**
+ * Function for sending responses backed by file FD.
+ * A sendfile() wrapper which also performs cork/uncork
+ * operations.
+ *
+ * @param connection the #MHD_Connection structure
+ * @return actual number of bytes sent
+ */
ssize_t
-MHD_sendfile_on_connection_ (struct MHD_Connection *connection,
- const char *buffer,
- size_t buffer_size,
- enum MHD_SendSocketOptions options)
+MHD_sendfile_on_connection_ (struct MHD_Connection *connection)
{
- // TODO
+ // I'm looking for a version of sendfile_adapter() that *also* performs
+ // cork/uncork operations. Specifically, we want to make sure that the
+ // buffer is flushed after sendfile() is done (so setsockopt() behavior
+ // equivalent to MHD_SSO_NO_CORK), and _also_ of course update the
+ // setsockopt state, i.e. connection->sk_tcp_nodelay_on = true;
+
+ bool want_cork = false;
+ bool have_cork;
+ bool have_more;
+ bool use_corknopush;
+ bool using_tls = false;
+ ssize_t ret;
+ const int file_fd = connection->response->fd;
+ uint64_t left;
+ uint64_t offsetu64;
+#ifndef HAVE_SENDFILE64
+ const uint64_t max_off_t = (uint64_t)OFF_T_MAX;
+#else /* HAVE_SENDFILE64 */
+ const uint64_t max_off_t = (uint64_t)OFF64_T_MAX;
+#endif /* HAVE_SENDFILE64 */
+#ifdef MHD_LINUX_SOLARIS_SENDFILE
+#ifndef HAVE_SENDFILE64
+ off_t offset;
+#else /* HAVE_SENDFILE64 */
+ off64_t offset;
+#endif /* HAVE_SENDFILE64 */
+#endif /* MHD_LINUX_SOLARIS_SENDFILE */
+#ifdef HAVE_FREEBSD_SENDFILE
+ off_t sent_bytes;
+ int flags = 0;
+#endif
+#ifdef HAVE_DARWIN_SENDFILE
+ off_t len;
+#endif /* HAVE_DARWIN_SENDFILE */
+ const bool used_thr_p_c = (0 != (connection->daemon->options & MHD_USE_THREAD_PER_CONNECTION));
+ const size_t chunk_size = used_thr_p_c ? MHD_SENFILE_CHUNK_THR_P_C_ : MHD_SENFILE_CHUNK_;
+ size_t send_size = 0;
+ mhd_assert (MHD_resp_sender_sendfile == connection->resp_sender);
+
+ offsetu64 = connection->response_write_position + connection->response->fd_off;
+ left = connection->response->total_size - connection->response_write_position;
+ /* Do not allow system to stick sending on single fast connection:
+ * use 128KiB chunks (2MiB for thread-per-connection). */
+ send_size = (left > chunk_size) ? chunk_size : (size_t) left;
+ if (max_off_t < offsetu64)
+ { /* Retry to send with standard 'send()'. */
+ connection->resp_sender = MHD_resp_sender_std;
+ return MHD_ERR_AGAIN_;
+ }
+#ifdef MHD_LINUX_SOLARIS_SENDFILE
+#ifndef HAVE_SENDFILE64
+ offset = (off_t) offsetu64;
+ ret = sendfile (connection->socket_fd,
+ file_fd,
+ &offset,
+ send_size);
+#else /* HAVE_SENDFILE64 */
+ offset = (off64_t) offsetu64;
+ ret = sendfile64 (connection->socket_fd,
+ file_fd,
+ &offset,
+ send_size);
+#endif /* HAVE_SENDFILE64 */
+ if (0 > ret)
+ {
+ const int err = MHD_socket_get_error_();
+ if (MHD_SCKT_ERR_IS_EAGAIN_(err))
+ {
+#ifdef EPOLL_SUPPORT
+ /* EAGAIN --- no longer write-ready */
+ connection->epoll_state &= ~MHD_EPOLL_STATE_WRITE_READY;
+#endif /* EPOLL_SUPPORT */
+ return MHD_ERR_AGAIN_;
+ }
+ if (MHD_SCKT_ERR_IS_EINTR_ (err))
+ return MHD_ERR_AGAIN_;
+#ifdef HAVE_LINUX_SENDFILE
+ if (MHD_SCKT_ERR_IS_(err,
+ MHD_SCKT_EBADF_))
+ return MHD_ERR_BADF_;
+ /* sendfile() failed with EINVAL if mmap()-like operations are not
+ supported for FD or other 'unusual' errors occurred, so we should try
+ to fall back to 'SEND'; see also this thread for info on
+ odd libc/Linux behavior with sendfile:
+ http://lists.gnu.org/archive/html/libmicrohttpd/2011-02/msg00015.html */
+ connection->resp_sender = MHD_resp_sender_std;
+ return MHD_ERR_AGAIN_;
+#else /* HAVE_SOLARIS_SENDFILE */
+ if ( (EAFNOSUPPORT == err) ||
+ (EINVAL == err) ||
+ (EOPNOTSUPP == err) )
+ { /* Retry with standard file reader. */
+ connection->resp_sender = MHD_resp_sender_std;
+ return MHD_ERR_AGAIN_;
+ }
+ if ( (ENOTCONN == err) ||
+ (EPIPE == err) )
+ {
+ return MHD_ERR_CONNRESET_;
+ }
+ return MHD_ERR_BADF_; /* Fail hard */
+#endif /* HAVE_SOLARIS_SENDFILE */
+ }
+#ifdef EPOLL_SUPPORT
+ else if (send_size > (size_t)ret)
+ connection->epoll_state &= ~MHD_EPOLL_STATE_WRITE_READY;
+#endif /* EPOLL_SUPPORT */
+#elif defined(HAVE_FREEBSD_SENDFILE)
+#ifdef SF_FLAGS
+ flags = used_thr_p_c ?
+ freebsd_sendfile_flags_thd_p_c_ : freebsd_sendfile_flags_;
+#endif /* SF_FLAGS */
+ if (0 != sendfile (file_fd,
+ connection->socket_fd,
+ (off_t) offsetu64,
+ send_size,
+ NULL,
+ &sent_bytes,
+ flags))
+ {
+ const int err = MHD_socket_get_error_();
+ if (MHD_SCKT_ERR_IS_EAGAIN_(err) ||
+ MHD_SCKT_ERR_IS_EINTR_(err) ||
+ EBUSY == err)
+ {
+ mhd_assert (SSIZE_MAX >= sent_bytes);
+ if (0 != sent_bytes)
+ return (ssize_t)sent_bytes;
+
+ return MHD_ERR_AGAIN_;
+ }
+ /* Some unrecoverable error. Possibly file FD is not suitable
+ * for sendfile(). Retry with standard send(). */
+ connection->resp_sender = MHD_resp_sender_std;
+ return MHD_ERR_AGAIN_;
+ }
+ mhd_assert (0 < sent_bytes);
+ mhd_assert (SSIZE_MAX >= sent_bytes);
+ ret = (ssize_t)sent_bytes;
+#elif defined(HAVE_DARWIN_SENDFILE)
+ len = (off_t)send_size; /* chunk always fit */
+ if (0 != sendfile (file_fd,
+ connection->socket_fd,
+ (off_t) offsetu64,
+ &len,
+ NULL,
+ 0))
+ {
+ const int err = MHD_socket_get_error_();
+ if (MHD_SCKT_ERR_IS_EAGAIN_(err) ||
+ MHD_SCKT_ERR_IS_EINTR_(err))
+ {
+ mhd_assert (0 <= len);
+ mhd_assert (SSIZE_MAX >= len);
+ mhd_assert (send_size >= (size_t)len);
+ if (0 != len)
+ return (ssize_t)len;
+
+ return MHD_ERR_AGAIN_;
+ }
+ if (ENOTCONN == err ||
+ EPIPE == err)
+ return MHD_ERR_CONNRESET_;
+ if (ENOTSUP == err ||
+ EOPNOTSUPP == err)
+ { /* This file FD is not suitable for sendfile().
+ * Retry with standard send(). */
+ connection->resp_sender = MHD_resp_sender_std;
+ return MHD_ERR_AGAIN_;
+ }
+ return MHD_ERR_BADF_; /* Return hard error. */
+ }
+ mhd_assert (0 <= len);
+ mhd_assert (SSIZE_MAX >= len);
+ mhd_assert (send_size >= (size_t)len);
+ ret = (ssize_t)len;
+#endif /* HAVE_FREEBSD_SENDFILE */
+
+ /* ! could be avoided by redefining the variable. */
+ have_cork = ! connection->sk_tcp_nodelay_on;
+
+#ifdef MSG_MORE
+ have_more = true;
+#else
+ have_more = false;
+#endif
+
+#if TCP_NODELAY
+ use_corknopush = false;
+#elif TCP_CORK
+ use_corknopush = true;
+#elif TCP_NOPUSH
+ use_corknopush = true;
+#endif
+
+#ifdef HTTPS_SUPPORT
+ using_tls = (0 != (connection->daemon->options & MHD_USE_TLS));
+#endif
+
+#if TCP_CORK
+ /* When we have CORK, we can have NODELAY on the same system,
+ * at least since Linux 2.2 and both can be combined since
+ * Linux 2.5.71. For more details refer to tcp(7) on Linux.
+ * No other system in 2019-06 has TCP_CORK. */
+ if ((! using_tls) && (use_corknopush) && (have_cork && ! want_cork))
+ {
+ if (0 == setsockopt (connection->socket_fd,
+ IPPROTO_TCP,
+ TCP_CORK,
+ (const void *) &off_val,
+ sizeof (off_val)))
+ {
+ connection->sk_tcp_nodelay_on = true;
+ }
+ else if (0 == setsockopt (connection->socket_fd,
+ IPPROTO_TCP,
+ TCP_NODELAY,
+ (const void *) &on_val,
+ sizeof (on_val)))
+ {
+ connection->sk_tcp_nodelay_on = true;
+ }
+ }
+#elif TCP_NOPUSH
+ /* TCP_NOPUSH on FreeBSD is equal to cork on Linux, with the
+ * exception that we know that TCP_NOPUSH will definitely
+ * exist and we can disregard TCP_NODELAY unless requested. */
+ if ((! using_tls) && (use_corknopush) && (have_cork && ! want_cork))
+ {
+ MHD_send_socket_state_nopush_ (connection, true, false);
+ /*
+ if (0 == setsockopt (connection->socket_fd,
+ IPPROTO_TCP,
+ TCP_NOPUSH,
+ (const void *) &on_val,
+ sizeof (on_val)))
+ {
+ connection->sk_tcp_nodelay_on = false;
+ }
+ */
+ }
+#endif
+
+ return ret;
}
+#endif /* _MHD_HAVE_SENDFILE */
diff --git a/src/microhttpd/mhd_send.h b/src/microhttpd/mhd_send.h
@@ -74,4 +74,7 @@ MHD_send_on_connection2_ (struct MHD_Connection *connection,
size_t header_size,
const char *buffer,
size_t buffer_size);
+
+ssize_t
+MHD_sendfile_on_connection_ (struct MHD_Connection *connection);
#endif /* MHD_SEND_H */
