From 07a4903396ac9dc3f9623718c8b308dee3916633 Mon Sep 17 00:00:00 2001 From: Timothy Copeland Date: Tue, 11 Aug 2026 13:37:25 +1000 Subject: [PATCH] Set TCP_NODELAY on sockets to reduce latency --- src/listen.c | 7 +++++++ src/proxy.c | 7 +++++++ test/test_listen.c | 34 +++++++++++++++++++++++++++++++++- test/test_proxy.c | 29 ++++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/listen.c b/src/listen.c index 175911d..56e4f64 100644 --- a/src/listen.c +++ b/src/listen.c @@ -224,6 +224,13 @@ int bind_listen_sock(listen_t *l, const char *host, "Couldn't enable tcp keepcnt on listen socket", TPX_ERR_ERRNO); + opt = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, + &opt, sizeof(opt)) == -1) + log_system_err(LL_WARN, + "Couldn't enable tcp nodelay on listen socket", + TPX_ERR_ERRNO); + opt = 0; if (lp->ai_family == AF_INET6 && setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, diff --git a/src/proxy.c b/src/proxy.c index a17b2bc..ccbc41b 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -163,6 +163,13 @@ int create_connect(proxy_t *proxy, int keepidle, int keepintvl, int keepcnt) { log_proxy(LL_WARN, proxy, "ioerror", "Couldn't set keepcnt on connect socket", strerror(errno)); + opt = 1; + if (setsockopt(conn_sock, IPPROTO_TCP, TCP_NODELAY, + &opt, sizeof(opt)) == -1) + log_proxy(LL_WARN, proxy, "ioerror", + "Couldn't set tcp nodelay on connect socket", + strerror(errno)); + return conn_sock; } diff --git a/test/test_listen.c b/test/test_listen.c index 382ac63..c2f4d48 100644 --- a/test/test_listen.c +++ b/test/test_listen.c @@ -830,7 +830,8 @@ static void bind_listen_sock_sets_keepalive(void **state) { Four queued returns because the mock queue applies to setsockopt in call order: SO_REUSEADDR, SO_REUSEPORT and SO_KEEPALIVE succeed, TCP_KEEPIDLE - fails, and the two after it go to the real syscall. */ + fails, and the three after it, TCP_KEEPINTVL, TCP_KEEPCNT and TCP_NODELAY, + go to the real syscall. */ static void bind_listen_sock_survives_a_refused_keepidle(void **state) { (void)state; listen_t l; @@ -873,6 +874,35 @@ static void bind_listen_sock_survives_a_refused_keepalive(void **state) { close(fd); } +/* Set on the listener rather than per accepted socket because Linux copies + TCP_NODELAY onto everything accept() returns, the same way it copies + SO_KEEPALIVE and the TCP_KEEP* values, measured on 6.18.41. So this one call + is what decides whether Nagle is off on every client leg the worker ever + serves, and reading it back off the listening socket is not quite the claim: + the value the kernel reports here is the one it will clone. + + What it costs when it is missing, measured rather than assumed: with 1000 + held connections each sending 200 bytes on a shared 33 ms tick, p99 was + 42 ms against nginx's 0.86, because a message small enough for Nagle to + hold waits for an ACK that the peer has already decided to delay. The + backend leg is the same argument and lives in create_connect(). */ +static void bind_listen_sock_sets_nodelay(void **state) { + (void)state; + listen_t l; + memset(&l, 0, sizeof(l)); + + int fd = bind_listen_sock(&l, "127.0.0.1", 0, 60, 10, 3); + assert_int_not_equal(fd, -1); + + int on = -1; + socklen_t len = sizeof(on); + assert_int_equal(getsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &on, &len), 0); + + close(fd); + assert_true(opt_was_set(IPPROTO_TCP, TCP_NODELAY)); + assert_int_equal(on, 1); +} + /* Port 0 rather than a hardcoded port: the old suite bound 47239 and left a comment accepting defeat if anything else had it. The kernel will hand out a free ephemeral port every time, so there is nothing to lose a race with. */ @@ -1083,6 +1113,8 @@ int main(void) { reset_recorders), cmocka_unit_test_setup(bind_listen_sock_sets_keepalive, reset_recorders), + cmocka_unit_test_setup(bind_listen_sock_sets_nodelay, + reset_recorders), cmocka_unit_test_setup(bind_listen_sock_fills_in_the_listen_address, reset_recorders), cmocka_unit_test_setup( diff --git a/test/test_proxy.c b/test/test_proxy.c index b7a8ce4..39abe88 100644 --- a/test/test_proxy.c +++ b/test/test_proxy.c @@ -551,13 +551,36 @@ static void create_connect_configures_keepalive_on_the_backend(void **state) { assert_false(was_closed(50)); } +/* Nagle on the backend leg costs the same delayed-ACK stall as on the client + leg, so both are turned off and each is asserted on its own: the two calls + are in different files and one can be added without the other, which reads + as "latency is fixed" while half the path still coalesces. + + Unlike the client leg, this one cannot ride on the listening socket, because + the kernel only clones options onto sockets it accepts and this one is + connecting outwards. */ +static void create_connect_disables_nagle_on_the_backend(void **state) { + (void)state; + proxy_t p; + memset(&p, 0, sizeof(p)); + p.listener = make_listener(); + + will_return(__wrap_socket, 50); + will_return(__wrap_fcntl, 0); + will_return(__wrap_fcntl, 0); + + assert_int_equal(create_connect(&p, 60, 10, 3), 50); + assert_true(opt_set_to(IPPROTO_TCP, TCP_NODELAY, 1)); + assert_false(was_closed(50)); +} + /* Keepalive tuning is a refinement, not a precondition. A kernel that refuses TCP_KEEPIDLE leaves a connection that still works and still has keepalive, just on the tcp_keepalive_* sysctl defaults instead. Failing the connection would convert a rejected tuning knob into a dropped client, so the socket is handed back and only the failure is logged. - The mock queue is armed for all four options, not just the failing one: + The mock queue is armed for all five options, not just the failing one: arming only the second would make the first call consume it and the test would be measuring a refused SO_KEEPALIVE while claiming otherwise. */ static void create_connect_survives_a_refused_keepalive_option(void **state) { @@ -573,6 +596,7 @@ static void create_connect_survives_a_refused_keepalive_option(void **state) { will_return(__wrap_setsockopt, -1); // TCP_KEEPIDLE refused will_return(__wrap_setsockopt, 0); // TCP_KEEPINTVL will_return(__wrap_setsockopt, 0); // TCP_KEEPCNT + will_return(__wrap_setsockopt, 0); // TCP_NODELAY assert_int_equal(create_connect(&p, 60, 10, 3), 50); assert_false(was_closed(50)); @@ -1661,6 +1685,9 @@ int main(void) { reset_recorders), cmocka_unit_test_setup(create_connect_closes_socket_when_setfl_fails, reset_recorders), + cmocka_unit_test_setup( + create_connect_disables_nagle_on_the_backend, + reset_recorders), cmocka_unit_test_setup( create_connect_configures_keepalive_on_the_backend, reset_recorders),