api: reduce syscall overhead in client and server

api/gr_api.h

@@ -174,6 +174,7 @@ enum gr_main_requests : uint32_t {
 	GR_LOG_LEVEL_SET,
 	GR_EVENT_SUBSCRIBE,
 	GR_EVENT_UNSUBSCRIBE,
+	GR_PING,
 };
 
 // Client handshake with API version negotiation.
@@ -242,4 +243,7 @@ struct gr_api_event {
 // Receive an event notification.
 // Caller must free(*event) after use.
 // Returns 0 on success, negative errno on failure.
-int gr_api_client_event_recv(const struct gr_api_client *, struct gr_api_event **);
+int gr_api_client_event_recv(struct gr_api_client *, struct gr_api_event **);
+
+// Send an arbitrary payload and receive it back echoed from the server.
+GR_REQ(GR_PING, struct gr_empty, struct gr_empty);

api/gr_api_client_impl.h

@@ -79,6 +79,7 @@ static void register_message(struct api_message *m) {
 
 #include <assert.h>
 #include <getopt.h>
+#include <poll.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -108,6 +109,9 @@ struct gr_api_client {
 	int sock_fd;
 	struct gr_hello_resp info;
 	STAILQ_HEAD(, response) responses;
+	size_t recv_buf_off;
+	size_t recv_buf_len;
+	char recv_buf[sizeof(struct gr_api_response) + GR_API_MAX_MSG_LEN];
 };
 
 const struct gr_hello_resp *gr_api_client_info(const struct gr_api_client *client) {
@@ -170,39 +174,43 @@ int gr_api_client_disconnect(struct gr_api_client *client) {
 	return ret;
 }
 
-static ssize_t send_all(const struct gr_api_client *c, const void *buf, size_t len) {
+static ssize_t recv_all(struct gr_api_client *c, void *buf, size_t len) {
 	size_t remaining = len;
-	const char *ptr = buf;
+	char *ptr = buf;
 	ssize_t n;
 
 	while (remaining > 0) {
-		n = send(c->sock_fd, ptr, remaining, MSG_NOSIGNAL);
-		if (n < 0)
-			return n;
-
-		ptr += n;
-		remaining -= n;
-	}
-
-	return len;
-}
+		if (c->recv_buf_len > 0) {
+			// First, consume any data present in the buffer.
+			n = remaining < c->recv_buf_len ? remaining : c->recv_buf_len;
+			memcpy(ptr, c->recv_buf + c->recv_buf_off, n);
+			c->recv_buf_off += n;
+			c->recv_buf_len -= n;
+			ptr += n;
+			remaining -= n;
+			continue;
+		}
 
-static ssize_t recv_all(const struct gr_api_client *c, void *buf, size_t len) {
-	size_t remaining = len;
-	char *ptr = buf;
-	ssize_t n;
+		// Then, receive as much data as possible in the buffer.
+		c->recv_buf_off = 0;
+		n = recv(c->sock_fd, c->recv_buf, sizeof(c->recv_buf), 0);
+		if (n > 0) {
+			c->recv_buf_len = n;
+			continue;
+		}
 
-	while (remaining > 0) {
-		n = recv(c->sock_fd, ptr, remaining, 0);
 		if (n == 0) {
 			errno = ECONNRESET;
 			return len - remaining;
 		} else if (n < 0) {
-			return n;
+			if (remaining < len && (errno == EAGAIN || errno == EWOULDBLOCK)) {
+				// Blocking mid-transmission, wait for socket to be readable again.
+				struct pollfd pfd = {.fd = c->sock_fd, .events = POLLIN};
+				poll(&pfd, 1, -1);
+				continue;
+			}
 		}
-
-		ptr += n;
-		remaining -= n;
+		return n;
 	}
 
 	return len;
@@ -224,12 +232,35 @@ long int gr_api_client_send(
 		.payload_len = tx_len,
 		.type = req_type,
 	};
-
-	if (send_all(client, &req, sizeof(req)) < 0)
-		return -errno;
-
-	if (tx_len > 0 && send_all(client, tx_data, tx_len) < 0)
-		return -errno;
+	struct iovec vec[2] = {{&req, sizeof(req)}, {(void *)tx_data, tx_len}};
+	struct msghdr msg = {.msg_iov = vec, .msg_iovlen = tx_len > 0 ? 2 : 1};
+	size_t total_len = vec[0].iov_len + vec[1].iov_len;
+	size_t remaining = total_len;
+
+	while (msg.msg_iovlen > 0) {
+		int n = sendmsg(client->sock_fd, &msg, MSG_NOSIGNAL);
+		if (n < 0) {
+			if (remaining < total_len && (errno == EAGAIN || errno == EWOULDBLOCK)) {
+				// Blocking mid-transmission, wait for socket to be writable again.
+				struct pollfd pfd = {.fd = client->sock_fd, .events = POLLOUT};
+				poll(&pfd, 1, -1);
+				continue;
+			}
+			return n;
+		}
+		remaining -= n;
+		while (n > 0) {
+			if ((int)msg.msg_iov->iov_len <= n) {
+				n -= msg.msg_iov->iov_len;
+				msg.msg_iov++;
+				msg.msg_iovlen--;
+			} else {
+				msg.msg_iov->iov_len -= n;
+				msg.msg_iov->iov_base = (char *)msg.msg_iov->iov_base + n;
+				n = 0;
+			}
+		}
+	}
 
 	return req.id;
 }
@@ -316,7 +347,7 @@ int gr_api_client_recv(
 	return -errno;
 }
 
-int gr_api_client_event_recv(const struct gr_api_client *c, struct gr_api_event **event) {
+int gr_api_client_event_recv(struct gr_api_client *c, struct gr_api_event **event) {
 	const struct api_message *m;
 	struct gr_api_event header;
 

frr/zebra_dplane_grout.c

@@ -13,6 +13,7 @@
 #include "log_grout.h"
 #include "rt_grout.h"
 
+#include <fcntl.h>
 #include <lib/bitfield.h>
 #include <lib/frr_pthread.h>
 #include <lib/libfrr.h>
@@ -702,6 +703,8 @@ static void dplane_grout_connect(struct event *) {
 		return;
 	}
 
+	fcntl(grout_ctx.dplane_notifs->sock_fd, F_SETFL, O_NONBLOCK);
+
 	event_add_read(
 		dg_master,
 		dplane_read_notifications,
@@ -731,6 +734,8 @@ static void zebra_grout_connect(struct event *) {
 		return;
 	}
 
+	fcntl(grout_ctx.zebra_notifs->sock_fd, F_SETFL, O_NONBLOCK);
+
 	event_add_read(
 		zrouter.master,
 		zebra_read_notifications,
@@ -773,6 +778,16 @@ static void dplane_read_notifications(struct event *event) {
 	bool new = false;
 
 	if (gr_api_client_event_recv(grout_ctx.dplane_notifs, &gr_e) < 0 || gr_e == NULL) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK) {
+			event_add_read(
+				dg_master,
+				dplane_read_notifications,
+				NULL,
+				grout_ctx.dplane_notifs->sock_fd,
+				&grout_ctx.dg_t_dplane_update
+			);
+			return;
+		}
 		gr_api_client_disconnect(grout_ctx.dplane_notifs);
 		grout_ctx.dplane_notifs = NULL;
 		gr_api_client_disconnect(grout_ctx.client);
@@ -820,12 +835,8 @@ static void dplane_read_notifications(struct event *event) {
 
 	free(gr_e);
 
-	event_add_read(
-		dg_master,
-		dplane_read_notifications,
-		NULL,
-		grout_ctx.dplane_notifs->sock_fd,
-		&grout_ctx.dg_t_dplane_update
+	event_add_event(
+		dg_master, dplane_read_notifications, NULL, 0, &grout_ctx.dg_t_dplane_update
 	);
 }
 
@@ -834,6 +845,16 @@ static void zebra_read_notifications(struct event *event) {
 	bool new = false;
 
 	if (gr_api_client_event_recv(grout_ctx.zebra_notifs, &gr_e) < 0 || gr_e == NULL) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK) {
+			event_add_read(
+				zrouter.master,
+				zebra_read_notifications,
+				NULL,
+				grout_ctx.zebra_notifs->sock_fd,
+				&grout_ctx.dg_t_zebra_update
+			);
+			return;
+		}
 		gr_api_client_disconnect(grout_ctx.zebra_notifs);
 		grout_ctx.zebra_notifs = NULL;
 		event_add_timer(
@@ -868,12 +889,8 @@ static void zebra_read_notifications(struct event *event) {
 
 	free(gr_e);
 
-	event_add_read(
-		zrouter.master,
-		zebra_read_notifications,
-		NULL,
-		grout_ctx.zebra_notifs->sock_fd,
-		&grout_ctx.dg_t_zebra_update
+	event_add_event(
+		zrouter.master, zebra_read_notifications, NULL, 0, &grout_ctx.dg_t_zebra_update
 	);
 }
 

main/api.c

@@ -225,6 +225,17 @@ static struct api_out hello(const void *request, struct api_ctx *) {
 	return api_out(0, sizeof(*resp), resp);
 }
 
+static struct api_out ping(const void *request, struct api_ctx *ctx) {
+	uint32_t len = ctx->header.payload_len;
+	if (len == 0)
+		return api_out(0, 0, NULL);
+	void *resp = malloc(len);
+	if (resp == NULL)
+		return api_out(ENOMEM, 0, NULL);
+	memcpy(resp, request, len);
+	return api_out(0, len, resp);
+}
+
 static void disconnect_client(struct api_ctx *ctx) {
 	assert(ctx != NULL);
 	assert(ctx->bev != NULL);
@@ -348,16 +359,8 @@ static void read_cb(struct bufferevent *bev, void *priv) {
 			LOG(ERR, "failed to write payload");
 	}
 
-	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
-
 	free(req_payload);
 	free(out.payload);
-
-	if (evbuffer_get_length(input) >= sizeof(ctx->header)) {
-		// More data is available in the input buffer.
-		// Force read_cb to be invoked again when possible.
-		bufferevent_flush(bev, EV_READ, BEV_NORMAL);
-	}
 	return;
 
 close:
@@ -473,4 +476,5 @@ RTE_INIT(init) {
 	api_handler(GR_EVENT_SUBSCRIBE, subscribe);
 	api_handler(GR_EVENT_UNSUBSCRIBE, unsubscribe);
 	api_handler(GR_HELLO, hello);
+	api_handler(GR_PING, ping);
 }

meson.build

@@ -206,6 +206,12 @@ executable(
   install: false,
 )
 
+executable(
+  'ping_perf', files('smoke/ping_perf.c') + grout_header,
+  include_directories: api_inc,
+  install: false,
+)
+
 # docs/ must come after grcli_exe since man pages are generated using grcli --man
 subdir('docs')
 

smoke/ping_perf.c

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+// clang-format off
+#include <gr_api_client_impl.h>
+// clang-format on
+
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int ping(struct gr_api_client *c, uint32_t count, size_t payload_len) {
+	void *buf = NULL;
+
+	if (payload_len > 0) {
+		buf = calloc(1, payload_len);
+		if (buf == NULL)
+			return -1;
+	}
+
+	for (uint32_t i = 0; i < count; i++) {
+		void *resp = NULL;
+		if (gr_api_client_send_recv(c, GR_PING, payload_len, buf, &resp) < 0) {
+			perror("GR_PING");
+			free(buf);
+			return -1;
+		}
+		free(resp);
+	}
+
+	free(buf);
+	return 0;
+}
+
+static void usage(const char *prog) {
+	fprintf(stderr, "Usage: %s [-s SOCK] [-n COUNT] [-l LEN]\n", prog);
+	fprintf(stderr, "  -s SOCK    API socket path (default: $GROUT_SOCK_PATH)\n");
+	fprintf(stderr, "  -n COUNT   Number of ping calls (default: 10000)\n");
+	fprintf(stderr, "  -l LEN     Payload length in bytes (default: 0)\n");
+}
+
+int main(int argc, char **argv) {
+	const char *sock_path = getenv("GROUT_SOCK_PATH");
+	unsigned int count = 10000;
+	struct gr_api_client *c;
+	size_t payload_len = 0;
+	gr_clock_ns_t time;
+	float duration;
+	int ret;
+	int o;
+
+	while ((o = getopt(argc, argv, "s:n:l:h")) != -1) {
+		switch (o) {
+		case 's':
+			sock_path = optarg;
+			break;
+		case 'n':
+			count = strtoul(optarg, NULL, 10);
+			break;
+		case 'l':
+			payload_len = strtoul(optarg, NULL, 10);
+			break;
+		case 'h':
+		default:
+			usage(argv[0]);
+			return o == 'h' ? EXIT_SUCCESS : EXIT_FAILURE;
+		}
+	}
+	if (sock_path == NULL)
+		sock_path = GR_DEFAULT_SOCK_PATH;
+
+	c = gr_api_client_connect(sock_path);
+	if (c == NULL) {
+		perror("gr_api_client_connect");
+		return EXIT_FAILURE;
+	}
+
+	printf("performing %u ping calls (payload %zu bytes)\n", count, payload_len);
+
+	time = gr_clock_ns();
+	ret = ping(c, count, payload_len);
+	duration = (float)(gr_clock_ns() - time) / (float)GR_NS_PER_S;
+
+	printf("total time: %.3f s (%.1f calls/s)\n", duration, (float)count / duration);
+
+	gr_api_client_disconnect(c);
+
+	return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
+}