ip,ip6,frr: flush routes like zebra on address and iface teardown

docs/grout.8.scdoc

@@ -144,6 +144,25 @@ enable and _0_, _false_, _off_, _no_ to disable.
 	is required when _net.ipv4.conf.all.rp_filter_ is set to _1_ (strict) in
 	the network namespace where grout is running.
 
+*GROUT_FLUSH_ROUTES_ON_IFACE_DOWN*
+	When set to _1_, _true_, _on_, or _yes_ (case insensitive), delete the
+	routes going out of an interface when it goes administratively down,
+	like zebra and Linux do. When disabled, these routes are kept with an
+	unresolved nexthop and start working again when the interface goes back
+	up. Can also be changed at runtime with *grcli interface config set
+	flush-routes-on-iface-down*.
+
+	Default: _false_.
+
+*GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN*
+	When set to _1_, _true_, _on_, or _yes_ (case insensitive), do not emit
+	route events for the routes deleted along with an interface. Only useful
+	for a control plane which already evicts those routes when it receives
+	the interface event. Can also be changed at runtime with *grcli
+	interface config set skip-route-events-on-iface-down*.
+
+	Default: _false_.
+
 ## Sizing
 
 These variables control the size of internal data structures. Reducing them

frr/zebra_dplane_grout.c

@@ -713,6 +713,14 @@ static void dplane_grout_connect(struct event *) {
 		&grout_ctx.dg_t_dplane_update
 	);
 
+	// redone on every connection: a restarted grout comes back with its defaults
+	struct gr_iface_config_set_req req = {
+		.flush_routes_on_iface_down = true,
+		.skip_route_events_on_iface_down = true,
+		.set_attrs = GR_IFACE_CONFIG_SET_FLUSH_ROUTES | GR_IFACE_CONFIG_SET_SKIP_EVENTS,
+	};
+	grout_client_send_recv(GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL);
+
 	gr_log_notice("connected, monitoring iface/ip events");
 }
 

main/config.c

@@ -274,6 +274,8 @@ int config_parse(int argc, char **argv) {
 	ENV_BOOL(log_packets, "GROUT_TRACE_PACKETS", false);
 	ENV_BOOL(override_default_route, "GROUT_OVERRIDE_DEFAULT_ROUTE", false);
 	ENV_BOOL(override_rp_filter, "GROUT_OVERRIDE_RP_FILTER", false);
+	ENV_BOOL(flush_routes_on_iface_down, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN", false);
+	ENV_BOOL(skip_route_events_on_iface_down, "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN", false);
 	ENV_INT(max_ifaces, "GROUT_MAX_IFACES", 1024, 16, UINT16_MAX);
 	ENV_INT(mempool_chunk_size, "GROUT_MEMPOOL_CHUNK_SIZE", (1 << 16) - 1, 255, (1 << 20) - 1);
 	ENV_INT(max_nexthops, "GROUT_MAX_NEXTHOPS", 1 << 17, 64, 1 << 24);
@@ -357,6 +359,10 @@ void config_print(void) {
 	LOG(INFO, "GROUT_TRACE_PACKETS=%hhu", gr_config.log_packets);
 	LOG(INFO, "GROUT_OVERRIDE_DEFAULT_ROUTE=%hhu", gr_config.override_default_route);
 	LOG(INFO, "GROUT_OVERRIDE_RP_FILTER=%hhu", gr_config.override_rp_filter);
+	LOG(INFO, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=%hhu", gr_config.flush_routes_on_iface_down);
+	LOG(INFO,
+	    "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=%hhu",
+	    gr_config.skip_route_events_on_iface_down);
 	LOG(INFO, "GROUT_MAX_IFACES=%u", gr_config.max_ifaces);
 	LOG(INFO, "GROUT_MEMPOOL_CHUNK_SIZE=%u", gr_config.mempool_chunk_size);
 	LOG(INFO, "GROUT_MAX_NEXTHOPS=%u", gr_config.max_nexthops);

main/config.h

@@ -25,6 +25,8 @@ struct gr_config {
 	bool log_packets;
 	bool override_default_route;
 	bool override_rp_filter;
+	bool flush_routes_on_iface_down;
+	bool skip_route_events_on_iface_down;
 	vec char **eal_extra_args;
 	cpu_set_t control_cpus; // control plane threads allowed CPUs
 	cpu_set_t datapath_cpus; // datapath threads allowed CPUs

main/grout.default

@@ -40,6 +40,16 @@ GROUT_OVERRIDE_RP_FILTER=true
 # Print all ingress/egress packets (default: false).
 #GROUT_TRACE_PACKETS=false
 
+# Delete the routes of an interface when it goes administratively down, like
+# zebra and Linux do. Off by default: the routes are kept with an unresolved
+# nexthop and come back when the interface goes up again.
+#GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=false
+
+# Do not emit route events for the routes deleted with an interface. Only
+# useful for a control plane which already evicts them on the interface
+# event (default: false).
+#GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=false
+
 # Maximum Transmission Unit (default: 1800).
 #GROUT_MAX_MTU=1800
 

modules/dhcp/control/client.c

@@ -198,7 +198,7 @@ static void dhcp_expire_callback(evutil_socket_t, short, void *arg) {
 	if (client->offered_ip != 0 && client->prefixlen != 0)
 		addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 	if (client->router_ip != 0)
-		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 	client->state = DHCP_STATE_INIT;
 	client->offered_ip = 0;
@@ -370,7 +370,7 @@ void dhcp_input_cb(void *obj, uintptr_t, const struct control_queue_drain *drain
 			if (client->offered_ip != 0 && client->prefixlen != 0)
 				addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 			if (client->router_ip != 0)
-				rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+				rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 			client->state = DHCP_STATE_INIT;
 			client->offered_ip = 0;
@@ -454,7 +454,7 @@ static int dhcp_stop(uint16_t iface_id) {
 	if (client->offered_ip != 0 && client->prefixlen != 0)
 		addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 	if (client->router_ip != 0)
-		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 	dhcp_cancel_timers(client);
 

modules/infra/api/gr_infra.h

@@ -253,6 +253,8 @@ enum gr_infra_requests : uint32_t {
 	GR_IFACE_MAC_DEL,
 	GR_IFACE_MAC_LIST,
 	GR_IFACE_MAC_SET,
+	GR_IFACE_CONFIG_GET,
+	GR_IFACE_CONFIG_SET,
 };
 
 enum gr_infra_events : uint32_t {
@@ -364,6 +366,29 @@ struct gr_iface_set_req {
 
 GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty);
 
+// Get the interface subsystem configuration.
+struct gr_iface_config_get_resp {
+	bool flush_routes_on_iface_down;
+	bool skip_route_events_on_iface_down;
+};
+
+GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp);
+
+// Interface subsystem configuration attribute flags.
+#define GR_IFACE_CONFIG_SET_FLUSH_ROUTES GR_BIT64(0)
+#define GR_IFACE_CONFIG_SET_SKIP_EVENTS GR_BIT64(1)
+
+// Change the interface subsystem configuration.
+struct gr_iface_config_set_req {
+	// Delete the routes of an interface when it goes administratively down.
+	bool flush_routes_on_iface_down;
+	// Do not emit route events for the routes deleted with an interface.
+	bool skip_route_events_on_iface_down;
+	uint64_t set_attrs; // Bit mask of GR_IFACE_CONFIG_SET_*.
+};
+
+GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty);
+
 // Get interface statistics.
 struct gr_iface_stats {
 	uint16_t iface_id;

modules/infra/api/iface.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "config.h"
 #include "event.h"
 #include "iface.h"
 #include "metrics.h"
@@ -326,6 +327,28 @@ static void iface_metrics_collect(struct metrics_writer *w) {
 	}
 }
 
+static struct api_out iface_config_get(const void *, struct api_ctx *) {
+	struct gr_iface_config_get_resp *resp = calloc(1, sizeof(*resp));
+	if (resp == NULL)
+		return api_out(ENOMEM, 0, NULL);
+
+	resp->flush_routes_on_iface_down = gr_config.flush_routes_on_iface_down;
+	resp->skip_route_events_on_iface_down = gr_config.skip_route_events_on_iface_down;
+
+	return api_out(0, sizeof(*resp), resp);
+}
+
+static struct api_out iface_config_set(const void *request, struct api_ctx *) {
+	const struct gr_iface_config_set_req *req = request;
+
+	if (req->set_attrs & GR_IFACE_CONFIG_SET_FLUSH_ROUTES)
+		gr_config.flush_routes_on_iface_down = req->flush_routes_on_iface_down;
+	if (req->set_attrs & GR_IFACE_CONFIG_SET_SKIP_EVENTS)
+		gr_config.skip_route_events_on_iface_down = req->skip_route_events_on_iface_down;
+
+	return api_out(0, 0, NULL);
+}
+
 static struct metrics_collector iface_collector = {
 	.name = "iface",
 	.collect = iface_metrics_collect,
@@ -341,6 +364,8 @@ RTE_INIT(infra_api_init) {
 	api_handler(GR_IFACE_MAC_LIST, iface_mac_list);
 	api_handler(GR_IFACE_MAC_SET, iface_mac_set);
 	api_handler(GR_IFACE_SET, iface_set);
+	api_handler(GR_IFACE_CONFIG_GET, iface_config_get);
+	api_handler(GR_IFACE_CONFIG_SET, iface_config_set);
 	event_serializer(GR_EVENT_IFACE_ADD, iface_event_serialize);
 	event_serializer(GR_EVENT_IFACE_POST_ADD, iface_event_serialize);
 	event_serializer(GR_EVENT_IFACE_PRE_REMOVE, iface_event_serialize);

modules/infra/api/nexthop.c

@@ -67,7 +67,7 @@ static struct api_out nh_del(const void *request, struct api_ctx *) {
 			return api_out(EBUSY, 0, NULL);
 	}
 
-	nexthop_routes_cleanup(nh);
+	nexthop_routes_cleanup(nh, true);
 	// The nexthop *may* still have one ref_count when it has been created
 	// manually from the API (see nh_add()). Implicit nexthops created when
 	// creating a gateway route will not have that extra ref_count.
@@ -142,7 +142,7 @@ static struct api_out nh_flush(const void *request, struct api_ctx *) {
 			if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS)
 				continue;
 		}
-		nexthop_routes_cleanup(nh);
+		nexthop_routes_cleanup(nh, true);
 		while (nh->ref_count > 0)
 			nexthop_decref(nh);
 	}

modules/infra/cli/cli_iface.h

@@ -72,6 +72,8 @@ int arg_iface(
 	CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("add", "Create an interface."))
 #define INTERFACE_SET_CTX(root)                                                                    \
 	CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("set", "Modify an existing interface."))
+#define INTERFACE_CONFIG_CTX(root)                                                                 \
+	CLI_CONTEXT(INTERFACE_CTX(root), CTX_ARG("config", "Interface subsystem configuration."))
 
 #define IFACE_ATTRS_CMD                                                                            \
 	"(up|down),(promisc PROMISC),(neigh_snoop NEIGH_SNOOP),(mtu MTU),"                         \

modules/infra/cli/iface.c

@@ -547,6 +547,57 @@ static cmd_status_t iface_show(struct gr_api_client *c, const struct ec_pnode *p
 	return CMD_SUCCESS;
 }
 
+static cmd_status_t iface_config_show(struct gr_api_client *c, const struct ec_pnode *) {
+	const struct gr_iface_config_get_resp *resp;
+	void *resp_ptr = NULL;
+
+	if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_GET, 0, NULL, &resp_ptr) < 0)
+		return CMD_ERROR;
+
+	resp = resp_ptr;
+	struct gr_object *o = gr_object_new(NULL);
+	gr_object_field(
+		o,
+		"flush_routes_on_iface_down",
+		GR_DISP_BOOL,
+		"%s",
+		resp->flush_routes_on_iface_down ? "true" : "false"
+	);
+	gr_object_field(
+		o,
+		"skip_route_events_on_iface_down",
+		GR_DISP_BOOL,
+		"%s",
+		resp->skip_route_events_on_iface_down ? "true" : "false"
+	);
+	gr_object_free(o);
+	free(resp_ptr);
+
+	return CMD_SUCCESS;
+}
+
+static cmd_status_t iface_config_set(struct gr_api_client *c, const struct ec_pnode *p) {
+	struct gr_iface_config_set_req req = {0};
+	const char *flush, *skip;
+
+	flush = arg_str(p, "FLUSH");
+	if (flush != NULL) {
+		req.flush_routes_on_iface_down = strcmp(flush, "on") == 0;
+		req.set_attrs |= GR_IFACE_CONFIG_SET_FLUSH_ROUTES;
+	}
+
+	skip = arg_str(p, "SKIP");
+	if (skip != NULL) {
+		req.skip_route_events_on_iface_down = strcmp(skip, "on") == 0;
+		req.set_attrs |= GR_IFACE_CONFIG_SET_SKIP_EVENTS;
+	}
+
+	if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0)
+		return CMD_ERROR;
+
+	return CMD_SUCCESS;
+}
+
 static int ctx_init(struct ec_node *root) {
 	int ret;
 
@@ -556,6 +607,34 @@ static int ctx_init(struct ec_node *root) {
 	if (INTERFACE_SET_CTX(root) == NULL)
 		return -1;
 
+	ret = CLI_COMMAND(
+		INTERFACE_CONFIG_CTX(root),
+		"set (flush-routes-on-iface-down FLUSH),(skip-route-events-on-iface-down SKIP)",
+		iface_config_set,
+		"Change the interface subsystem configuration.",
+		with_help(
+			"Delete the routes going out of an interface when it goes "
+			"administratively down.",
+			EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off"))
+		),
+		with_help(
+			"Do not emit route events for the routes deleted along with an "
+			"interface.",
+			EC_NODE_OR("SKIP", ec_node_str("", "on"), ec_node_str("", "off"))
+		)
+	);
+	if (ret < 0)
+		return ret;
+
+	ret = CLI_COMMAND(
+		INTERFACE_CONFIG_CTX(root),
+		"[show]",
+		iface_config_show,
+		"Show the interface subsystem configuration."
+	);
+	if (ret < 0)
+		return ret;
+
 	ret = CLI_COMMAND(
 		INTERFACE_CTX(root),
 		"del NAME",

modules/infra/control/l3_nexthop.c

@@ -142,12 +142,12 @@ static int l3_reconfig(const struct gr_nexthop_config *c) {
 	return 0;
 }
 
-void nexthop_routes_cleanup(struct nexthop *nh) {
+void nexthop_routes_cleanup(struct nexthop *nh, bool notify) {
 	const struct nexthop_af_ops *ops;
 	for (unsigned i = 0; i < ARRAY_DIM(af_ops); i++) {
 		ops = af_ops[i];
 		if (ops != NULL)
-			ops->cleanup_routes(nh);
+			ops->cleanup_routes(nh, notify);
 	}
 }
 

modules/infra/control/nexthop.c

@@ -479,7 +479,7 @@ static void nh_cleanup_interface_cb(struct nexthop *nh, void *priv) {
 			if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS)
 				return; // addresses are cleaned per address family
 		}
-		nexthop_routes_cleanup(nh);
+		nexthop_routes_cleanup(nh, !gr_config.skip_route_events_on_iface_down);
 		while (nh->ref_count)
 			nexthop_decref(nh);
 	}

modules/infra/control/nexthop.h

@@ -128,7 +128,7 @@ struct gr_nexthop *nexthop_to_api(const struct nexthop *, size_t *len);
 int nexthop_serialize(const void *obj, void **buf);
 
 // Clean all routes that reference a given nexthop.
-void nexthop_routes_cleanup(struct nexthop *);
+void nexthop_routes_cleanup(struct nexthop *, bool notify);
 
 // Increment the reference counter of a nexthop.
 void nexthop_incref(struct nexthop *);
@@ -156,7 +156,7 @@ struct nexthop_af_ops {
 	// Callback that will be invoked when a nexthop needs to be refreshed by sending a probe.
 	int (*solicit)(struct nexthop *);
 	// Callback that will be invoked to delete all routes which reference a given nexthop.
-	void (*cleanup_routes)(struct nexthop *);
+	void (*cleanup_routes)(struct nexthop *, bool notify);
 	// Callback invoked by resolve() to flush held packets when the nexthop becomes
 	// reachable.
 	int (*resubmit)(struct rte_mbuf *, struct nexthop *);

modules/ip/control/address.c

@@ -197,13 +197,16 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) {
 		}
 	);
 
-	nexthop_routes_cleanup(nh);
+	nexthop_routes_cleanup(nh, true);
 	while (nh->ref_count > 0)
 		nexthop_decref(nh);
 
 	vec_del(addrs->nh, i);
-	if (vec_len(addrs->nh) == 0)
+	if (vec_len(addrs->nh) == 0) {
 		vec_free(addrs->nh);
+		// no address left to send ARP requests from
+		rib4_cleanup_iface(iface_id, true);
+	}
 
 	iface = iface_from_id(iface_id);
 	if (iface && iface->cp_id != 0) {

modules/ip/control/ip4.h

@@ -34,8 +34,16 @@ int rib4_insert(
 	gr_nh_origin_t origin,
 	struct nexthop *nh
 );
-int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type);
-void rib4_cleanup(struct nexthop *);
+int rib4_delete(
+	uint16_t vrf_id,
+	ip4_addr_t ip,
+	uint8_t prefixlen,
+	gr_nh_type_t nh_type,
+	bool notify
+);
+void rib4_cleanup(struct nexthop *, bool notify);
+// delete the routes of an interface, except those owned by a routing daemon
+void rib4_cleanup_iface(uint16_t iface_id, bool notify);
 
 typedef int (*rib4_iter_cb_t)(
 	uint16_t vrf_id,