From 6cb950dbcddd4890fce427af28f7d1f8efcbca0a Mon Sep 17 00:00:00 2001 From: Maxime Leroy Date: Mon, 24 Aug 2026 12:06:23 +0200 Subject: [PATCH 1/5] ip: flush interface routes when its last address is deleted Without any IPv4 address left on an interface, ARP requests can no longer be sent from it: the nexthops reachable through that interface can never be resolved again, and the routes using them are dead weight. Linux flushes them (fib_del_ifaddr) and control planes rely on it. Since FRR 10.7, zebra reacts to the deletion of the last IPv4 address of an interface by dropping such routes from its RIB on its own, without sending anything to the dataplane, which left grout and zebra with two different views for good. Flush them in grout instead, and notify as usual so that zebra follows on every FRR version. Routes installed by a routing daemon are left alone, their owner reevaluates them when the nexthop stops resolving. This also means a lost DHCP lease now removes the IPv4 routes of the interface, as it does on Linux. IPv6 is left untouched: on-link information does not come from the configured addresses there, and the link-local address remains available to source neighbor solicitations. Signed-off-by: Maxime Leroy Reviewed-by: Robin Jarry --- modules/ip/control/address.c | 5 +- modules/ip/control/ip4.h | 2 + modules/ip/control/route.c | 44 +++++++++--- smoke/addr_del_kernel_route_frr_test.sh | 95 +++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 11 deletions(-) create mode 100755 smoke/addr_del_kernel_route_frr_test.sh diff --git a/modules/ip/control/address.c b/modules/ip/control/address.c index 2720e5dc0..56a260a6d 100644 --- a/modules/ip/control/address.c +++ b/modules/ip/control/address.c @@ -202,8 +202,11 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) { nexthop_decref(nh); vec_del(addrs->nh, i); - if (vec_len(addrs->nh) == 0) + if (vec_len(addrs->nh) == 0) { vec_free(addrs->nh); + // no address left to send ARP requests from + rib4_cleanup_iface(iface_id); + } iface = iface_from_id(iface_id); if (iface && iface->cp_id != 0) { diff --git a/modules/ip/control/ip4.h b/modules/ip/control/ip4.h index 53daf6a97..247d22d54 100644 --- a/modules/ip/control/ip4.h +++ b/modules/ip/control/ip4.h @@ -36,6 +36,8 @@ int rib4_insert( ); int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type); void rib4_cleanup(struct nexthop *); +// delete the routes of an interface, except those owned by a routing daemon +void rib4_cleanup_iface(uint16_t iface_id); typedef int (*rib4_iter_cb_t)( uint16_t vrf_id, diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index 8846d5662..822b78c08 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -537,6 +537,7 @@ struct rib4_cleanup_entry { struct rib4_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib4_cleanup_entry *entries; }; @@ -544,12 +545,21 @@ static int rib4_cleanup_cb( uint16_t vrf_id, ip4_addr_t ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib4_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib4_cleanup_entry entry = { .vrf_id = vrf_id, .ip = ip, @@ -561,21 +571,35 @@ static int rib4_cleanup_cb( return 0; } -void rib4_cleanup(struct nexthop *nh) { - struct rib4_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx) { struct rib4_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib4_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib4_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) + vec_foreach_ref (struct rib4_cleanup_entry *r, ctx->entries) rib4_delete(r->vrf_id, r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_free(ctx->entries); +} + +void rib4_cleanup(struct nexthop *nh) { + struct rib4_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib4_cleanup_run(&ctx); +} + +void rib4_cleanup_iface(uint16_t iface_id) { + struct rib4_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib4_cleanup_run(&ctx); } METRIC_GAUGE(m_routes, "rib4_routes", "Number of IPv4 routes by origin."); diff --git a/smoke/addr_del_kernel_route_frr_test.sh b/smoke/addr_del_kernel_route_frr_test.sh new file mode 100755 index 000000000..773924089 --- /dev/null +++ b/smoke/addr_del_kernel_route_frr_test.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface whose addresses +# are deleted. +# +# The addresses and routes are configured with grcli on purpose: an address +# configured in FRR keeps its connected entry flagged ZEBRA_IFC_CONFIGURED, +# and zebra would not consider the interface address-less. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.100.2 +gw6=fd00:ba4::2 + +# assert_agree +# +# Check that grout and zebra hold the same view of , and that this +# view is the expected one. Retries to let the notification propagate. +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +# Addresses and routes owned by grout: zebra learns them through the +# plugin, so the connected entries are not flagged ZEBRA_IFC_CONFIGURED +# and really leave ifp->connected when deleted. +grcli address add 192.168.0.1/24 iface p0 +grcli address add 192.168.100.1/24 iface p0 +grcli address add fd00:f00::1/64 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Deleting an address which is not the last one of its family changes +# nothing. +grcli address del 192.168.0.1/24 iface p0 +grcli address del fd00:f00::1/64 iface p0 + +assert_agree $prefix4 yes "after deleting a non-last address" +assert_agree $prefix6 yes "after deleting a non-last address" + +# Deleting the last IPv6 address keeps both routes: the interface still +# has its link-local address to source neighbor solicitations from. +grcli address del fd00:ba4::1/64 iface p0 + +assert_agree $prefix6 yes "after deleting the last IPv6 address" +assert_agree $prefix4 yes "after deleting the last IPv6 address" + +# Deleting the last IPv4 address flushes the IPv4 routes of the +# interface, and only those. +grcli address del 192.168.100.1/24 iface p0 + +assert_agree $prefix4 no "after deleting the last IPv4 address" +assert_agree $prefix6 yes "after deleting the last IPv4 address" + +true From b3276b82e554cf81aefb566fb0f7326935f612a7 Mon Sep 17 00:00:00 2001 From: Maxime Leroy Date: Mon, 24 Aug 2026 12:16:41 +0200 Subject: [PATCH 2/5] ip,ip6: optionally flush interface routes on admin down zebra removes the kernel routes of an interface from its RIB as soon as the interface goes administratively down, and it does not tell the dataplane about it: rib_update_handle_kernel_route_down_possibility() considers a nexthop dead when its interface is not up, for both address families. Linux behaves the same way (fib_disable_ip flushes, and IPv6 loses its addresses with the link), so the routes never come back. grout keeps them and leaves their nexthops unresolved, which is arguably the better model for a reversible state, but it means the two tables disagree for good as soon as an interface is set down. Add interface config set flush-routes-on-iface-down to make grout follow the control plane on that point. It is off by default: grout alone keeps its current behaviour, and the routes of a down interface come back when it goes up again. Carrier loss is deliberately not a trigger, the interface is still configured up and neither Linux nor zebra flush anything in that case. Linux ranges the equivalent knobs under the interface too, net.ipv4.conf..ignore_routes_with_linkdown and net.ipv6.conf..keep_addr_on_down, the trigger being the interface rather than the routes. The value is set at startup with GROUT_FLUSH_ROUTES_ON_IFACE_DOWN, or at runtime over the API for a control plane which needs it without touching the unit file. Routes installed by a routing daemon are left alone, their owner reevaluates them on the interface event. Signed-off-by: Maxime Leroy Reviewed-by: Robin Jarry --- docs/grout.8.scdoc | 10 ++++++ main/config.c | 2 ++ main/config.h | 1 + main/grout.default | 5 +++ modules/infra/api/gr_infra.h | 17 ++++++++++ modules/infra/api/iface.c | 21 +++++++++++++ modules/infra/cli/cli_iface.h | 2 ++ modules/infra/cli/iface.c | 59 +++++++++++++++++++++++++++++++++++ modules/ip/control/route.c | 14 +++++++++ modules/ip6/control/ip6.h | 2 ++ modules/ip6/control/route.c | 58 ++++++++++++++++++++++++++++------ 11 files changed, 181 insertions(+), 10 deletions(-) diff --git a/docs/grout.8.scdoc b/docs/grout.8.scdoc index d198b7e32..2c37925d2 100644 --- a/docs/grout.8.scdoc +++ b/docs/grout.8.scdoc @@ -144,6 +144,16 @@ enable and _0_, _false_, _off_, _no_ to disable. is required when _net.ipv4.conf.all.rp_filter_ is set to _1_ (strict) in the network namespace where grout is running. +*GROUT_FLUSH_ROUTES_ON_IFACE_DOWN* + When set to _1_, _true_, _on_, or _yes_ (case insensitive), delete the + routes going out of an interface when it goes administratively down, + like zebra and Linux do. When disabled, these routes are kept with an + unresolved nexthop and start working again when the interface goes back + up. Can also be changed at runtime with *grcli interface config set + flush-routes-on-iface-down*. + + Default: _false_. + ## Sizing These variables control the size of internal data structures. Reducing them diff --git a/main/config.c b/main/config.c index 8941ba110..53f15fe9c 100644 --- a/main/config.c +++ b/main/config.c @@ -274,6 +274,7 @@ int config_parse(int argc, char **argv) { ENV_BOOL(log_packets, "GROUT_TRACE_PACKETS", false); ENV_BOOL(override_default_route, "GROUT_OVERRIDE_DEFAULT_ROUTE", false); ENV_BOOL(override_rp_filter, "GROUT_OVERRIDE_RP_FILTER", false); + ENV_BOOL(flush_routes_on_iface_down, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN", false); ENV_INT(max_ifaces, "GROUT_MAX_IFACES", 1024, 16, UINT16_MAX); ENV_INT(mempool_chunk_size, "GROUT_MEMPOOL_CHUNK_SIZE", (1 << 16) - 1, 255, (1 << 20) - 1); ENV_INT(max_nexthops, "GROUT_MAX_NEXTHOPS", 1 << 17, 64, 1 << 24); @@ -357,6 +358,7 @@ void config_print(void) { LOG(INFO, "GROUT_TRACE_PACKETS=%hhu", gr_config.log_packets); LOG(INFO, "GROUT_OVERRIDE_DEFAULT_ROUTE=%hhu", gr_config.override_default_route); LOG(INFO, "GROUT_OVERRIDE_RP_FILTER=%hhu", gr_config.override_rp_filter); + LOG(INFO, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=%hhu", gr_config.flush_routes_on_iface_down); LOG(INFO, "GROUT_MAX_IFACES=%u", gr_config.max_ifaces); LOG(INFO, "GROUT_MEMPOOL_CHUNK_SIZE=%u", gr_config.mempool_chunk_size); LOG(INFO, "GROUT_MAX_NEXTHOPS=%u", gr_config.max_nexthops); diff --git a/main/config.h b/main/config.h index c5e608230..18e4694da 100644 --- a/main/config.h +++ b/main/config.h @@ -25,6 +25,7 @@ struct gr_config { bool log_packets; bool override_default_route; bool override_rp_filter; + bool flush_routes_on_iface_down; vec char **eal_extra_args; cpu_set_t control_cpus; // control plane threads allowed CPUs cpu_set_t datapath_cpus; // datapath threads allowed CPUs diff --git a/main/grout.default b/main/grout.default index a7f212930..01a0272c6 100644 --- a/main/grout.default +++ b/main/grout.default @@ -40,6 +40,11 @@ GROUT_OVERRIDE_RP_FILTER=true # Print all ingress/egress packets (default: false). #GROUT_TRACE_PACKETS=false +# Delete the routes of an interface when it goes administratively down, like +# zebra and Linux do. Off by default: the routes are kept with an unresolved +# nexthop and come back when the interface goes up again. +#GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=false + # Maximum Transmission Unit (default: 1800). #GROUT_MAX_MTU=1800 diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index a088b9fa9..3c9943abb 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -253,6 +253,8 @@ enum gr_infra_requests : uint32_t { GR_IFACE_MAC_DEL, GR_IFACE_MAC_LIST, GR_IFACE_MAC_SET, + GR_IFACE_CONFIG_GET, + GR_IFACE_CONFIG_SET, }; enum gr_infra_events : uint32_t { @@ -364,6 +366,21 @@ struct gr_iface_set_req { GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty); +// Get the interface subsystem configuration. +struct gr_iface_config_get_resp { + bool flush_routes_on_iface_down; +}; + +GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp); + +// Change the interface subsystem configuration. +struct gr_iface_config_set_req { + // Delete the routes of an interface when it goes administratively down. + bool flush_routes_on_iface_down; +}; + +GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty); + // Get interface statistics. struct gr_iface_stats { uint16_t iface_id; diff --git a/modules/infra/api/iface.c b/modules/infra/api/iface.c index bac9ca253..c0cc0a9a0 100644 --- a/modules/infra/api/iface.c +++ b/modules/infra/api/iface.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Robin Jarry +#include "config.h" #include "event.h" #include "iface.h" #include "metrics.h" @@ -326,6 +327,24 @@ static void iface_metrics_collect(struct metrics_writer *w) { } } +static struct api_out iface_config_get(const void *, struct api_ctx *) { + struct gr_iface_config_get_resp *resp = calloc(1, sizeof(*resp)); + if (resp == NULL) + return api_out(ENOMEM, 0, NULL); + + resp->flush_routes_on_iface_down = gr_config.flush_routes_on_iface_down; + + return api_out(0, sizeof(*resp), resp); +} + +static struct api_out iface_config_set(const void *request, struct api_ctx *) { + const struct gr_iface_config_set_req *req = request; + + gr_config.flush_routes_on_iface_down = req->flush_routes_on_iface_down; + + return api_out(0, 0, NULL); +} + static struct metrics_collector iface_collector = { .name = "iface", .collect = iface_metrics_collect, @@ -341,6 +360,8 @@ RTE_INIT(infra_api_init) { api_handler(GR_IFACE_MAC_LIST, iface_mac_list); api_handler(GR_IFACE_MAC_SET, iface_mac_set); api_handler(GR_IFACE_SET, iface_set); + api_handler(GR_IFACE_CONFIG_GET, iface_config_get); + api_handler(GR_IFACE_CONFIG_SET, iface_config_set); event_serializer(GR_EVENT_IFACE_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_POST_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_PRE_REMOVE, iface_event_serialize); diff --git a/modules/infra/cli/cli_iface.h b/modules/infra/cli/cli_iface.h index 10baa868f..6b00d87c5 100644 --- a/modules/infra/cli/cli_iface.h +++ b/modules/infra/cli/cli_iface.h @@ -72,6 +72,8 @@ int arg_iface( CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("add", "Create an interface.")) #define INTERFACE_SET_CTX(root) \ CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("set", "Modify an existing interface.")) +#define INTERFACE_CONFIG_CTX(root) \ + CLI_CONTEXT(INTERFACE_CTX(root), CTX_ARG("config", "Interface subsystem configuration.")) #define IFACE_ATTRS_CMD \ "(up|down),(promisc PROMISC),(neigh_snoop NEIGH_SNOOP),(mtu MTU)," \ diff --git a/modules/infra/cli/iface.c b/modules/infra/cli/iface.c index e5c88f67b..1ae09bd25 100644 --- a/modules/infra/cli/iface.c +++ b/modules/infra/cli/iface.c @@ -547,6 +547,42 @@ static cmd_status_t iface_show(struct gr_api_client *c, const struct ec_pnode *p return CMD_SUCCESS; } +static cmd_status_t iface_config_show(struct gr_api_client *c, const struct ec_pnode *) { + const struct gr_iface_config_get_resp *resp; + void *resp_ptr = NULL; + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_GET, 0, NULL, &resp_ptr) < 0) + return CMD_ERROR; + + resp = resp_ptr; + struct gr_object *o = gr_object_new(NULL); + gr_object_field( + o, + "flush_routes_on_iface_down", + GR_DISP_BOOL, + "%s", + resp->flush_routes_on_iface_down ? "true" : "false" + ); + gr_object_free(o); + free(resp_ptr); + + return CMD_SUCCESS; +} + +static cmd_status_t iface_config_set(struct gr_api_client *c, const struct ec_pnode *p) { + struct gr_iface_config_set_req req = {0}; + const char *flush; + + flush = arg_str(p, "FLUSH"); + if (flush != NULL) + req.flush_routes_on_iface_down = strcmp(flush, "on") == 0; + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0) + return CMD_ERROR; + + return CMD_SUCCESS; +} + static int ctx_init(struct ec_node *root) { int ret; @@ -556,6 +592,29 @@ static int ctx_init(struct ec_node *root) { if (INTERFACE_SET_CTX(root) == NULL) return -1; + ret = CLI_COMMAND( + INTERFACE_CONFIG_CTX(root), + "set flush-routes-on-iface-down FLUSH", + iface_config_set, + "Change the interface subsystem configuration.", + with_help( + "Delete the routes going out of an interface when it goes " + "administratively down.", + EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off")) + ) + ); + if (ret < 0) + return ret; + + ret = CLI_COMMAND( + INTERFACE_CONFIG_CTX(root), + "[show]", + iface_config_show, + "Show the interface subsystem configuration." + ); + if (ret < 0) + return ret; + ret = CLI_COMMAND( INTERFACE_CTX(root), "del NAME", diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index 822b78c08..011124c2b 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -902,6 +902,18 @@ static const struct vrf_fib_ops fib4_ops = { .fini = fib4_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + if (!gr_config.flush_routes_on_iface_down) + return; + + rib4_cleanup_iface(iface->id); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP4_ROUTE_ADD, route4_add); api_handler(GR_IP4_ROUTE_DEL, route4_del); @@ -914,4 +926,6 @@ RTE_INIT(control_ip_init) { module_register(&route4_module); metrics_register(&rib4_collector); vrf_fib_ops_register(GR_AF_IP4, &fib4_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } diff --git a/modules/ip6/control/ip6.h b/modules/ip6/control/ip6.h index 83fb9c975..35c537298 100644 --- a/modules/ip6/control/ip6.h +++ b/modules/ip6/control/ip6.h @@ -71,6 +71,8 @@ int rib6_delete( gr_nh_type_t nh_type ); void rib6_cleanup(struct nexthop *); +// delete the routes of an interface, except those owned by a routing daemon +void rib6_cleanup_iface(uint16_t iface_id); struct nexthop *rib6_lookup(uint16_t vrf_id, uint16_t iface_id, const struct rte_ipv6_addr *); struct nexthop *rib6_lookup_exact( uint16_t vrf_id, diff --git a/modules/ip6/control/route.c b/modules/ip6/control/route.c index a775b8ffe..665273704 100644 --- a/modules/ip6/control/route.c +++ b/modules/ip6/control/route.c @@ -579,6 +579,7 @@ struct rib6_cleanup_entry { struct rib6_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib6_cleanup_entry *entries; }; @@ -586,12 +587,21 @@ static int rib6_cleanup_cb( uint16_t vrf_id, const struct rte_ipv6_addr *ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib6_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib6_cleanup_entry entry = { .vrf_id = vrf_id, .iface_id = nh->iface_id, @@ -604,21 +614,35 @@ static int rib6_cleanup_cb( return 0; } -void rib6_cleanup(struct nexthop *nh) { - struct rib6_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx) { struct rib6_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib6_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib6_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) + vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx->entries) rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_free(ctx->entries); +} + +void rib6_cleanup(struct nexthop *nh) { + struct rib6_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib6_cleanup_run(&ctx); +} + +void rib6_cleanup_iface(uint16_t iface_id) { + struct rib6_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib6_cleanup_run(&ctx); } METRIC_GAUGE(m_routes, "rib6_routes", "Number of IPv6 routes by origin."); @@ -915,6 +939,18 @@ static const struct vrf_fib_ops fib6_ops = { .fini = fib6_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + if (!gr_config.flush_routes_on_iface_down) + return; + + rib6_cleanup_iface(iface->id); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP6_ROUTE_ADD, route6_add); api_handler(GR_IP6_ROUTE_DEL, route6_del); @@ -927,4 +963,6 @@ RTE_INIT(control_ip_init) { module_register(&route6_module); metrics_register(&rib6_collector); vrf_fib_ops_register(GR_AF_IP6, &fib6_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } From d748c688c2b537710d3c3f077f0fb730acff1db7 Mon Sep 17 00:00:00 2001 From: Maxime Leroy Date: Mon, 24 Aug 2026 12:18:08 +0200 Subject: [PATCH 3/5] frr: keep zebra and grout in sync on interface down zebra drops the kernel routes of an interface from its RIB as soon as the interface goes administratively down, for both address families, and sends nothing to the dataplane: it assumes the routes are already gone, as they would be on Linux. grout keeps them, so the two tables diverged for good on every supported FRR version. Enable flush-on-iface-down when connecting to grout, and do it on every connection since a restarted grout comes back with its defaults. Signed-off-by: Maxime Leroy Reviewed-by: Robin Jarry --- frr/zebra_dplane_grout.c | 4 + smoke/iface_down_kernel_route_frr_test.sh | 91 +++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100755 smoke/iface_down_kernel_route_frr_test.sh diff --git a/frr/zebra_dplane_grout.c b/frr/zebra_dplane_grout.c index 1c1563b64..631bed13f 100644 --- a/frr/zebra_dplane_grout.c +++ b/frr/zebra_dplane_grout.c @@ -713,6 +713,10 @@ static void dplane_grout_connect(struct event *) { &grout_ctx.dg_t_dplane_update ); + // redone on every connection: a restarted grout comes back with its defaults + struct gr_iface_config_set_req req = {.flush_routes_on_iface_down = true}; + grout_client_send_recv(GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL); + gr_log_notice("connected, monitoring iface/ip events"); } diff --git a/smoke/iface_down_kernel_route_frr_test.sh b/smoke/iface_down_kernel_route_frr_test.sh new file mode 100755 index 000000000..9a49a5200 --- /dev/null +++ b/smoke/iface_down_kernel_route_frr_test.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface which goes +# administratively down, and both must leave them alone on carrier loss. +# +# The addresses and routes are configured with grcli on purpose, see +# addr_del_kernel_route_frr_test.sh. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.0.2 +gw6=fd00:ba4::2 + +# assert_agree +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +grcli address add 192.168.0.1/24 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +# The plugin enables the flush when it connects to grout. +grcli -j interface config show | jq -e .flush_routes_on_iface_down || + fail "the plugin did not enable flush-routes-on-iface-down" + +# The knob must also be reachable from the CLI, the plugin is not the only +# writer. Put it back on afterwards, the rest of the test needs the flush. +grcli interface config set flush-routes-on-iface-down off +grcli -j interface config show | jq -e '.flush_routes_on_iface_down == false' || + fail "interface config set off had no effect" +grcli interface config set flush-routes-on-iface-down on +grcli -j interface config show | jq -e '.flush_routes_on_iface_down == true' || + fail "interface config set on had no effect" + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Carrier loss only: the interface stays up, nothing is flushed. +ip -n n0 link set x-p0 down + +assert_agree $prefix4 yes "after carrier loss" +assert_agree $prefix6 yes "after carrier loss" + +ip -n n0 link set x-p0 up + +# Administratively down: both families are flushed on both sides. +grcli interface set port p0 down + +assert_agree $prefix4 no "after setting p0 down" +assert_agree $prefix6 no "after setting p0 down" + +true From b18f8e64e6a29b67f30aeab14a077f703e3074fb Mon Sep 17 00:00:00 2001 From: Maxime Leroy Date: Mon, 24 Aug 2026 12:40:42 +0200 Subject: [PATCH 4/5] ip,ip6: do not notify the routes deleted with an interface The routes deleted along with an interface, whether it goes down or is removed, are of no use to a control plane which evicts them on the interface event: zebra has already dropped them from its RIB by the time grout reports them, and each notification costs a full rib_delete() for nothing, plus a failed reinstall attempt for the routes zebra owns. Linux is silent in that case too, and its consumers are expected to react to the interface event. Add skip-route-events-on-iface-down next to the flush knob, off by default, and let the deletion paths say whether they want the events instead of guessing from ambient state. The route deletion which follows the removal of the last IPv4 address of an interface keeps notifying: nothing tells the control plane about it otherwise. It also has its GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN startup counterpart. The configuration request now carries a set_attrs mask, so that a client setting one attribute does not silently reset the ones it does not know about. The plugin enables both attributes at once. Signed-off-by: Maxime Leroy Reviewed-by: Robin Jarry --- docs/grout.8.scdoc | 9 +++++++++ frr/zebra_dplane_grout.c | 6 +++++- main/config.c | 4 ++++ main/config.h | 1 + main/grout.default | 5 +++++ modules/dhcp/control/client.c | 6 +++--- modules/infra/api/gr_infra.h | 8 ++++++++ modules/infra/api/iface.c | 6 +++++- modules/infra/api/nexthop.c | 4 ++-- modules/infra/cli/iface.c | 26 +++++++++++++++++++++++--- modules/infra/control/l3_nexthop.c | 4 ++-- modules/infra/control/nexthop.c | 2 +- modules/infra/control/nexthop.h | 4 ++-- modules/ip/control/address.c | 4 ++-- modules/ip/control/ip4.h | 12 +++++++++--- modules/ip/control/route.c | 28 +++++++++++++++++----------- modules/ip6/control/address.c | 2 +- modules/ip6/control/ip6.h | 7 ++++--- modules/ip6/control/route.c | 24 +++++++++++++----------- modules/policy/api/dnat44.c | 2 +- 20 files changed, 117 insertions(+), 47 deletions(-) diff --git a/docs/grout.8.scdoc b/docs/grout.8.scdoc index 2c37925d2..44f8351a2 100644 --- a/docs/grout.8.scdoc +++ b/docs/grout.8.scdoc @@ -154,6 +154,15 @@ enable and _0_, _false_, _off_, _no_ to disable. Default: _false_. +*GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN* + When set to _1_, _true_, _on_, or _yes_ (case insensitive), do not emit + route events for the routes deleted along with an interface. Only useful + for a control plane which already evicts those routes when it receives + the interface event. Can also be changed at runtime with *grcli + interface config set skip-route-events-on-iface-down*. + + Default: _false_. + ## Sizing These variables control the size of internal data structures. Reducing them diff --git a/frr/zebra_dplane_grout.c b/frr/zebra_dplane_grout.c index 631bed13f..bd205d2d9 100644 --- a/frr/zebra_dplane_grout.c +++ b/frr/zebra_dplane_grout.c @@ -714,7 +714,11 @@ static void dplane_grout_connect(struct event *) { ); // redone on every connection: a restarted grout comes back with its defaults - struct gr_iface_config_set_req req = {.flush_routes_on_iface_down = true}; + struct gr_iface_config_set_req req = { + .flush_routes_on_iface_down = true, + .skip_route_events_on_iface_down = true, + .set_attrs = GR_IFACE_CONFIG_SET_FLUSH_ROUTES | GR_IFACE_CONFIG_SET_SKIP_EVENTS, + }; grout_client_send_recv(GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL); gr_log_notice("connected, monitoring iface/ip events"); diff --git a/main/config.c b/main/config.c index 53f15fe9c..36fbd9247 100644 --- a/main/config.c +++ b/main/config.c @@ -275,6 +275,7 @@ int config_parse(int argc, char **argv) { ENV_BOOL(override_default_route, "GROUT_OVERRIDE_DEFAULT_ROUTE", false); ENV_BOOL(override_rp_filter, "GROUT_OVERRIDE_RP_FILTER", false); ENV_BOOL(flush_routes_on_iface_down, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN", false); + ENV_BOOL(skip_route_events_on_iface_down, "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN", false); ENV_INT(max_ifaces, "GROUT_MAX_IFACES", 1024, 16, UINT16_MAX); ENV_INT(mempool_chunk_size, "GROUT_MEMPOOL_CHUNK_SIZE", (1 << 16) - 1, 255, (1 << 20) - 1); ENV_INT(max_nexthops, "GROUT_MAX_NEXTHOPS", 1 << 17, 64, 1 << 24); @@ -359,6 +360,9 @@ void config_print(void) { LOG(INFO, "GROUT_OVERRIDE_DEFAULT_ROUTE=%hhu", gr_config.override_default_route); LOG(INFO, "GROUT_OVERRIDE_RP_FILTER=%hhu", gr_config.override_rp_filter); LOG(INFO, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=%hhu", gr_config.flush_routes_on_iface_down); + LOG(INFO, + "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=%hhu", + gr_config.skip_route_events_on_iface_down); LOG(INFO, "GROUT_MAX_IFACES=%u", gr_config.max_ifaces); LOG(INFO, "GROUT_MEMPOOL_CHUNK_SIZE=%u", gr_config.mempool_chunk_size); LOG(INFO, "GROUT_MAX_NEXTHOPS=%u", gr_config.max_nexthops); diff --git a/main/config.h b/main/config.h index 18e4694da..838d63fc2 100644 --- a/main/config.h +++ b/main/config.h @@ -26,6 +26,7 @@ struct gr_config { bool override_default_route; bool override_rp_filter; bool flush_routes_on_iface_down; + bool skip_route_events_on_iface_down; vec char **eal_extra_args; cpu_set_t control_cpus; // control plane threads allowed CPUs cpu_set_t datapath_cpus; // datapath threads allowed CPUs diff --git a/main/grout.default b/main/grout.default index 01a0272c6..2a44809dc 100644 --- a/main/grout.default +++ b/main/grout.default @@ -45,6 +45,11 @@ GROUT_OVERRIDE_RP_FILTER=true # nexthop and come back when the interface goes up again. #GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=false +# Do not emit route events for the routes deleted with an interface. Only +# useful for a control plane which already evicts them on the interface +# event (default: false). +#GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=false + # Maximum Transmission Unit (default: 1800). #GROUT_MAX_MTU=1800 diff --git a/modules/dhcp/control/client.c b/modules/dhcp/control/client.c index 6dc94dec3..7f76389f7 100644 --- a/modules/dhcp/control/client.c +++ b/modules/dhcp/control/client.c @@ -198,7 +198,7 @@ static void dhcp_expire_callback(evutil_socket_t, short, void *arg) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -370,7 +370,7 @@ void dhcp_input_cb(void *obj, uintptr_t, const struct control_queue_drain *drain if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -454,7 +454,7 @@ static int dhcp_stop(uint16_t iface_id) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); dhcp_cancel_timers(client); diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index 3c9943abb..80183a881 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -369,14 +369,22 @@ GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty); // Get the interface subsystem configuration. struct gr_iface_config_get_resp { bool flush_routes_on_iface_down; + bool skip_route_events_on_iface_down; }; GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp); +// Interface subsystem configuration attribute flags. +#define GR_IFACE_CONFIG_SET_FLUSH_ROUTES GR_BIT64(0) +#define GR_IFACE_CONFIG_SET_SKIP_EVENTS GR_BIT64(1) + // Change the interface subsystem configuration. struct gr_iface_config_set_req { // Delete the routes of an interface when it goes administratively down. bool flush_routes_on_iface_down; + // Do not emit route events for the routes deleted with an interface. + bool skip_route_events_on_iface_down; + uint64_t set_attrs; // Bit mask of GR_IFACE_CONFIG_SET_*. }; GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty); diff --git a/modules/infra/api/iface.c b/modules/infra/api/iface.c index c0cc0a9a0..f360de91c 100644 --- a/modules/infra/api/iface.c +++ b/modules/infra/api/iface.c @@ -333,6 +333,7 @@ static struct api_out iface_config_get(const void *, struct api_ctx *) { return api_out(ENOMEM, 0, NULL); resp->flush_routes_on_iface_down = gr_config.flush_routes_on_iface_down; + resp->skip_route_events_on_iface_down = gr_config.skip_route_events_on_iface_down; return api_out(0, sizeof(*resp), resp); } @@ -340,7 +341,10 @@ static struct api_out iface_config_get(const void *, struct api_ctx *) { static struct api_out iface_config_set(const void *request, struct api_ctx *) { const struct gr_iface_config_set_req *req = request; - gr_config.flush_routes_on_iface_down = req->flush_routes_on_iface_down; + if (req->set_attrs & GR_IFACE_CONFIG_SET_FLUSH_ROUTES) + gr_config.flush_routes_on_iface_down = req->flush_routes_on_iface_down; + if (req->set_attrs & GR_IFACE_CONFIG_SET_SKIP_EVENTS) + gr_config.skip_route_events_on_iface_down = req->skip_route_events_on_iface_down; return api_out(0, 0, NULL); } diff --git a/modules/infra/api/nexthop.c b/modules/infra/api/nexthop.c index df6b3e599..78c0678ca 100644 --- a/modules/infra/api/nexthop.c +++ b/modules/infra/api/nexthop.c @@ -67,7 +67,7 @@ static struct api_out nh_del(const void *request, struct api_ctx *) { return api_out(EBUSY, 0, NULL); } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); // The nexthop *may* still have one ref_count when it has been created // manually from the API (see nh_add()). Implicit nexthops created when // creating a gateway route will not have that extra ref_count. @@ -142,7 +142,7 @@ static struct api_out nh_flush(const void *request, struct api_ctx *) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) continue; } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); } diff --git a/modules/infra/cli/iface.c b/modules/infra/cli/iface.c index 1ae09bd25..34fc7c423 100644 --- a/modules/infra/cli/iface.c +++ b/modules/infra/cli/iface.c @@ -563,6 +563,13 @@ static cmd_status_t iface_config_show(struct gr_api_client *c, const struct ec_p "%s", resp->flush_routes_on_iface_down ? "true" : "false" ); + gr_object_field( + o, + "skip_route_events_on_iface_down", + GR_DISP_BOOL, + "%s", + resp->skip_route_events_on_iface_down ? "true" : "false" + ); gr_object_free(o); free(resp_ptr); @@ -571,11 +578,19 @@ static cmd_status_t iface_config_show(struct gr_api_client *c, const struct ec_p static cmd_status_t iface_config_set(struct gr_api_client *c, const struct ec_pnode *p) { struct gr_iface_config_set_req req = {0}; - const char *flush; + const char *flush, *skip; flush = arg_str(p, "FLUSH"); - if (flush != NULL) + if (flush != NULL) { req.flush_routes_on_iface_down = strcmp(flush, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_FLUSH_ROUTES; + } + + skip = arg_str(p, "SKIP"); + if (skip != NULL) { + req.skip_route_events_on_iface_down = strcmp(skip, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_SKIP_EVENTS; + } if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0) return CMD_ERROR; @@ -594,13 +609,18 @@ static int ctx_init(struct ec_node *root) { ret = CLI_COMMAND( INTERFACE_CONFIG_CTX(root), - "set flush-routes-on-iface-down FLUSH", + "set (flush-routes-on-iface-down FLUSH),(skip-route-events-on-iface-down SKIP)", iface_config_set, "Change the interface subsystem configuration.", with_help( "Delete the routes going out of an interface when it goes " "administratively down.", EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off")) + ), + with_help( + "Do not emit route events for the routes deleted along with an " + "interface.", + EC_NODE_OR("SKIP", ec_node_str("", "on"), ec_node_str("", "off")) ) ); if (ret < 0) diff --git a/modules/infra/control/l3_nexthop.c b/modules/infra/control/l3_nexthop.c index 381c4a921..5276a8a34 100644 --- a/modules/infra/control/l3_nexthop.c +++ b/modules/infra/control/l3_nexthop.c @@ -142,12 +142,12 @@ static int l3_reconfig(const struct gr_nexthop_config *c) { return 0; } -void nexthop_routes_cleanup(struct nexthop *nh) { +void nexthop_routes_cleanup(struct nexthop *nh, bool notify) { const struct nexthop_af_ops *ops; for (unsigned i = 0; i < ARRAY_DIM(af_ops); i++) { ops = af_ops[i]; if (ops != NULL) - ops->cleanup_routes(nh); + ops->cleanup_routes(nh, notify); } } diff --git a/modules/infra/control/nexthop.c b/modules/infra/control/nexthop.c index 3703c2d74..3bda6255b 100644 --- a/modules/infra/control/nexthop.c +++ b/modules/infra/control/nexthop.c @@ -479,7 +479,7 @@ static void nh_cleanup_interface_cb(struct nexthop *nh, void *priv) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) return; // addresses are cleaned per address family } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, !gr_config.skip_route_events_on_iface_down); while (nh->ref_count) nexthop_decref(nh); } diff --git a/modules/infra/control/nexthop.h b/modules/infra/control/nexthop.h index 14fef399a..2e16f2e88 100644 --- a/modules/infra/control/nexthop.h +++ b/modules/infra/control/nexthop.h @@ -128,7 +128,7 @@ struct gr_nexthop *nexthop_to_api(const struct nexthop *, size_t *len); int nexthop_serialize(const void *obj, void **buf); // Clean all routes that reference a given nexthop. -void nexthop_routes_cleanup(struct nexthop *); +void nexthop_routes_cleanup(struct nexthop *, bool notify); // Increment the reference counter of a nexthop. void nexthop_incref(struct nexthop *); @@ -156,7 +156,7 @@ struct nexthop_af_ops { // Callback that will be invoked when a nexthop needs to be refreshed by sending a probe. int (*solicit)(struct nexthop *); // Callback that will be invoked to delete all routes which reference a given nexthop. - void (*cleanup_routes)(struct nexthop *); + void (*cleanup_routes)(struct nexthop *, bool notify); // Callback invoked by resolve() to flush held packets when the nexthop becomes // reachable. int (*resubmit)(struct rte_mbuf *, struct nexthop *); diff --git a/modules/ip/control/address.c b/modules/ip/control/address.c index 56a260a6d..7daaa852c 100644 --- a/modules/ip/control/address.c +++ b/modules/ip/control/address.c @@ -197,7 +197,7 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) { } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); @@ -205,7 +205,7 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) { if (vec_len(addrs->nh) == 0) { vec_free(addrs->nh); // no address left to send ARP requests from - rib4_cleanup_iface(iface_id); + rib4_cleanup_iface(iface_id, true); } iface = iface_from_id(iface_id); diff --git a/modules/ip/control/ip4.h b/modules/ip/control/ip4.h index 247d22d54..3e8d48ce3 100644 --- a/modules/ip/control/ip4.h +++ b/modules/ip/control/ip4.h @@ -34,10 +34,16 @@ int rib4_insert( gr_nh_origin_t origin, struct nexthop *nh ); -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type); -void rib4_cleanup(struct nexthop *); +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +); +void rib4_cleanup(struct nexthop *, bool notify); // delete the routes of an interface, except those owned by a routing daemon -void rib4_cleanup_iface(uint16_t iface_id); +void rib4_cleanup_iface(uint16_t iface_id, bool notify); typedef int (*rib4_iter_cb_t)( uint16_t vrf_id, diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index 011124c2b..ed9e965b0 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -284,7 +284,13 @@ int rib4_insert( return rib4_insert_or_replace(vrf_id, ip, prefixlen, origin, nh, false); } -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type) { +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +) { struct rte_fib *fib = get_fib(vrf_id); gr_nh_origin_t *o, origin; struct rte_rib_node *rn; @@ -312,7 +318,7 @@ int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t if ((ret = rte_fib_delete(fib, rte_be_to_cpu_32(ip), prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP_ROUTE_DEL, &(const struct route4_event) { @@ -393,7 +399,7 @@ static struct api_out route4_del(const void *request, struct api_ctx *) { nh = rib4_lookup(req->vrf_id, req->dest.ip); ret = rib4_delete( - req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3 + req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3, true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -571,7 +577,7 @@ static int rib4_cleanup_cb( return 0; } -static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx) { +static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx, bool notify) { struct rib4_iterator iter = { .max_count = 0, .skip_internal = false, @@ -580,26 +586,26 @@ static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx) { }; rib4_iter(GR_VRF_ID_UNDEF, &iter); vec_foreach_ref (struct rib4_cleanup_entry *r, ctx->entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, notify); vec_free(ctx->entries); } -void rib4_cleanup(struct nexthop *nh) { +void rib4_cleanup(struct nexthop *nh, bool notify) { struct rib4_cleanup_ctx ctx = { .nh = nh, .iface_id = GR_IFACE_ID_UNDEF, .entries = NULL, }; - rib4_cleanup_run(&ctx); + rib4_cleanup_run(&ctx, notify); } -void rib4_cleanup_iface(uint16_t iface_id) { +void rib4_cleanup_iface(uint16_t iface_id, bool notify) { struct rib4_cleanup_ctx ctx = { .nh = NULL, .iface_id = iface_id, .entries = NULL, }; - rib4_cleanup_run(&ctx); + rib4_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib4_routes", "Number of IPv4 routes by origin."); @@ -872,7 +878,7 @@ static void fib4_fini(struct iface *vrf) { rib4_iter_vrf(rte_fib_get_rib(fib), vrf->id, &iter); vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib4 = NULL; @@ -911,7 +917,7 @@ static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { if (!gr_config.flush_routes_on_iface_down) return; - rib4_cleanup_iface(iface->id); + rib4_cleanup_iface(iface->id, !gr_config.skip_route_events_on_iface_down); } RTE_INIT(control_ip_init) { diff --git a/modules/ip6/control/address.c b/modules/ip6/control/address.c index 7b62d84de..051ebcd3d 100644 --- a/modules/ip6/control/address.c +++ b/modules/ip6/control/address.c @@ -326,7 +326,7 @@ int addr6_delete(uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t pref } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); diff --git a/modules/ip6/control/ip6.h b/modules/ip6/control/ip6.h index 35c537298..20b4b33af 100644 --- a/modules/ip6/control/ip6.h +++ b/modules/ip6/control/ip6.h @@ -68,11 +68,12 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ); -void rib6_cleanup(struct nexthop *); +void rib6_cleanup(struct nexthop *, bool notify); // delete the routes of an interface, except those owned by a routing daemon -void rib6_cleanup_iface(uint16_t iface_id); +void rib6_cleanup_iface(uint16_t iface_id, bool notify); struct nexthop *rib6_lookup(uint16_t vrf_id, uint16_t iface_id, const struct rte_ipv6_addr *); struct nexthop *rib6_lookup_exact( uint16_t vrf_id, diff --git a/modules/ip6/control/route.c b/modules/ip6/control/route.c index 665273704..bf68bd60c 100644 --- a/modules/ip6/control/route.c +++ b/modules/ip6/control/route.c @@ -312,7 +312,8 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ) { struct rte_fib6 *fib = get_fib6(vrf_id); const struct rte_ipv6_addr *scoped_ip; @@ -344,7 +345,7 @@ int rib6_delete( if ((ret = rte_fib6_delete(fib, scoped_ip, prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP6_ROUTE_DEL, &(const struct route6_event) { @@ -438,7 +439,8 @@ static struct api_out route6_del(const void *request, struct api_ctx *) { GR_IFACE_ID_UNDEF, &req->dest.ip, req->dest.prefixlen, - nh ? nh->type : GR_NH_T_L3 + nh ? nh->type : GR_NH_T_L3, + true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -614,7 +616,7 @@ static int rib6_cleanup_cb( return 0; } -static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx) { +static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx, bool notify) { struct rib6_iterator iter = { .max_count = 0, .skip_internal = false, @@ -623,26 +625,26 @@ static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx) { }; rib6_iter(GR_VRF_ID_UNDEF, &iter); vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx->entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, notify); vec_free(ctx->entries); } -void rib6_cleanup(struct nexthop *nh) { +void rib6_cleanup(struct nexthop *nh, bool notify) { struct rib6_cleanup_ctx ctx = { .nh = nh, .iface_id = GR_IFACE_ID_UNDEF, .entries = NULL, }; - rib6_cleanup_run(&ctx); + rib6_cleanup_run(&ctx, notify); } -void rib6_cleanup_iface(uint16_t iface_id) { +void rib6_cleanup_iface(uint16_t iface_id, bool notify) { struct rib6_cleanup_ctx ctx = { .nh = NULL, .iface_id = iface_id, .entries = NULL, }; - rib6_cleanup_run(&ctx); + rib6_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib6_routes", "Number of IPv6 routes by origin."); @@ -908,7 +910,7 @@ static void fib6_fini(struct iface *vrf) { rib6_iter_vrf(rte_fib6_get_rib(fib), vrf->id, &iter); vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib6 = NULL; @@ -948,7 +950,7 @@ static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { if (!gr_config.flush_routes_on_iface_down) return; - rib6_cleanup_iface(iface->id); + rib6_cleanup_iface(iface->id, !gr_config.skip_route_events_on_iface_down); } RTE_INIT(control_ip_init) { diff --git a/modules/policy/api/dnat44.c b/modules/policy/api/dnat44.c index 2c8f00903..ee56f0d1c 100644 --- a/modules/policy/api/dnat44.c +++ b/modules/policy/api/dnat44.c @@ -176,7 +176,7 @@ static struct api_out dnat44_del(const void *request, struct api_ctx *) { if (iface == NULL) return api_out(ENODEV, 0, NULL); - ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT); + ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT, true); if (ret == -ENOENT && req->missing_ok) ret = 0; From 335d58e7a48964f96f529f5b5a60611e36eea93a Mon Sep 17 00:00:00 2001 From: Maxime Leroy Date: Mon, 24 Aug 2026 12:43:58 +0200 Subject: [PATCH 5/5] ip,ip6: accept an unspecified default FIB size The route config set command dispatches to every registered address family with the same parse node, so a family which was not given an argument gets zero. Both handlers rejected it with EINVAL, which made "route config set default rib4-routes N" report a failure although the value had been applied, and contradicted the request documentation which says that zero means unchanged. Treat zero as nothing to change, like the neighbouring FIB fields already do. A set_attrs mask would only be needed if zero were a valid FIB size. Fixes: e1b0bde7f67a ("ip,ip6: allow configuring default FIB size") Signed-off-by: Maxime Leroy Reviewed-by: Robin Jarry --- modules/ip/control/route.c | 2 +- modules/ip6/control/route.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index ed9e965b0..83457a77a 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -892,7 +892,7 @@ static struct api_out fib4_default_set(const void *request, struct api_ctx *) { const struct gr_ip4_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv4 default max_routes %u -> %u", max_routes_default, req->max_routes); diff --git a/modules/ip6/control/route.c b/modules/ip6/control/route.c index bf68bd60c..c0f7adb71 100644 --- a/modules/ip6/control/route.c +++ b/modules/ip6/control/route.c @@ -925,7 +925,7 @@ static struct api_out fib6_default_set(const void *request, struct api_ctx *) { const struct gr_ip6_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv6 default max_routes %u -> %u", max_routes_default, req->max_routes);