diff --git a/docs/grout.8.scdoc b/docs/grout.8.scdoc index d198b7e32..44f8351a2 100644 --- a/docs/grout.8.scdoc +++ b/docs/grout.8.scdoc @@ -144,6 +144,25 @@ enable and _0_, _false_, _off_, _no_ to disable. is required when _net.ipv4.conf.all.rp_filter_ is set to _1_ (strict) in the network namespace where grout is running. +*GROUT_FLUSH_ROUTES_ON_IFACE_DOWN* + When set to _1_, _true_, _on_, or _yes_ (case insensitive), delete the + routes going out of an interface when it goes administratively down, + like zebra and Linux do. When disabled, these routes are kept with an + unresolved nexthop and start working again when the interface goes back + up. Can also be changed at runtime with *grcli interface config set + flush-routes-on-iface-down*. + + Default: _false_. + +*GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN* + When set to _1_, _true_, _on_, or _yes_ (case insensitive), do not emit + route events for the routes deleted along with an interface. Only useful + for a control plane which already evicts those routes when it receives + the interface event. Can also be changed at runtime with *grcli + interface config set skip-route-events-on-iface-down*. + + Default: _false_. + ## Sizing These variables control the size of internal data structures. Reducing them diff --git a/frr/zebra_dplane_grout.c b/frr/zebra_dplane_grout.c index 1c1563b64..bd205d2d9 100644 --- a/frr/zebra_dplane_grout.c +++ b/frr/zebra_dplane_grout.c @@ -713,6 +713,14 @@ static void dplane_grout_connect(struct event *) { &grout_ctx.dg_t_dplane_update ); + // redone on every connection: a restarted grout comes back with its defaults + struct gr_iface_config_set_req req = { + .flush_routes_on_iface_down = true, + .skip_route_events_on_iface_down = true, + .set_attrs = GR_IFACE_CONFIG_SET_FLUSH_ROUTES | GR_IFACE_CONFIG_SET_SKIP_EVENTS, + }; + grout_client_send_recv(GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL); + gr_log_notice("connected, monitoring iface/ip events"); } diff --git a/main/config.c b/main/config.c index 8941ba110..36fbd9247 100644 --- a/main/config.c +++ b/main/config.c @@ -274,6 +274,8 @@ int config_parse(int argc, char **argv) { ENV_BOOL(log_packets, "GROUT_TRACE_PACKETS", false); ENV_BOOL(override_default_route, "GROUT_OVERRIDE_DEFAULT_ROUTE", false); ENV_BOOL(override_rp_filter, "GROUT_OVERRIDE_RP_FILTER", false); + ENV_BOOL(flush_routes_on_iface_down, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN", false); + ENV_BOOL(skip_route_events_on_iface_down, "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN", false); ENV_INT(max_ifaces, "GROUT_MAX_IFACES", 1024, 16, UINT16_MAX); ENV_INT(mempool_chunk_size, "GROUT_MEMPOOL_CHUNK_SIZE", (1 << 16) - 1, 255, (1 << 20) - 1); ENV_INT(max_nexthops, "GROUT_MAX_NEXTHOPS", 1 << 17, 64, 1 << 24); @@ -357,6 +359,10 @@ void config_print(void) { LOG(INFO, "GROUT_TRACE_PACKETS=%hhu", gr_config.log_packets); LOG(INFO, "GROUT_OVERRIDE_DEFAULT_ROUTE=%hhu", gr_config.override_default_route); LOG(INFO, "GROUT_OVERRIDE_RP_FILTER=%hhu", gr_config.override_rp_filter); + LOG(INFO, "GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=%hhu", gr_config.flush_routes_on_iface_down); + LOG(INFO, + "GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=%hhu", + gr_config.skip_route_events_on_iface_down); LOG(INFO, "GROUT_MAX_IFACES=%u", gr_config.max_ifaces); LOG(INFO, "GROUT_MEMPOOL_CHUNK_SIZE=%u", gr_config.mempool_chunk_size); LOG(INFO, "GROUT_MAX_NEXTHOPS=%u", gr_config.max_nexthops); diff --git a/main/config.h b/main/config.h index c5e608230..838d63fc2 100644 --- a/main/config.h +++ b/main/config.h @@ -25,6 +25,8 @@ struct gr_config { bool log_packets; bool override_default_route; bool override_rp_filter; + bool flush_routes_on_iface_down; + bool skip_route_events_on_iface_down; vec char **eal_extra_args; cpu_set_t control_cpus; // control plane threads allowed CPUs cpu_set_t datapath_cpus; // datapath threads allowed CPUs diff --git a/main/grout.default b/main/grout.default index a7f212930..2a44809dc 100644 --- a/main/grout.default +++ b/main/grout.default @@ -40,6 +40,16 @@ GROUT_OVERRIDE_RP_FILTER=true # Print all ingress/egress packets (default: false). #GROUT_TRACE_PACKETS=false +# Delete the routes of an interface when it goes administratively down, like +# zebra and Linux do. Off by default: the routes are kept with an unresolved +# nexthop and come back when the interface goes up again. +#GROUT_FLUSH_ROUTES_ON_IFACE_DOWN=false + +# Do not emit route events for the routes deleted with an interface. Only +# useful for a control plane which already evicts them on the interface +# event (default: false). +#GROUT_SKIP_ROUTE_EVENTS_ON_IFACE_DOWN=false + # Maximum Transmission Unit (default: 1800). #GROUT_MAX_MTU=1800 diff --git a/modules/dhcp/control/client.c b/modules/dhcp/control/client.c index 6dc94dec3..7f76389f7 100644 --- a/modules/dhcp/control/client.c +++ b/modules/dhcp/control/client.c @@ -198,7 +198,7 @@ static void dhcp_expire_callback(evutil_socket_t, short, void *arg) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -370,7 +370,7 @@ void dhcp_input_cb(void *obj, uintptr_t, const struct control_queue_drain *drain if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -454,7 +454,7 @@ static int dhcp_stop(uint16_t iface_id) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); dhcp_cancel_timers(client); diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index a088b9fa9..80183a881 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -253,6 +253,8 @@ enum gr_infra_requests : uint32_t { GR_IFACE_MAC_DEL, GR_IFACE_MAC_LIST, GR_IFACE_MAC_SET, + GR_IFACE_CONFIG_GET, + GR_IFACE_CONFIG_SET, }; enum gr_infra_events : uint32_t { @@ -364,6 +366,29 @@ struct gr_iface_set_req { GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty); +// Get the interface subsystem configuration. +struct gr_iface_config_get_resp { + bool flush_routes_on_iface_down; + bool skip_route_events_on_iface_down; +}; + +GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp); + +// Interface subsystem configuration attribute flags. +#define GR_IFACE_CONFIG_SET_FLUSH_ROUTES GR_BIT64(0) +#define GR_IFACE_CONFIG_SET_SKIP_EVENTS GR_BIT64(1) + +// Change the interface subsystem configuration. +struct gr_iface_config_set_req { + // Delete the routes of an interface when it goes administratively down. + bool flush_routes_on_iface_down; + // Do not emit route events for the routes deleted with an interface. + bool skip_route_events_on_iface_down; + uint64_t set_attrs; // Bit mask of GR_IFACE_CONFIG_SET_*. +}; + +GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty); + // Get interface statistics. struct gr_iface_stats { uint16_t iface_id; diff --git a/modules/infra/api/iface.c b/modules/infra/api/iface.c index bac9ca253..f360de91c 100644 --- a/modules/infra/api/iface.c +++ b/modules/infra/api/iface.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2024 Robin Jarry +#include "config.h" #include "event.h" #include "iface.h" #include "metrics.h" @@ -326,6 +327,28 @@ static void iface_metrics_collect(struct metrics_writer *w) { } } +static struct api_out iface_config_get(const void *, struct api_ctx *) { + struct gr_iface_config_get_resp *resp = calloc(1, sizeof(*resp)); + if (resp == NULL) + return api_out(ENOMEM, 0, NULL); + + resp->flush_routes_on_iface_down = gr_config.flush_routes_on_iface_down; + resp->skip_route_events_on_iface_down = gr_config.skip_route_events_on_iface_down; + + return api_out(0, sizeof(*resp), resp); +} + +static struct api_out iface_config_set(const void *request, struct api_ctx *) { + const struct gr_iface_config_set_req *req = request; + + if (req->set_attrs & GR_IFACE_CONFIG_SET_FLUSH_ROUTES) + gr_config.flush_routes_on_iface_down = req->flush_routes_on_iface_down; + if (req->set_attrs & GR_IFACE_CONFIG_SET_SKIP_EVENTS) + gr_config.skip_route_events_on_iface_down = req->skip_route_events_on_iface_down; + + return api_out(0, 0, NULL); +} + static struct metrics_collector iface_collector = { .name = "iface", .collect = iface_metrics_collect, @@ -341,6 +364,8 @@ RTE_INIT(infra_api_init) { api_handler(GR_IFACE_MAC_LIST, iface_mac_list); api_handler(GR_IFACE_MAC_SET, iface_mac_set); api_handler(GR_IFACE_SET, iface_set); + api_handler(GR_IFACE_CONFIG_GET, iface_config_get); + api_handler(GR_IFACE_CONFIG_SET, iface_config_set); event_serializer(GR_EVENT_IFACE_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_POST_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_PRE_REMOVE, iface_event_serialize); diff --git a/modules/infra/api/nexthop.c b/modules/infra/api/nexthop.c index df6b3e599..78c0678ca 100644 --- a/modules/infra/api/nexthop.c +++ b/modules/infra/api/nexthop.c @@ -67,7 +67,7 @@ static struct api_out nh_del(const void *request, struct api_ctx *) { return api_out(EBUSY, 0, NULL); } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); // The nexthop *may* still have one ref_count when it has been created // manually from the API (see nh_add()). Implicit nexthops created when // creating a gateway route will not have that extra ref_count. @@ -142,7 +142,7 @@ static struct api_out nh_flush(const void *request, struct api_ctx *) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) continue; } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); } diff --git a/modules/infra/cli/cli_iface.h b/modules/infra/cli/cli_iface.h index 10baa868f..6b00d87c5 100644 --- a/modules/infra/cli/cli_iface.h +++ b/modules/infra/cli/cli_iface.h @@ -72,6 +72,8 @@ int arg_iface( CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("add", "Create an interface.")) #define INTERFACE_SET_CTX(root) \ CLI_CONTEXT(root, INTERFACE_ARG, CTX_ARG("set", "Modify an existing interface.")) +#define INTERFACE_CONFIG_CTX(root) \ + CLI_CONTEXT(INTERFACE_CTX(root), CTX_ARG("config", "Interface subsystem configuration.")) #define IFACE_ATTRS_CMD \ "(up|down),(promisc PROMISC),(neigh_snoop NEIGH_SNOOP),(mtu MTU)," \ diff --git a/modules/infra/cli/iface.c b/modules/infra/cli/iface.c index e5c88f67b..34fc7c423 100644 --- a/modules/infra/cli/iface.c +++ b/modules/infra/cli/iface.c @@ -547,6 +547,57 @@ static cmd_status_t iface_show(struct gr_api_client *c, const struct ec_pnode *p return CMD_SUCCESS; } +static cmd_status_t iface_config_show(struct gr_api_client *c, const struct ec_pnode *) { + const struct gr_iface_config_get_resp *resp; + void *resp_ptr = NULL; + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_GET, 0, NULL, &resp_ptr) < 0) + return CMD_ERROR; + + resp = resp_ptr; + struct gr_object *o = gr_object_new(NULL); + gr_object_field( + o, + "flush_routes_on_iface_down", + GR_DISP_BOOL, + "%s", + resp->flush_routes_on_iface_down ? "true" : "false" + ); + gr_object_field( + o, + "skip_route_events_on_iface_down", + GR_DISP_BOOL, + "%s", + resp->skip_route_events_on_iface_down ? "true" : "false" + ); + gr_object_free(o); + free(resp_ptr); + + return CMD_SUCCESS; +} + +static cmd_status_t iface_config_set(struct gr_api_client *c, const struct ec_pnode *p) { + struct gr_iface_config_set_req req = {0}; + const char *flush, *skip; + + flush = arg_str(p, "FLUSH"); + if (flush != NULL) { + req.flush_routes_on_iface_down = strcmp(flush, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_FLUSH_ROUTES; + } + + skip = arg_str(p, "SKIP"); + if (skip != NULL) { + req.skip_route_events_on_iface_down = strcmp(skip, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_SKIP_EVENTS; + } + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0) + return CMD_ERROR; + + return CMD_SUCCESS; +} + static int ctx_init(struct ec_node *root) { int ret; @@ -556,6 +607,34 @@ static int ctx_init(struct ec_node *root) { if (INTERFACE_SET_CTX(root) == NULL) return -1; + ret = CLI_COMMAND( + INTERFACE_CONFIG_CTX(root), + "set (flush-routes-on-iface-down FLUSH),(skip-route-events-on-iface-down SKIP)", + iface_config_set, + "Change the interface subsystem configuration.", + with_help( + "Delete the routes going out of an interface when it goes " + "administratively down.", + EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off")) + ), + with_help( + "Do not emit route events for the routes deleted along with an " + "interface.", + EC_NODE_OR("SKIP", ec_node_str("", "on"), ec_node_str("", "off")) + ) + ); + if (ret < 0) + return ret; + + ret = CLI_COMMAND( + INTERFACE_CONFIG_CTX(root), + "[show]", + iface_config_show, + "Show the interface subsystem configuration." + ); + if (ret < 0) + return ret; + ret = CLI_COMMAND( INTERFACE_CTX(root), "del NAME", diff --git a/modules/infra/control/l3_nexthop.c b/modules/infra/control/l3_nexthop.c index 381c4a921..5276a8a34 100644 --- a/modules/infra/control/l3_nexthop.c +++ b/modules/infra/control/l3_nexthop.c @@ -142,12 +142,12 @@ static int l3_reconfig(const struct gr_nexthop_config *c) { return 0; } -void nexthop_routes_cleanup(struct nexthop *nh) { +void nexthop_routes_cleanup(struct nexthop *nh, bool notify) { const struct nexthop_af_ops *ops; for (unsigned i = 0; i < ARRAY_DIM(af_ops); i++) { ops = af_ops[i]; if (ops != NULL) - ops->cleanup_routes(nh); + ops->cleanup_routes(nh, notify); } } diff --git a/modules/infra/control/nexthop.c b/modules/infra/control/nexthop.c index 3703c2d74..3bda6255b 100644 --- a/modules/infra/control/nexthop.c +++ b/modules/infra/control/nexthop.c @@ -479,7 +479,7 @@ static void nh_cleanup_interface_cb(struct nexthop *nh, void *priv) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) return; // addresses are cleaned per address family } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, !gr_config.skip_route_events_on_iface_down); while (nh->ref_count) nexthop_decref(nh); } diff --git a/modules/infra/control/nexthop.h b/modules/infra/control/nexthop.h index 14fef399a..2e16f2e88 100644 --- a/modules/infra/control/nexthop.h +++ b/modules/infra/control/nexthop.h @@ -128,7 +128,7 @@ struct gr_nexthop *nexthop_to_api(const struct nexthop *, size_t *len); int nexthop_serialize(const void *obj, void **buf); // Clean all routes that reference a given nexthop. -void nexthop_routes_cleanup(struct nexthop *); +void nexthop_routes_cleanup(struct nexthop *, bool notify); // Increment the reference counter of a nexthop. void nexthop_incref(struct nexthop *); @@ -156,7 +156,7 @@ struct nexthop_af_ops { // Callback that will be invoked when a nexthop needs to be refreshed by sending a probe. int (*solicit)(struct nexthop *); // Callback that will be invoked to delete all routes which reference a given nexthop. - void (*cleanup_routes)(struct nexthop *); + void (*cleanup_routes)(struct nexthop *, bool notify); // Callback invoked by resolve() to flush held packets when the nexthop becomes // reachable. int (*resubmit)(struct rte_mbuf *, struct nexthop *); diff --git a/modules/ip/control/address.c b/modules/ip/control/address.c index 2720e5dc0..7daaa852c 100644 --- a/modules/ip/control/address.c +++ b/modules/ip/control/address.c @@ -197,13 +197,16 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) { } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); vec_del(addrs->nh, i); - if (vec_len(addrs->nh) == 0) + if (vec_len(addrs->nh) == 0) { vec_free(addrs->nh); + // no address left to send ARP requests from + rib4_cleanup_iface(iface_id, true); + } iface = iface_from_id(iface_id); if (iface && iface->cp_id != 0) { diff --git a/modules/ip/control/ip4.h b/modules/ip/control/ip4.h index 53daf6a97..3e8d48ce3 100644 --- a/modules/ip/control/ip4.h +++ b/modules/ip/control/ip4.h @@ -34,8 +34,16 @@ int rib4_insert( gr_nh_origin_t origin, struct nexthop *nh ); -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type); -void rib4_cleanup(struct nexthop *); +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +); +void rib4_cleanup(struct nexthop *, bool notify); +// delete the routes of an interface, except those owned by a routing daemon +void rib4_cleanup_iface(uint16_t iface_id, bool notify); typedef int (*rib4_iter_cb_t)( uint16_t vrf_id, diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index 8846d5662..83457a77a 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -284,7 +284,13 @@ int rib4_insert( return rib4_insert_or_replace(vrf_id, ip, prefixlen, origin, nh, false); } -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type) { +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +) { struct rte_fib *fib = get_fib(vrf_id); gr_nh_origin_t *o, origin; struct rte_rib_node *rn; @@ -312,7 +318,7 @@ int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t if ((ret = rte_fib_delete(fib, rte_be_to_cpu_32(ip), prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP_ROUTE_DEL, &(const struct route4_event) { @@ -393,7 +399,7 @@ static struct api_out route4_del(const void *request, struct api_ctx *) { nh = rib4_lookup(req->vrf_id, req->dest.ip); ret = rib4_delete( - req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3 + req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3, true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -537,6 +543,7 @@ struct rib4_cleanup_entry { struct rib4_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib4_cleanup_entry *entries; }; @@ -544,12 +551,21 @@ static int rib4_cleanup_cb( uint16_t vrf_id, ip4_addr_t ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib4_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib4_cleanup_entry entry = { .vrf_id = vrf_id, .ip = ip, @@ -561,21 +577,35 @@ static int rib4_cleanup_cb( return 0; } -void rib4_cleanup(struct nexthop *nh) { - struct rib4_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx, bool notify) { struct rib4_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib4_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib4_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_foreach_ref (struct rib4_cleanup_entry *r, ctx->entries) + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, notify); + vec_free(ctx->entries); +} + +void rib4_cleanup(struct nexthop *nh, bool notify) { + struct rib4_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib4_cleanup_run(&ctx, notify); +} + +void rib4_cleanup_iface(uint16_t iface_id, bool notify) { + struct rib4_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib4_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib4_routes", "Number of IPv4 routes by origin."); @@ -848,7 +878,7 @@ static void fib4_fini(struct iface *vrf) { rib4_iter_vrf(rte_fib_get_rib(fib), vrf->id, &iter); vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib4 = NULL; @@ -862,7 +892,7 @@ static struct api_out fib4_default_set(const void *request, struct api_ctx *) { const struct gr_ip4_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv4 default max_routes %u -> %u", max_routes_default, req->max_routes); @@ -878,6 +908,18 @@ static const struct vrf_fib_ops fib4_ops = { .fini = fib4_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + if (!gr_config.flush_routes_on_iface_down) + return; + + rib4_cleanup_iface(iface->id, !gr_config.skip_route_events_on_iface_down); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP4_ROUTE_ADD, route4_add); api_handler(GR_IP4_ROUTE_DEL, route4_del); @@ -890,4 +932,6 @@ RTE_INIT(control_ip_init) { module_register(&route4_module); metrics_register(&rib4_collector); vrf_fib_ops_register(GR_AF_IP4, &fib4_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } diff --git a/modules/ip6/control/address.c b/modules/ip6/control/address.c index 7b62d84de..051ebcd3d 100644 --- a/modules/ip6/control/address.c +++ b/modules/ip6/control/address.c @@ -326,7 +326,7 @@ int addr6_delete(uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t pref } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); diff --git a/modules/ip6/control/ip6.h b/modules/ip6/control/ip6.h index 83fb9c975..20b4b33af 100644 --- a/modules/ip6/control/ip6.h +++ b/modules/ip6/control/ip6.h @@ -68,9 +68,12 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ); -void rib6_cleanup(struct nexthop *); +void rib6_cleanup(struct nexthop *, bool notify); +// delete the routes of an interface, except those owned by a routing daemon +void rib6_cleanup_iface(uint16_t iface_id, bool notify); struct nexthop *rib6_lookup(uint16_t vrf_id, uint16_t iface_id, const struct rte_ipv6_addr *); struct nexthop *rib6_lookup_exact( uint16_t vrf_id, diff --git a/modules/ip6/control/route.c b/modules/ip6/control/route.c index a775b8ffe..c0f7adb71 100644 --- a/modules/ip6/control/route.c +++ b/modules/ip6/control/route.c @@ -312,7 +312,8 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ) { struct rte_fib6 *fib = get_fib6(vrf_id); const struct rte_ipv6_addr *scoped_ip; @@ -344,7 +345,7 @@ int rib6_delete( if ((ret = rte_fib6_delete(fib, scoped_ip, prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP6_ROUTE_DEL, &(const struct route6_event) { @@ -438,7 +439,8 @@ static struct api_out route6_del(const void *request, struct api_ctx *) { GR_IFACE_ID_UNDEF, &req->dest.ip, req->dest.prefixlen, - nh ? nh->type : GR_NH_T_L3 + nh ? nh->type : GR_NH_T_L3, + true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -579,6 +581,7 @@ struct rib6_cleanup_entry { struct rib6_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib6_cleanup_entry *entries; }; @@ -586,12 +589,21 @@ static int rib6_cleanup_cb( uint16_t vrf_id, const struct rte_ipv6_addr *ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib6_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib6_cleanup_entry entry = { .vrf_id = vrf_id, .iface_id = nh->iface_id, @@ -604,21 +616,35 @@ static int rib6_cleanup_cb( return 0; } -void rib6_cleanup(struct nexthop *nh) { - struct rib6_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx, bool notify) { struct rib6_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib6_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib6_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx->entries) + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, notify); + vec_free(ctx->entries); +} + +void rib6_cleanup(struct nexthop *nh, bool notify) { + struct rib6_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib6_cleanup_run(&ctx, notify); +} + +void rib6_cleanup_iface(uint16_t iface_id, bool notify) { + struct rib6_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib6_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib6_routes", "Number of IPv6 routes by origin."); @@ -884,7 +910,7 @@ static void fib6_fini(struct iface *vrf) { rib6_iter_vrf(rte_fib6_get_rib(fib), vrf->id, &iter); vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib6 = NULL; @@ -899,7 +925,7 @@ static struct api_out fib6_default_set(const void *request, struct api_ctx *) { const struct gr_ip6_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv6 default max_routes %u -> %u", max_routes_default, req->max_routes); @@ -915,6 +941,18 @@ static const struct vrf_fib_ops fib6_ops = { .fini = fib6_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + if (!gr_config.flush_routes_on_iface_down) + return; + + rib6_cleanup_iface(iface->id, !gr_config.skip_route_events_on_iface_down); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP6_ROUTE_ADD, route6_add); api_handler(GR_IP6_ROUTE_DEL, route6_del); @@ -927,4 +965,6 @@ RTE_INIT(control_ip_init) { module_register(&route6_module); metrics_register(&rib6_collector); vrf_fib_ops_register(GR_AF_IP6, &fib6_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } diff --git a/modules/policy/api/dnat44.c b/modules/policy/api/dnat44.c index 2c8f00903..ee56f0d1c 100644 --- a/modules/policy/api/dnat44.c +++ b/modules/policy/api/dnat44.c @@ -176,7 +176,7 @@ static struct api_out dnat44_del(const void *request, struct api_ctx *) { if (iface == NULL) return api_out(ENODEV, 0, NULL); - ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT); + ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT, true); if (ret == -ENOENT && req->missing_ok) ret = 0; diff --git a/smoke/addr_del_kernel_route_frr_test.sh b/smoke/addr_del_kernel_route_frr_test.sh new file mode 100755 index 000000000..773924089 --- /dev/null +++ b/smoke/addr_del_kernel_route_frr_test.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface whose addresses +# are deleted. +# +# The addresses and routes are configured with grcli on purpose: an address +# configured in FRR keeps its connected entry flagged ZEBRA_IFC_CONFIGURED, +# and zebra would not consider the interface address-less. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.100.2 +gw6=fd00:ba4::2 + +# assert_agree +# +# Check that grout and zebra hold the same view of , and that this +# view is the expected one. Retries to let the notification propagate. +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +# Addresses and routes owned by grout: zebra learns them through the +# plugin, so the connected entries are not flagged ZEBRA_IFC_CONFIGURED +# and really leave ifp->connected when deleted. +grcli address add 192.168.0.1/24 iface p0 +grcli address add 192.168.100.1/24 iface p0 +grcli address add fd00:f00::1/64 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Deleting an address which is not the last one of its family changes +# nothing. +grcli address del 192.168.0.1/24 iface p0 +grcli address del fd00:f00::1/64 iface p0 + +assert_agree $prefix4 yes "after deleting a non-last address" +assert_agree $prefix6 yes "after deleting a non-last address" + +# Deleting the last IPv6 address keeps both routes: the interface still +# has its link-local address to source neighbor solicitations from. +grcli address del fd00:ba4::1/64 iface p0 + +assert_agree $prefix6 yes "after deleting the last IPv6 address" +assert_agree $prefix4 yes "after deleting the last IPv6 address" + +# Deleting the last IPv4 address flushes the IPv4 routes of the +# interface, and only those. +grcli address del 192.168.100.1/24 iface p0 + +assert_agree $prefix4 no "after deleting the last IPv4 address" +assert_agree $prefix6 yes "after deleting the last IPv4 address" + +true diff --git a/smoke/iface_down_kernel_route_frr_test.sh b/smoke/iface_down_kernel_route_frr_test.sh new file mode 100755 index 000000000..9a49a5200 --- /dev/null +++ b/smoke/iface_down_kernel_route_frr_test.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface which goes +# administratively down, and both must leave them alone on carrier loss. +# +# The addresses and routes are configured with grcli on purpose, see +# addr_del_kernel_route_frr_test.sh. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.0.2 +gw6=fd00:ba4::2 + +# assert_agree +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +grcli address add 192.168.0.1/24 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +# The plugin enables the flush when it connects to grout. +grcli -j interface config show | jq -e .flush_routes_on_iface_down || + fail "the plugin did not enable flush-routes-on-iface-down" + +# The knob must also be reachable from the CLI, the plugin is not the only +# writer. Put it back on afterwards, the rest of the test needs the flush. +grcli interface config set flush-routes-on-iface-down off +grcli -j interface config show | jq -e '.flush_routes_on_iface_down == false' || + fail "interface config set off had no effect" +grcli interface config set flush-routes-on-iface-down on +grcli -j interface config show | jq -e '.flush_routes_on_iface_down == true' || + fail "interface config set on had no effect" + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Carrier loss only: the interface stays up, nothing is flushed. +ip -n n0 link set x-p0 down + +assert_agree $prefix4 yes "after carrier loss" +assert_agree $prefix6 yes "after carrier loss" + +ip -n n0 link set x-p0 up + +# Administratively down: both families are flushed on both sides. +grcli interface set port p0 down + +assert_agree $prefix4 no "after setting p0 down" +assert_agree $prefix6 no "after setting p0 down" + +true