From d63e16d25bdd0a28a3d03d057d2a6644f65a8241 Mon Sep 17 00:00:00 2001 From: Simone Scarduzio Date: Sat, 12 Sep 2026 13:45:32 +0000 Subject: [PATCH 1/5] Take a restarting Kibana replica out of the proxy rotation The docker env runs two Kibana replicas behind kbn-proxy. The upstream block had no connect timeout and no failover, so while one replica restarted - which Kibana-config.cy.ts makes it do, and which `restart: always` on kbn-ror exists for - half of every request went to a container that was not listening. A stopped container drops the packets rather than refusing them, so those requests hung instead of failing. nginx waited its default 60s connect timeout, which is longer than every Cypress timeout, so the browser gave up first, the failure never counted against the peer, and the peer stayed in the rotation. Cypress retries did not help: all three attempts hit the same alternation. Measured on a two-replica reproduction with one replica stopped. before: 000 200 000 200 000 200 000 200 000 200 (50% lost, each hanging to the client's limit) after: 200/2.00s 200/0.0005s 200/0.0005s ... (one failover, then the peer is out) Sustained for 30s with the peer down: 110 requests, 0 failures, 3 over 0.5s (the 10s re-probe), worst 2.0s - inside every Cypress timeout. The replica returns to the rotation on its own when it answers again. This fits the evidence: the failures only ever appear on the `docker` legs, never on `eck`, and eck runs a single Kibana (kind-cluster/ror/base/kbn.yml, count: 1) with no proxy in front. Co-Authored-By: Claude Opus 5 --- .../elk-ror/conf/kbn/kbn-proxy-nginx.conf | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf index ad5a9b3e..183fb4ad 100644 --- a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf +++ b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf @@ -1,8 +1,12 @@ events { } http { + # `kbn-ror` resolves to both Kibana replicas, so this one line becomes two round-robin peers. + # max_fails/fail_timeout are the defaults, written out because the whole point of this block is + # that a peer which stops answering leaves the rotation: one failure ejects it for 10 seconds, + # then nginx probes it again and takes it back when it answers. upstream kbn-ror { - server kbn-ror:5601; + server kbn-ror:5601 max_fails=1 fail_timeout=10s; } server { @@ -22,6 +26,21 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_ssl_verify off; # Disable SSL verification for internal requests (only if Kibana uses self-signed certs) + + # A Kibana replica that is restarting drops the packets instead of refusing them, so a + # connection to it hangs rather than failing. Without a connect timeout nginx waits the + # default 60 seconds, which is longer than every Cypress timeout, so the browser gives up + # first and the failure never reaches nginx to be counted against the peer. Two seconds is + # far above a healthy connect on the container network and far below the 10s Cypress spends + # on cy.wait(). + proxy_connect_timeout 2s; + + # Send the request to the other replica when this one cannot take it. http_503 is here + # because a Kibana that is listening but still starting answers 503, which is not an error + # to nginx by default. Two tries, because there are two replicas. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; } } } \ No newline at end of file From 12de3bec1765a3e756055c47ac238a89171eb958 Mon Sep 17 00:00:00 2001 From: Simone Scarduzio Date: Sat, 12 Sep 2026 21:33:34 +0000 Subject: [PATCH 2/5] Never eject a Kibana replica; fail over per request instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version of this change set max_fails=1 fail_timeout=10s, and it made the suite worse: the 9.5.3 docker leg failed 10 of 27 specs, every one of them on `502: Bad Gateway`, with `no live upstreams while connecting to upstream` in the nginx log. Ejection cannot work here. There are two peers, and a settings change restarts Kibana, so BOTH answer 503 for a few seconds. One 503 each ejects both, nginx has no peer left, and every request gets a 502 — including for ten seconds after the replicas are healthy again. Measured on a two-replica reproduction with controllable backends, 24 requests per case: scenario max_fails=1 max_fails=0 one peer stopped 24 ok 24 ok one peer answering 503 24 ok 24 ok both peers 503 0 ok, 23x 502, 1x 503 0 ok, 24x 503 both recover first 200 at 10.49s first 200 at 0.01s http_503 stays in proxy_next_upstream. Taking it out looked like the fix at first, and it is not: with one replica restarting and the other healthy it turned 0 failures into 12 of 24, because the 503 is then served to the client instead of being retried on the peer that can answer. What is left is failover without ejection. A peer that cannot take a request costs that request one failover, and a peer that recovers is used again on the next request. Co-Authored-By: Claude Opus 5 --- .../elk-ror/conf/kbn/kbn-proxy-nginx.conf | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf index 183fb4ad..9d1f98c2 100644 --- a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf +++ b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf @@ -2,11 +2,21 @@ events { } http { # `kbn-ror` resolves to both Kibana replicas, so this one line becomes two round-robin peers. - # max_fails/fail_timeout are the defaults, written out because the whole point of this block is - # that a peer which stops answering leaves the rotation: one failure ejects it for 10 seconds, - # then nginx probes it again and takes it back when it answers. + # + # max_fails=0 turns ejection off. The default, max_fails=1 fail_timeout=10s, cannot work with two + # peers and a suite that restarts Kibana: a settings change makes BOTH replicas answer 503 for a + # few seconds, one 503 each ejects both, and nginx then has no peer left and answers 502 — to + # every request, including for ten seconds after the replicas are healthy again. + # + # Measured on a two-replica reproduction, both peers answering 503 and then recovering: + # + # max_fails=1 fail_timeout=10s 502 "no live upstreams", first 200 at 10.49s after recovery + # max_fails=0 503 passed through, first 200 at 0.01s after recovery + # + # With ejection off, a peer that cannot take a request costs that request one failover and + # nothing more, and a peer that recovers is used again immediately. upstream kbn-ror { - server kbn-ror:5601 max_fails=1 fail_timeout=10s; + server kbn-ror:5601 max_fails=0; } server { @@ -38,6 +48,9 @@ http { # Send the request to the other replica when this one cannot take it. http_503 is here # because a Kibana that is listening but still starting answers 503, which is not an error # to nginx by default. Two tries, because there are two replicas. + # + # Keeping http_503 matters: with one replica restarting and the other healthy, retrying the + # 503 elsewhere turned 12 failures in 24 requests into 0 on the same reproduction. proxy_next_upstream error timeout http_502 http_503 http_504; proxy_next_upstream_tries 2; proxy_next_upstream_timeout 5s; From 664fdba0ddc1d7fc198d099ad9105b579f073126 Mon Sep 17 00:00:00 2001 From: Simone Scarduzio Date: Sun, 13 Sep 2026 11:47:58 +0000 Subject: [PATCH 3/5] DIAGNOSTIC: print container events and Kibana shutdown lines when the suite fails Temporary, to be reverted. On the red 9.4.6 docker legs the kbn-ror containers restart with exit code 0 during Tenancy, Test-settings and User-settings; on the green legs they never restart. The EXIT trap tears the stack down before any later step can look, so this prints `docker events` and the shutdown-related Kibana log lines from inside the failure path. Co-Authored-By: Claude Fable 5.1 --- runner.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/runner.sh b/runner.sh index e599707e..9c8bc428 100755 --- a/runner.sh +++ b/runner.sh @@ -150,7 +150,24 @@ time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK if [[ "$MODE" == "e2e" ]]; then echo -e "Running E2E tests...\n" + # DIAGNOSTIC (temporary, this PR only): the kbn-ror containers restart mid-suite with exit code 0 + # on the red legs and never on the green ones. The EXIT trap tears the stack down, so the only + # moment to ask docker who stopped them, and what Kibana logged before it went, is right here. + SUITE_START=$(date +%s) + set +e time ./e2e-tests/run-tests.sh "$ELK_VERSION" "$ENV_NAME" + E2E_STATUS=$? + set -e + if [[ $E2E_STATUS -ne 0 ]]; then + echo "=== DIAG: container events since the suite started ===" + docker events --since "$SUITE_START" --until "$(date +%s)" --filter type=container \ + --format '{{.Time}} {{.Actor.Attributes.name}} {{.Action}} exit={{index .Actor.Attributes "exitCode"}} signal={{index .Actor.Attributes "signal"}}' || true + for c in $(docker ps -a --filter 'name=^elk-ror-kbn' --format '{{.Names}}'); do + echo "=== DIAG: $c — last lines around shutdowns and listens ===" + docker logs --tail 600 "$c" 2>&1 | grep -inE 'sigint|sigterm|shutdown|shutting|stopping|fatal|heap|out of memory|listening|http server running|server running|EADDRINUSE|closed|deleteAllSessions|license|edition|config refresh|restart|unhandled|uncaught' | tail -100 || true + done + fi + exit $E2E_STATUS else echo -e "Bootstrap mode: Cluster setup completed.\n" fi \ No newline at end of file From 0e77dfc4f2fe0ed9c53d754e62fd07db72340f5a Mon Sep 17 00:00:00 2001 From: Simone Scarduzio Date: Sun, 13 Sep 2026 13:28:09 +0000 Subject: [PATCH 4/5] Revert "DIAGNOSTIC: print container events and Kibana shutdown lines when the suite fails" This reverts commit 664fdba0ddc1d7fc198d099ad9105b579f073126. --- runner.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/runner.sh b/runner.sh index 9c8bc428..e599707e 100755 --- a/runner.sh +++ b/runner.sh @@ -150,24 +150,7 @@ time ./environments/$ENV_NAME/start.sh --cluster-type "$CLUSTER_TYPE" --es "$ELK if [[ "$MODE" == "e2e" ]]; then echo -e "Running E2E tests...\n" - # DIAGNOSTIC (temporary, this PR only): the kbn-ror containers restart mid-suite with exit code 0 - # on the red legs and never on the green ones. The EXIT trap tears the stack down, so the only - # moment to ask docker who stopped them, and what Kibana logged before it went, is right here. - SUITE_START=$(date +%s) - set +e time ./e2e-tests/run-tests.sh "$ELK_VERSION" "$ENV_NAME" - E2E_STATUS=$? - set -e - if [[ $E2E_STATUS -ne 0 ]]; then - echo "=== DIAG: container events since the suite started ===" - docker events --since "$SUITE_START" --until "$(date +%s)" --filter type=container \ - --format '{{.Time}} {{.Actor.Attributes.name}} {{.Action}} exit={{index .Actor.Attributes "exitCode"}} signal={{index .Actor.Attributes "signal"}}' || true - for c in $(docker ps -a --filter 'name=^elk-ror-kbn' --format '{{.Names}}'); do - echo "=== DIAG: $c — last lines around shutdowns and listens ===" - docker logs --tail 600 "$c" 2>&1 | grep -inE 'sigint|sigterm|shutdown|shutting|stopping|fatal|heap|out of memory|listening|http server running|server running|EADDRINUSE|closed|deleteAllSessions|license|edition|config refresh|restart|unhandled|uncaught' | tail -100 || true - done - fi - exit $E2E_STATUS else echo -e "Bootstrap mode: Cluster setup completed.\n" fi \ No newline at end of file From be30543a0b7bdc0b9dd0199c160ac9aec930c75e Mon Sep 17 00:00:00 2001 From: Simone Scarduzio Date: Mon, 14 Sep 2026 13:49:26 +0000 Subject: [PATCH 5/5] Rewrite the kbn-proxy nginx comments to the code-style rule Each comment states what the directive does and why the value is set. Measurements and change history stay in the PR description. Co-Authored-By: Claude Opus 5 (1M context) --- .../elk-ror/conf/kbn/kbn-proxy-nginx.conf | 38 ++++++------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf index 9d1f98c2..006d307b 100644 --- a/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf +++ b/environments/elk-ror/conf/kbn/kbn-proxy-nginx.conf @@ -1,20 +1,10 @@ events { } http { - # `kbn-ror` resolves to both Kibana replicas, so this one line becomes two round-robin peers. - # - # max_fails=0 turns ejection off. The default, max_fails=1 fail_timeout=10s, cannot work with two - # peers and a suite that restarts Kibana: a settings change makes BOTH replicas answer 503 for a - # few seconds, one 503 each ejects both, and nginx then has no peer left and answers 502 — to - # every request, including for ten seconds after the replicas are healthy again. - # - # Measured on a two-replica reproduction, both peers answering 503 and then recovering: - # - # max_fails=1 fail_timeout=10s 502 "no live upstreams", first 200 at 10.49s after recovery - # max_fails=0 503 passed through, first 200 at 0.01s after recovery - # - # With ejection off, a peer that cannot take a request costs that request one failover and - # nothing more, and a peer that recovers is used again immediately. + # `kbn-ror` resolves to both Kibana replicas, so this one line is two round-robin peers. + # max_fails=0 keeps a peer in rotation after a failed request. A settings change makes both + # replicas answer 503 at once. With the default max_fails=1, that ejects both peers and nginx + # answers 502 for the next ten seconds, also after the replicas recover. upstream kbn-ror { server kbn-ror:5601 max_fails=0; } @@ -37,23 +27,17 @@ http { proxy_set_header X-Forwarded-Proto $scheme; proxy_ssl_verify off; # Disable SSL verification for internal requests (only if Kibana uses self-signed certs) - # A Kibana replica that is restarting drops the packets instead of refusing them, so a - # connection to it hangs rather than failing. Without a connect timeout nginx waits the - # default 60 seconds, which is longer than every Cypress timeout, so the browser gives up - # first and the failure never reaches nginx to be counted against the peer. Two seconds is - # far above a healthy connect on the container network and far below the 10s Cypress spends - # on cy.wait(). + # A restarting Kibana replica drops packets, so a connection to it hangs instead of failing. + # The default 60s is longer than every Cypress timeout. 2s is far above a healthy connect + # on the container network and below the 10s cy.wait() timeout. proxy_connect_timeout 2s; - # Send the request to the other replica when this one cannot take it. http_503 is here - # because a Kibana that is listening but still starting answers 503, which is not an error - # to nginx by default. Two tries, because there are two replicas. - # - # Keeping http_503 matters: with one replica restarting and the other healthy, retrying the - # 503 elsewhere turned 12 failures in 24 requests into 0 on the same reproduction. + # Retry on the other replica when this one cannot take the request. A Kibana that is + # listening but still starting answers 503, so http_503 is a retry condition too. + # Two tries, one per replica. 5s covers two 2s connect timeouts and stays below cy.wait(). proxy_next_upstream error timeout http_502 http_503 http_504; proxy_next_upstream_tries 2; proxy_next_upstream_timeout 5s; } } -} \ No newline at end of file +}