From a1f4cc495e2b8fb69326b106a9e07b01e8341958 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 5 Sep 2026 07:39:38 +0200 Subject: [PATCH 01/40] Add spec parameter to cypress run script --- e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts | 1 - e2e-tests/package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts b/e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts index b24a6e55..a3bc48ff 100644 --- a/e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts +++ b/e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts @@ -4,7 +4,6 @@ * Written by Beshu Limited in London, UK */ -import { Login } from '../support/page-objects/Login'; import { rorApiClient } from '../support/helpers/RorApiClient'; // api_only users — allowed_api_paths enforcement is active diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 6d155870..32d8fc56 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -7,7 +7,7 @@ "lint": "eslint .", "lint:fix": "yarn lint -- --fix", "open": "./node_modules/.bin/cypress open", - "run": "ELECTRON_ENABLE_LOGGING=1 ELECTRON_EXTRA_LAUNCH_ARGS='--ignore-gpu-blocklist' ./node_modules/.bin/cypress run" + "run": "ELECTRON_ENABLE_LOGGING=1 ELECTRON_EXTRA_LAUNCH_ARGS='--ignore-gpu-blocklist' ./node_modules/.bin/cypress run --spec \"${SPEC:-cypress/e2e/**/*.cy.ts}\"" }, "license": "Beshu Limited, All rights reserved", "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e", From 82dc444ab7b241cb9c74b485ad169f83a2e747d9 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 8 Sep 2026 05:02:28 +0200 Subject: [PATCH 02/40] Add propagation delay and timeouts to E2E tests --- e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts | 6 +++--- e2e-tests/cypress/support/helpers/RorApiClient.ts | 14 +++++++++++++- e2e-tests/cypress/support/helpers/index.ts | 8 ++++++-- e2e-tests/cypress/support/page-objects/Settings.ts | 4 ++++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts index 9567d8ed..2e007086 100644 --- a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts +++ b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts @@ -87,7 +87,7 @@ describe('Readonlyrest-settings', () => { cy.reload(); - cy.get('h1').shouldHaveStyle('color', 'rgb(0,128,0)'); + cy.get('h1', { timeout: 30000 }).shouldHaveStyle('color', 'rgb(0,128,0)'); }); it('should verify custom Kibana JS', () => { @@ -104,7 +104,7 @@ describe('Readonlyrest-settings', () => { cy.reload(); - cy.get('[data-testid="metadata-alert-message"]') + cy.get('[data-testid="metadata-alert-message"]', { timeout: 30000 }) .should('exist') .then($el => { cy.log(`Alert message: ${$el.text()}`); @@ -139,7 +139,7 @@ describe('Readonlyrest-settings', () => { cy.reload(); - cy.get('[data-testid="metadata-enriched-data"]') + cy.get('[data-testid="metadata-enriched-data"]', { timeout: 30000 }) .should('exist') .then($el => { cy.log(`Entiched data: ${$el.text()}`); diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index 30629e11..e4024218 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -1,3 +1,11 @@ +// The ROR Kibana plugin picks up a new index-stored config asynchronously - each kbn-ror node +// only refreshes its in-memory settings when its own cache goes stale, not the instant the POST +// below returns. Proceeding immediately (e.g. straight into Login.initialization()) races that +// refresh: observed lag between a successful POST and the new config being active was up to ~10s. +// See run-20260831-073804-1.log for the flaky "should disable multitenancy" / "should verify index +// based session" failures this caused. +const SETTINGS_PROPAGATION_DELAY_MS = 8000; + export class RorApiClient { public configureRorIndexMainSettings(yamlContent: string): Cypress.Chainable { return cy @@ -13,9 +21,13 @@ export class RorApiClient { // The endpoint no-ops (status: FAILURE) when the posted content is already the active // config - e.g. two specs in a row both resetting to the same default fixture. That's // the desired state, not an error; only a genuinely different failure should throw. - if (response.status !== 'SUCCESS' && response.message !== 'Current settings are already loaded') { + if (response.status === 'SUCCESS') { + return cy.wait(SETTINGS_PROPAGATION_DELAY_MS); + } + if (response.message !== 'Current settings are already loaded') { throw new Error(`Failed to configure ROR index main settings: ${JSON.stringify(response)}`); } + return undefined; }) .then(() => undefined); } diff --git a/e2e-tests/cypress/support/helpers/index.ts b/e2e-tests/cypress/support/helpers/index.ts index c00cf347..0d263b76 100644 --- a/e2e-tests/cypress/support/helpers/index.ts +++ b/e2e-tests/cypress/support/helpers/index.ts @@ -1,8 +1,12 @@ +import * as semver from 'semver'; + export const getKibanaVersion = () => { const kibanaVersion: string = Cypress.env('kibanaVersion'); console.log('kibana version', kibanaVersion); - if (!kibanaVersion) { - throw new Error('Kibana version not specified in the config file'); + if (!kibanaVersion || !semver.valid(kibanaVersion)) { + throw new Error( + `Kibana version not specified correctly (got "${kibanaVersion}"). Pass it via --env kibanaVersion=.` + ); } return kibanaVersion; diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index 11a893d9..733d3f88 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -105,6 +105,10 @@ export class Settings { ...(yaml.load(esYamlSettings) as object), readonlyrest_kbn: { cookiePass: '12312313123213123213123adadasdasdasd', + // elk-ror runs 2 kbn-ror replicas behind kbn-proxy's round robin. Without index-backed + // sessions, each node keeps sessions in memory, so a login on one replica isn't + // recognized by the other and the next request bounces back to /login. + store_sessions_in_index: true, ...parseKbnSettings(readonlyRestKbnSettings) } }; From 06778f6aca57f32ed3a897eb18c30d64bcd3f4f1 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 8 Sep 2026 07:15:41 +0200 Subject: [PATCH 03/40] Add connection pooling and transport error retries to e2e fetch Use a shared HTTPS agent with keep-alive enabled to avoid dropped connections in the eck-ror CI environment, and add automatic retries for transient network errors like ECONNRESET and socket hang ups. --- e2e-tests/cypress/plugins/index.ts | 108 +++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 20 deletions(-) diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index fb3339ef..da63616c 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -7,6 +7,19 @@ import { inspect } from 'util'; import path from 'node:path'; import * as fs from 'node:fs'; +// Shared and kept alive across calls so requests reuse an established TCP+TLS connection instead +// of each `httpCall`/`uploadFile` negotiating a brand-new one. In the eck-ror CI environment, +// Kibana is reached through kind's NodePort/iptables overlay rather than a direct docker port +// mapping, and that extra hop is where the many short-lived connections a fresh Agent-per-call +// created were most likely to get dropped or hang. +const sharedHttpsAgent: Agent = new Agent({ + rejectUnauthorized: false, + secureProtocol: 'TLSv1_2_method', + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 50 +}); + let embeddedServer: ReturnType | null = null; const EMBEDDED_SERVER_PORT = 8080; const ROOT_DIR = path.join(__dirname, '..', '..', '..'); @@ -32,6 +45,30 @@ const formatLoggerData = (data: unknown) => const NON_JSON_RETRY_ATTEMPTS = 5; const NON_JSON_RETRY_DELAY_MS = 2000; +// The eck-ror CI environment reaches Kibana through kind's NodePort/iptables overlay instead of +// a direct docker port mapping, which occasionally drops or hangs a TCP connection outright +// (ECONNRESET, socket hang up) rather than serving a slow-but-valid response. Without a retry +// here, a single dropped connection burns the whole cy.task timeout and, since Cypress only +// prints failures once the spec finishes, can silently take the rest of the spec down with it. +const TRANSPORT_ERROR_RETRY_ATTEMPTS = 3; +const TRANSPORT_ERROR_RETRY_DELAY_MS = 1000; +const TRANSIENT_NETWORK_ERROR_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EPIPE', + 'EHOSTUNREACH', + 'ENETUNREACH' +]); + +const isTransientNetworkError = (error: unknown): boolean => { + const err = error as { code?: string; message?: string }; + if (err?.code && TRANSIENT_NETWORK_ERROR_CODES.has(err.code)) { + return true; + } + return typeof err?.message === 'string' && err.message.includes('socket hang up'); +}; + const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); // Right after a Kibana restart, ROR-KBN can still be finishing its own settings load (an ES @@ -42,11 +79,19 @@ const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(r // the caller to read exactly as before. // `createInit` is a factory (not a static object) because a retried attempt needs its own // request body - a FormData upload's underlying stream can only be read once. -const fetchWithJsonRetry = async (url: string, createInit: () => Parameters[1]): Promise => { +// +// `retryOnTransportError` is off for calls that intentionally expect the connection to be reset +// (e.g. /pkp/api/kibanaConfig SIGINTs Kibana before writing its reply) - those should fail fast +// into the caller's own handling instead of burning retries on an error that's the expected outcome. +const fetchWithJsonRetry = async ( + url: string, + createInit: () => Parameters[1], + retryOnTransportError = true +): Promise => { let response: Response; for (let attempt = 1; attempt <= NON_JSON_RETRY_ATTEMPTS; attempt++) { // eslint-disable-next-line no-await-in-loop - response = await fetch(url, createInit()); + response = await fetchWithTransportRetry(url, createInit, retryOnTransportError); const contentType = response.headers.get('content-type') || ''; // The startup race serves Kibana's login page (text/html) in place of the expected @@ -61,7 +106,9 @@ const fetchWithJsonRetry = async (url: string, createInit: () => Parameters Parameters Parameters[1], + retryOnTransportError: boolean +): Promise => { + for (let attempt = 1; attempt <= TRANSPORT_ERROR_RETRY_ATTEMPTS; attempt++) { + try { + // eslint-disable-next-line no-await-in-loop + return await fetch(url, createInit()); + } catch (error) { + const isLastAttempt = attempt === TRANSPORT_ERROR_RETRY_ATTEMPTS; + if (!retryOnTransportError || !isTransientNetworkError(error) || isLastAttempt) { + throw error; + } + console.log( + `Transient network error (${ + (error as Error).message + }) for ${url} - retrying (${attempt}/${TRANSPORT_ERROR_RETRY_ATTEMPTS})...` + ); + // eslint-disable-next-line no-await-in-loop + await sleep(TRANSPORT_ERROR_RETRY_DELAY_MS); + } + } + // Unreachable: the loop above always either returns or throws. + throw new Error(`Unreachable: exhausted retries for ${url} without returning or throwing`); +}; + module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { on('task', { async httpCall(options: HttpCallOptions): Promise { const { method, url, headers, body, failOnStatusCode, allowTransportError } = options; - const agent: Agent = new Agent({ - rejectUnauthorized: false, - secureProtocol: 'TLSv1_2_method' - }); - try { - const response: Response = await fetchWithJsonRetry(url, () => ({ - method, - headers, - body: body ?? undefined, - agent - })); + const response: Response = await fetchWithJsonRetry( + url, + () => ({ + method, + headers, + body: body ?? undefined, + agent: sharedHttpsAgent + }), + !allowTransportError + ); if (!response.ok && failOnStatusCode) { throw new Error( @@ -122,11 +195,6 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) async uploadFile(options: UploadFileOptions): Promise { const { url, headers, file } = options; - const agent: Agent = new Agent({ - rejectUnauthorized: false, - secureProtocol: 'TLSv1_2_method' - }); - const buildForm = (): { form: FormData; combinedHeaders: { [key: string]: string } } => { const form = new FormData(); form.append('file', file.fileBinaryContent, { @@ -142,7 +210,7 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) try { const response: Response = await fetchWithJsonRetry(url, () => { const { form, combinedHeaders } = buildForm(); - return { method, headers: combinedHeaders, body: form, agent }; + return { method, headers: combinedHeaders, body: form, agent: sharedHttpsAgent }; }); if (!response.ok) { From f55d292c1ee041133665a745e58cae246935396d Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 9 Sep 2026 06:48:44 +0200 Subject: [PATCH 04/40] Improve e2e test reliability and fix connection flakiness Replace static propagation delays with dynamic polling, destroy HTTPS agent pools on transport errors, adjust Kibana readiness probes, and add CPU/memory resource requests. --- .../defaultReadonlyRestEsAndKbnSettings.yaml | 1 - e2e-tests/cypress/plugins/index.ts | 5 +++ .../cypress/support/helpers/RorApiClient.ts | 37 ++++++++++++++++++- .../eck-ror/kind-cluster/ror/base/kbn.yml | 26 ++++++++++++- 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml index 2cbbc2eb..37a5e906 100644 --- a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml +++ b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml @@ -52,7 +52,6 @@ readonlyrest: kibana: <<: *common-kibana-rules access: admin - hide_apps: ['Enterprise Search|Overview', 'Management'] metadata: alert_message: 'Dear @{acl:user}' - name: infosec diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index da63616c..c1f9422f 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -136,6 +136,11 @@ const fetchWithTransportRetry = async ( (error as Error).message }) for ${url} - retrying (${attempt}/${TRANSPORT_ERROR_RETRY_ATTEMPTS})...` ); + // A transient error on one request can leave other keep-alive sockets in the shared pool + // half-broken too (the same kind NodePort hop dropped them all around the same time), and a + // retry that happens to grab one of those instead of opening a fresh connection fails the + // same way. Destroying the whole pool forces the retry onto a brand-new TCP+TLS connection. + sharedHttpsAgent.destroy(); // eslint-disable-next-line no-await-in-loop await sleep(TRANSPORT_ERROR_RETRY_DELAY_MS); } diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index e4024218..43774600 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -4,7 +4,13 @@ // refresh: observed lag between a successful POST and the new config being active was up to ~10s. // See run-20260831-073804-1.log for the flaky "should disable multitenancy" / "should verify index // based session" failures this caused. -const SETTINGS_PROPAGATION_DELAY_MS = 8000; +// +// Rather than blindly sleeping a fixed delay (too short under load, wastefully long otherwise), +// re-POST the same content and rely on the endpoint's own idempotency check: once the node has +// actually picked up the new config, re-posting it answers "Current settings are already loaded" +// instead of SUCCESS. That's the readiness signal we poll for. +const SETTINGS_PROPAGATION_POLL_INTERVAL_MS = 1000; +const SETTINGS_PROPAGATION_POLL_TIMEOUT_MS = 15000; export class RorApiClient { public configureRorIndexMainSettings(yamlContent: string): Cypress.Chainable { @@ -22,7 +28,7 @@ export class RorApiClient { // config - e.g. two specs in a row both resetting to the same default fixture. That's // the desired state, not an error; only a genuinely different failure should throw. if (response.status === 'SUCCESS') { - return cy.wait(SETTINGS_PROPAGATION_DELAY_MS); + return this.waitForSettingsPropagation(yamlContent); } if (response.message !== 'Current settings are already loaded') { throw new Error(`Failed to configure ROR index main settings: ${JSON.stringify(response)}`); @@ -32,6 +38,33 @@ export class RorApiClient { .then(() => undefined); } + private waitForSettingsPropagation(yamlContent: string, elapsedMs = 0): Cypress.Chainable { + return cy + .wait(SETTINGS_PROPAGATION_POLL_INTERVAL_MS) + .kbnPost<{ status: string; message: string }>({ + endpoint: 'api/ror/settings?override=true', + headers: { + 'Content-Type': 'application/yaml' + }, + credentials: Cypress.env().kibanaUserCredentials, + payload: yamlContent + }) + .then(response => { + if (response.message === 'Current settings are already loaded') { + return undefined; + } + + const nextElapsedMs = elapsedMs + SETTINGS_PROPAGATION_POLL_INTERVAL_MS; + if (nextElapsedMs >= SETTINGS_PROPAGATION_POLL_TIMEOUT_MS) { + throw new Error( + `Timed out after ${SETTINGS_PROPAGATION_POLL_TIMEOUT_MS}ms waiting for ROR index main settings to propagate` + ); + } + + return this.waitForSettingsPropagation(yamlContent, nextElapsedMs); + }); + } + public configureRorIndexMainSettingsFromFixture(fixtureYamlFileName: string): Cypress.Chainable { return cy.fixture(fixtureYamlFileName).then(yaml => this.configureRorIndexMainSettings(yaml)); } diff --git a/environments/eck-ror/kind-cluster/ror/base/kbn.yml b/environments/eck-ror/kind-cluster/ror/base/kbn.yml index 78bc2b9e..75569bcb 100644 --- a/environments/eck-ror/kind-cluster/ror/base/kbn.yml +++ b/environments/eck-ror/kind-cluster/ror/base/kbn.yml @@ -35,4 +35,28 @@ spec: - name: I_UNDERSTAND_AND_ACCEPT_KBN_PATCHING value: "yes" - name: NODE_OPTIONS - value: "--max-old-space-size=768" \ No newline at end of file + value: "--max-old-space-size=768" + # The default readiness probe (timeoutSeconds: 5, failureThreshold: 3) flips the pod + # NotReady under normal kind/CI CPU contention, which pulls it out of the Service's + # endpoints and turns any in-flight request into a bare TCP reset instead of a slow-but- + # valid response - the ECONNRESET flakiness seen in e2e runs traces back to this, not to + # the e2e HTTP client. Widening the tolerance keeps flaky-but-alive Kibana in rotation. + readinessProbe: + httpGet: + path: /api/status + port: 5601 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + failureThreshold: 6 + successThreshold: 1 + # No CPU limit is set deliberately (avoids throttling), but an explicit request gives + # Kibana a fairer scheduling share against ES/APM/kind's own control-plane pods sharing + # the same Docker Desktop CPU budget. + resources: + requests: + cpu: '500m' + memory: '2Gi' + limits: + memory: '2Gi' \ No newline at end of file From 7994db3836553c962b66ede2cac180a637729f24 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 9 Sep 2026 18:14:29 +0200 Subject: [PATCH 05/40] Fix E2E test stability and infrastructure flakiness Fixes multiple sources of flakiness in CI pipelines by updating test fixtures, improving HTTP retry logic, adjusting Kubernetes probes and resources, and fixing S3 artifact upload path handling. --- .../scripts/upload-cypress-artifacts-to-s3.sh | 6 + e2e-tests/cypress/e2e/Observability.cy.ts | 5 + .../fixtures/allowedApiPathsSettings.yaml | 1 + .../fixtures/hiddenAllAppsSettings.yaml | 1 + .../fixtures/hiddenHomePageSettings.yaml | 1 + .../hiddenSpaceManagementSettings.yaml | 1 + .../observabilityVisibleSettings.yaml | 127 ++++++++++++++++++ .../cypress/fixtures/reportingSettings.yaml | 1 + e2e-tests/cypress/fixtures/roSettings.yaml | 1 + .../cypress/fixtures/roStrictSettings.yaml | 1 + e2e-tests/cypress/plugins/index.ts | 15 ++- .../cypress/support/helpers/RorApiClient.ts | 43 ++---- .../eck-ror/kind-cluster/ror/base/kbn-np.yml | 6 + .../eck-ror/kind-cluster/ror/base/kbn.yml | 21 +-- environments/eck-ror/stop-and-clean.sh | 13 ++ 15 files changed, 199 insertions(+), 44 deletions(-) create mode 100644 e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml diff --git a/.github/scripts/upload-cypress-artifacts-to-s3.sh b/.github/scripts/upload-cypress-artifacts-to-s3.sh index c50861f3..f0e95a59 100755 --- a/.github/scripts/upload-cypress-artifacts-to-s3.sh +++ b/.github/scripts/upload-cypress-artifacts-to-s3.sh @@ -51,7 +51,13 @@ AK="${!AK_VAR}" SK="${!SK_VAR}" BUCKET="${!BUCKET_VAR}" REGION="${!REGION_VAR}" +# Strip any trailing slash regardless of how the caller formatted it - upload-videos/action.yml +# builds this from "/build_" and if path_prefix itself already ends in "/" +# (as it did for the DGP endpoint), the unstripped value produces a "//" in every S3 key below, +# which the endpoint rejects outright: "InvalidArgument: Key must not contain empty path +# segments ('//')" - the cause of every artifact upload failing in run 34312427155. PATH_PREFIX="${!PREFIX_VAR:-}" +PATH_PREFIX="${PATH_PREFIX%/}" SOURCE_DIR="${1:?Usage: upload-cypress-artifacts-to-s3.sh }" S3_SUBFOLDER="${2:?Usage: upload-cypress-artifacts-to-s3.sh }" diff --git a/e2e-tests/cypress/e2e/Observability.cy.ts b/e2e-tests/cypress/e2e/Observability.cy.ts index 8e46f4d0..b2410c4f 100644 --- a/e2e-tests/cypress/e2e/Observability.cy.ts +++ b/e2e-tests/cypress/e2e/Observability.cy.ts @@ -1,17 +1,22 @@ import { Login } from '../support/page-objects/Login'; import { KibanaNavigation } from '../support/page-objects/KibanaNavigation'; import { Observability } from '../support/page-objects/Observability'; +import { Settings } from '../support/page-objects/Settings'; import { esApiClient } from '../support/helpers/EsApiClient'; import * as semver from 'semver'; import { getKibanaVersion } from '../support/helpers'; describe('Observability', () => { beforeEach(() => { + // The shared default fixture hides the Observability app (hide_apps), so this spec needs + // its own fixture with that app left visible instead of relying on the shared default. + Settings.setSettingsData('observabilityVisibleSettings.yaml'); Login.initialization(); }); afterEach(() => { esApiClient.deleteIndexDocsByQuery(Observability.APM_DATA_INDEXES_WILDCARD); + Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); }); it('should verify APM functionality', () => { diff --git a/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml b/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml index 2e7ea844..b248a99e 100644 --- a/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml +++ b/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml @@ -60,3 +60,4 @@ readonlyrest: http_path: '^/api/ror/user/tenants$' readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml b/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml index 142cc3c8..ab593443 100644 --- a/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml @@ -18,3 +18,4 @@ readonlyrest: name: administrators readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml b/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml index c40f315e..a9e3d23e 100644 --- a/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml @@ -18,3 +18,4 @@ readonlyrest: name: administrators readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml b/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml index 4fcd1d2a..a320a23c 100644 --- a/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml @@ -17,3 +17,4 @@ readonlyrest: name: administrators readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml new file mode 100644 index 00000000..48dc3517 --- /dev/null +++ b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml @@ -0,0 +1,127 @@ +helpers: + ckr: &common-kibana-rules + access: rw + hide_apps: ['Enterprise Search|Overview'] + index: '.kibana_@{acl:current_group}' + + ag: &all-groups + groups: + - id: admins_group + name: administrators + - id: infosec_group + name: infosec + - id: template_group + name: template + +readonlyrest: + response_if_req_forbidden: You shall not pass! + audit: + enabled: true + outputs: + - type: index + index_template: "'readonlyrest_audit_'yyyy-MM-dd" + + access_control_rules: + - name: 'Kibana service account - user/pass' + verbosity: error + auth_key: kibana:kibana + + - name: JWT_AUTH + jwt_auth: + name: 'jwt1' + groups_any_of: ['administrators', 'infosec', 'template'] + kibana: + access: admin + + - name: USER_DEFAULT + auth_key: user2:dev + verbosity: error + indices: ['kibana_sample_data_*'] + kibana: + access: rw + index: '.default_index' + + - name: PERSONAL_GRP + groups: [Personal] + kibana: + <<: *common-kibana-rules + index: '.kibana_@{user}' + + - name: ADMIN_GRP + groups: [admins_group] + kibana: + <<: *common-kibana-rules + access: admin + metadata: + alert_message: 'Dear @{acl:user}' + - name: infosec + groups: [infosec_group] + kibana: + <<: *common-kibana-rules + access: admin + hide_apps: ['Enterprise Search|Overview', 'Observability', 'Management'] + + - name: Template Tenancy + groups: [template_group] + kibana: + <<: *common-kibana-rules + + - name: 'ReadonlyREST Enterprise instance #1' + kibana_index: '.kibana_external_auth' + ror_kbn_auth: + name: 'kbn1' + + users: + - username: admin + auth_key: admin:dev + <<: *all-groups + + - username: user1 + auth_key: user1:dev + <<: *all-groups + + - username: '*' + jwt_auth: + name: 'jwt1' + groups: + - local_group: + id: admins_group + name: administrators + external_group_ids: ['administrators'] + - local_group: + id: infosec_group + name: infosec + external_group_ids: ['infosec'] + - local_group: + id: template_group + name: template + external_group_ids: ['template'] + + jwt: + - name: jwt1 + signature_key: 'a-string-secret-at-least-256-bits-long' + group_ids_claim: group + user_claim: sub + header_name: Authorization + + ror_kbn: + - name: kbn1 + signature_key: '9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf' + + impersonation: + - impersonator: admin + users: ['*'] + auth_key: admin:dev + +readonlyrest_kbn: + cookiePass: '12312313123213123213123adadasdasdasd' + logLevel: 'trace' + logPrettyPrintEnabled: true + whitelistedPaths: [".*/api/status$"] + clearSessionOnEvents: [login, tenancyHop] + sessions_probe_interval_seconds: 60 + store_sessions_in_index: true + login_title: Loaded from index! + login_subtitle: 'PRO/Enterprise: You should see a red border, a tiny unicorn logo, a two column page, and this text. You should see none of these customisation when testing ROR Free.' + login_custom_logo: 'https://i.imgur.com/MdRBUfV.gif' + login_html_head_inject: '' diff --git a/e2e-tests/cypress/fixtures/reportingSettings.yaml b/e2e-tests/cypress/fixtures/reportingSettings.yaml index d98cce6d..5bfb0d93 100644 --- a/e2e-tests/cypress/fixtures/reportingSettings.yaml +++ b/e2e-tests/cypress/fixtures/reportingSettings.yaml @@ -71,3 +71,4 @@ readonlyrest: auth_key: admin:dev readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/roSettings.yaml b/e2e-tests/cypress/fixtures/roSettings.yaml index 798e99a2..c5473a2e 100644 --- a/e2e-tests/cypress/fixtures/roSettings.yaml +++ b/e2e-tests/cypress/fixtures/roSettings.yaml @@ -71,3 +71,4 @@ readonlyrest: auth_key: admin:dev readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/fixtures/roStrictSettings.yaml b/e2e-tests/cypress/fixtures/roStrictSettings.yaml index 204193a6..b64a4cda 100644 --- a/e2e-tests/cypress/fixtures/roStrictSettings.yaml +++ b/e2e-tests/cypress/fixtures/roStrictSettings.yaml @@ -71,3 +71,4 @@ readonlyrest: auth_key: admin:dev readonlyrest_kbn: cookiePass: '12312313123213123213123adadasdasdasd' + store_sessions_in_index: true diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index c1f9422f..7566feea 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -52,6 +52,12 @@ const NON_JSON_RETRY_DELAY_MS = 2000; // prints failures once the spec finishes, can silently take the rest of the spec down with it. const TRANSPORT_ERROR_RETRY_ATTEMPTS = 3; const TRANSPORT_ERROR_RETRY_DELAY_MS = 1000; +// Without a per-request timeout, a socket that hangs instead of dropping outright (no +// ECONNRESET, just silence) burns the entire cy.task `taskTimeout` (20000ms) on its first +// attempt, so the retry loop above never even gets a chance to run. Capping each attempt well +// under a third of that budget guarantees all TRANSPORT_ERROR_RETRY_ATTEMPTS attempts (plus their +// TRANSPORT_ERROR_RETRY_DELAY_MS sleeps) fit inside taskTimeout even in the worst case. +const FETCH_TIMEOUT_MS = 5000; const TRANSIENT_NETWORK_ERROR_CODES = new Set([ 'ECONNRESET', 'ECONNREFUSED', @@ -62,10 +68,15 @@ const TRANSIENT_NETWORK_ERROR_CODES = new Set([ ]); const isTransientNetworkError = (error: unknown): boolean => { - const err = error as { code?: string; message?: string }; + const err = error as { code?: string; type?: string; message?: string }; if (err?.code && TRANSIENT_NETWORK_ERROR_CODES.has(err.code)) { return true; } + // node-fetch's own `timeout` option (set via FETCH_TIMEOUT_MS above) surfaces as a + // FetchError with type 'request-timeout' rather than one of the Node error codes above. + if (err?.type === 'request-timeout') { + return true; + } return typeof err?.message === 'string' && err.message.includes('socket hang up'); }; @@ -125,7 +136,7 @@ const fetchWithTransportRetry = async ( for (let attempt = 1; attempt <= TRANSPORT_ERROR_RETRY_ATTEMPTS; attempt++) { try { // eslint-disable-next-line no-await-in-loop - return await fetch(url, createInit()); + return await fetch(url, { timeout: FETCH_TIMEOUT_MS, ...createInit() }); } catch (error) { const isLastAttempt = attempt === TRANSPORT_ERROR_RETRY_ATTEMPTS; if (!retryOnTransportError || !isTransientNetworkError(error) || isLastAttempt) { diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index 43774600..df1061b2 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -5,12 +5,14 @@ // See run-20260831-073804-1.log for the flaky "should disable multitenancy" / "should verify index // based session" failures this caused. // -// Rather than blindly sleeping a fixed delay (too short under load, wastefully long otherwise), -// re-POST the same content and rely on the endpoint's own idempotency check: once the node has -// actually picked up the new config, re-posting it answers "Current settings are already loaded" -// instead of SUCCESS. That's the readiness signal we poll for. -const SETTINGS_PROPAGATION_POLL_INTERVAL_MS = 1000; -const SETTINGS_PROPAGATION_POLL_TIMEOUT_MS = 15000; +// A re-POST-and-check-for-idempotency poll was tried here instead of a flat sleep, but the +// idempotency check compares against the config already persisted in the ES index - which the +// first POST writes immediately - not against any single node's in-memory cache. Against the +// docker env's 2 kbn-ror replicas behind kbn-proxy's round robin, that made the "poll" resolve +// after its very first iteration (~1s) regardless of whether either replica had actually +// refreshed, which was worse than the flat delay it replaced. See run-34312427155 for the +// resulting "should verify index based session" / Hide_apps / Sanity-check flakiness. +const SETTINGS_PROPAGATION_DELAY_MS = 8000; export class RorApiClient { public configureRorIndexMainSettings(yamlContent: string): Cypress.Chainable { @@ -28,7 +30,7 @@ export class RorApiClient { // config - e.g. two specs in a row both resetting to the same default fixture. That's // the desired state, not an error; only a genuinely different failure should throw. if (response.status === 'SUCCESS') { - return this.waitForSettingsPropagation(yamlContent); + return cy.wait(SETTINGS_PROPAGATION_DELAY_MS); } if (response.message !== 'Current settings are already loaded') { throw new Error(`Failed to configure ROR index main settings: ${JSON.stringify(response)}`); @@ -38,33 +40,6 @@ export class RorApiClient { .then(() => undefined); } - private waitForSettingsPropagation(yamlContent: string, elapsedMs = 0): Cypress.Chainable { - return cy - .wait(SETTINGS_PROPAGATION_POLL_INTERVAL_MS) - .kbnPost<{ status: string; message: string }>({ - endpoint: 'api/ror/settings?override=true', - headers: { - 'Content-Type': 'application/yaml' - }, - credentials: Cypress.env().kibanaUserCredentials, - payload: yamlContent - }) - .then(response => { - if (response.message === 'Current settings are already loaded') { - return undefined; - } - - const nextElapsedMs = elapsedMs + SETTINGS_PROPAGATION_POLL_INTERVAL_MS; - if (nextElapsedMs >= SETTINGS_PROPAGATION_POLL_TIMEOUT_MS) { - throw new Error( - `Timed out after ${SETTINGS_PROPAGATION_POLL_TIMEOUT_MS}ms waiting for ROR index main settings to propagate` - ); - } - - return this.waitForSettingsPropagation(yamlContent, nextElapsedMs); - }); - } - public configureRorIndexMainSettingsFromFixture(fixtureYamlFileName: string): Cypress.Chainable { return cy.fixture(fixtureYamlFileName).then(yaml => this.configureRorIndexMainSettings(yaml)); } diff --git a/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml b/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml index f83644bb..3b848815 100644 --- a/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml +++ b/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml @@ -4,6 +4,12 @@ metadata: name: eck-ror-kbn-np spec: type: NodePort + # Keeps the pod's endpoint in the Service even while it's failing readiness. With only a + # single Kibana replica in this cluster, pulling the endpoint on every readiness flap (CPU + # contention on the kind node, GC pause, etc.) turns any in-flight request into a bare TCP + # reset (ECONNRESET) instead of a slow-but-valid response - that's the source of the + # widespread ECONNRESET flakiness seen in e2e runs, not the e2e HTTP client itself. + publishNotReadyAddresses: true ports: - port: 5601 targetPort: 5601 diff --git a/environments/eck-ror/kind-cluster/ror/base/kbn.yml b/environments/eck-ror/kind-cluster/ror/base/kbn.yml index 75569bcb..fc3b7daa 100644 --- a/environments/eck-ror/kind-cluster/ror/base/kbn.yml +++ b/environments/eck-ror/kind-cluster/ror/base/kbn.yml @@ -35,12 +35,16 @@ spec: - name: I_UNDERSTAND_AND_ACCEPT_KBN_PATCHING value: "yes" - name: NODE_OPTIONS - value: "--max-old-space-size=768" + # 768 was tight enough on its own to be a plausible contributor to the readiness + # flapping below (GC pressure blocking the event loop long enough to miss a probe). + # Raised in step with the memory limit increase below. + value: "--max-old-space-size=1536" # The default readiness probe (timeoutSeconds: 5, failureThreshold: 3) flips the pod - # NotReady under normal kind/CI CPU contention, which pulls it out of the Service's - # endpoints and turns any in-flight request into a bare TCP reset instead of a slow-but- - # valid response - the ECONNRESET flakiness seen in e2e runs traces back to this, not to - # the e2e HTTP client. Widening the tolerance keeps flaky-but-alive Kibana in rotation. + # NotReady under normal kind/CI CPU contention. `publishNotReadyAddresses` on the + # kbn-np Service (see kbn-np.yml) now stops that flapping from turning into a hard + # ECONNRESET at the network level, but the probe's own timing was still inverted - + # timeoutSeconds must be <= periodSeconds or a slow-but-alive response can be judged + # a failure before the next check even starts. readinessProbe: httpGet: path: /api/status @@ -48,15 +52,16 @@ spec: scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 15 + timeoutSeconds: 10 failureThreshold: 6 successThreshold: 1 # No CPU limit is set deliberately (avoids throttling), but an explicit request gives # Kibana a fairer scheduling share against ES/APM/kind's own control-plane pods sharing - # the same Docker Desktop CPU budget. + # the same Docker Desktop CPU budget. Memory limit raised alongside NODE_OPTIONS above - + # 2Gi left barely any headroom over the 768Mi heap cap for Kibana's own process memory. resources: requests: cpu: '500m' memory: '2Gi' limits: - memory: '2Gi' \ No newline at end of file + memory: '3Gi' \ No newline at end of file diff --git a/environments/eck-ror/stop-and-clean.sh b/environments/eck-ror/stop-and-clean.sh index 1866d3a8..865b392b 100755 --- a/environments/eck-ror/stop-and-clean.sh +++ b/environments/eck-ror/stop-and-clean.sh @@ -3,4 +3,17 @@ set -e cd "$(dirname "$0")" +# runner.sh runs this via `trap cleanup EXIT`, so it fires before the calling GitHub Actions step +# returns - including the "Stop Docker memory monitor" step, whose final log lines were always +# empty ("== kubectl pod status ==" with nothing after it) because `kind delete cluster` below had +# already torn the cluster down by the time that step's `kubectl` calls ran. Dumping the pod/event +# state here, one last time before deletion, keeps a non-empty final snapshot for post-mortem. +echo "== final pod status before teardown ==" +kubectl --request-timeout=10s get pods -A 2>/dev/null || true +echo "== final warning events before teardown ==" +kubectl --request-timeout=10s get events -A --field-selector=type=Warning \ + --sort-by=.lastTimestamp \ + -o custom-columns='LAST:.lastTimestamp,COUNT:.count,NS:.metadata.namespace,REASON:.reason,OBJECT:.involvedObject.name,MESSAGE:.message' \ + 2>/dev/null | tail -25 || true + kind delete cluster --name eck-ror From d1fc0ad2b791a2581d1abae94144d6db807355c5 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 9 Sep 2026 18:15:26 +0200 Subject: [PATCH 06/40] Remove retry mechanism from E2E test workflow and Cypress config --- .github/workflows/all-e2e-tests.yml | 44 +++++++++-------------------- e2e-tests/cypress.config.ts | 4 +-- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index d1ef4fbc..567c9b81 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -52,22 +52,14 @@ jobs: uses: ./.github/docker-memory-monitor with: action: start - # Retry mechanism to handle transient infrastructure issues: - # - npm error 429 Too Many Requests from registry.npmjs.org - # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) - # These errors are typically temporary and resolve with a simple retry. + # Retries disabled (was nick-fields/retry, max_attempts: 2, retry_on: any): a real spec + # failure and a transient infra hiccup both exited non-zero, so this re-ran the entire + # 27-spec suite from scratch on any failure, masking flakiness instead of surfacing it - + # see run-34312427155's logs, where the whole suite ran twice back to back. - name: Run E2E tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 - with: - max_attempts: 2 - # eck envs are slower per-request than docker (K8s networking overhead vs docker-compose - # bridge), and the full 27-spec suite plus the ~9-minute Tenancy.cy.ts spec doesn't - # reliably fit in 35 minutes there. - timeout_minutes: 45 - retry_wait_seconds: 120 - retry_on: any - command: | - ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --mode prod + timeout-minutes: 45 + run: | + ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --mode prod env: ROR_ACTIVATION_KEY: ${{ secrets.ROR_ENT_ACTIVATION_TOKEN }} ELECTRON_EXTRA_LAUNCH_ARGS: '--disable-gpu' @@ -221,22 +213,14 @@ jobs: uses: ./.github/docker-memory-monitor with: action: start - # Retry mechanism to handle transient infrastructure issues: - # - npm error 429 Too Many Requests from registry.npmjs.org - # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) - # These errors are typically temporary and resolve with a simple retry. + # Retries disabled (was nick-fields/retry, max_attempts: 2, retry_on: any): a real spec + # failure and a transient infra hiccup both exited non-zero, so this re-ran the entire + # 27-spec suite from scratch on any failure, masking flakiness instead of surfacing it - + # see run-34312427155's logs, where the whole suite ran twice back to back. - name: Run E2E tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 - with: - max_attempts: 2 - # eck envs are slower per-request than docker (K8s networking overhead vs docker-compose - # bridge), and the full 27-spec suite plus the ~9-minute Tenancy.cy.ts spec doesn't - # reliably fit in 35 minutes there. - timeout_minutes: 45 - retry_wait_seconds: 120 - retry_on: any - command: | - ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-es ${{ env.ROR_IMAGE_TAG }} --ror-kbn ${{ env.ROR_IMAGE_TAG }} --mode dev + timeout-minutes: 45 + run: | + ./runner.sh --run e2e --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-es ${{ env.ROR_IMAGE_TAG }} --ror-kbn ${{ env.ROR_IMAGE_TAG }} --mode dev env: ROR_ACTIVATION_KEY: ${{ secrets.ROR_ENT_ACTIVATION_TOKEN }} diff --git a/e2e-tests/cypress.config.ts b/e2e-tests/cypress.config.ts index 84512337..cc30761f 100644 --- a/e2e-tests/cypress.config.ts +++ b/e2e-tests/cypress.config.ts @@ -25,8 +25,8 @@ export default defineConfig({ pageLoadTimeout: 20000, taskTimeout: 20000, retries: { - openMode: 2, - runMode: 2 + openMode: 0, + runMode: 0 }, e2e: { // We've imported your old cypress plugins here. From 7849407caec338b722f332269bc51cb03d2826bc Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Fri, 11 Sep 2026 06:06:07 +0200 Subject: [PATCH 07/40] Fix flaky DevTools tour and editor interactions in E2E tests --- e2e-tests/cypress/support/page-objects/DevTools.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/DevTools.ts b/e2e-tests/cypress/support/page-objects/DevTools.ts index 0d81786d..91712321 100644 --- a/e2e-tests/cypress/support/page-objects/DevTools.ts +++ b/e2e-tests/cypress/support/page-objects/DevTools.ts @@ -10,7 +10,10 @@ export class DevTools { if (semver.lt(getKibanaVersion(), '9.2.0')) { if (semver.gte(getKibanaVersion(), '8.16.0')) { - cy.get("[data-test-subj='consoleSkipTourButton']").click(); + // Cypress actionability checks flag this button as "covered" by its own text label + // (``) - a false positive, not a real overlay - + // so force the click instead of waiting out a cover that will never clear. + cy.get("[data-test-subj='consoleSkipTourButton']").click({ force: true }); } else { cy.get('[data-test-subj="help-close-button"]').click(); } @@ -22,7 +25,13 @@ export class DevTools { cy.intercept({ method: 'POST', pathname: '/s/default/api/console/proxy' }).as('sendRequest'); if (semver.gte(getKibanaVersion(), '8.16.0')) { cy.get('[data-test-subj="clearConsoleInput"]').click(); - cy.get('[data-test-subj="consoleMonacoEditor"]').click().type(text); + // Monaco mounts its hidden input textarea asynchronously after the container appears; + // typing before it exists lands on the static "view-lines" rendering div instead, which + // isn't a typeable element and throws. Wait for the real input to exist first. + cy.get('[data-test-subj="consoleMonacoEditor"] textarea.inputarea').should('exist'); + // The console's action-icon toolbar (euiFlexGroup) can overlap the editor while it settles, + // tripping Cypress's actionability check even though the editor is interactable - force the click. + cy.get('[data-test-subj="consoleMonacoEditor"]').click({ force: true }).type(text); cy.get('[data-test-subj="sendRequestButton"]').click(); } else if (semver.lte(getKibanaVersion(), '7.9.0')) { // Select editor, delete, write From c845eafda4c559bdab6988aef66794f4bdd899ff Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 12 Sep 2026 07:23:59 +0200 Subject: [PATCH 08/40] Support custom timeouts and retries in Cypress e2e API and settings --- e2e-tests/cypress/e2e/Tenancy.cy.ts | 13 +++- e2e-tests/cypress/plugins/index.ts | 19 ++++-- e2e-tests/cypress/support/commands.ts | 63 ++++++++++++++----- e2e-tests/cypress/support/e2e.ts | 10 ++- .../cypress/support/helpers/KbnApiClient.ts | 39 ++++++++++-- .../cypress/support/page-objects/Settings.ts | 29 +++++++++ 6 files changed, 141 insertions(+), 32 deletions(-) diff --git a/e2e-tests/cypress/e2e/Tenancy.cy.ts b/e2e-tests/cypress/e2e/Tenancy.cy.ts index 5d6c2afb..2e5da140 100644 --- a/e2e-tests/cypress/e2e/Tenancy.cy.ts +++ b/e2e-tests/cypress/e2e/Tenancy.cy.ts @@ -41,9 +41,15 @@ describe('Tenancy', () => { describe('should run tests when tenancy switched in a different tab', () => { const urlWithInfosecTenancyId = `/s/default/app/discover?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedInfosecGroup}`; + // Retries re-run the whole test body, including this callback, so a failed attempt opens + // another popup on top of whatever the previous attempt left behind. Left uncollected, they + // pile up (each one polling Kibana unauthenticated) until a later attempt's login hangs + // waiting for a Kibana request queue the orphaned tabs are still contending for. + let openedWindow: Window | null = null; + const openAnotherTabs = () => { cy.window().then(win => { - win.open(urlWithInfosecTenancyId, '_blank'); + openedWindow = win.open(urlWithInfosecTenancyId, '_blank'); }); }; @@ -52,6 +58,11 @@ describe('Tenancy', () => { cy.clearLocalStorage(); }); + afterEach(() => { + openedWindow?.close(); + openedWindow = null; + }); + // eslint-disable-next-line no-use-before-define runTests({ callbackBeforeLogin: openAnotherTabs }); }); diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index 3fe47149..dd50eff6 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -97,12 +97,13 @@ const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(r const fetchWithJsonRetry = async ( url: string, createInit: () => Parameters[1], - retryOnTransportError = true + retryOnTransportError = true, + timeoutMs = FETCH_TIMEOUT_MS ): Promise => { let response: Response; for (let attempt = 1; attempt <= NON_JSON_RETRY_ATTEMPTS; attempt++) { // eslint-disable-next-line no-await-in-loop - response = await fetchWithTransportRetry(url, createInit, retryOnTransportError); + response = await fetchWithTransportRetry(url, createInit, retryOnTransportError, timeoutMs); const contentType = response.headers.get('content-type') || ''; // The startup race serves Kibana's login page (text/html) in place of the expected @@ -131,12 +132,13 @@ const fetchWithJsonRetry = async ( const fetchWithTransportRetry = async ( url: string, createInit: () => Parameters[1], - retryOnTransportError: boolean + retryOnTransportError: boolean, + timeoutMs: number ): Promise => { for (let attempt = 1; attempt <= TRANSPORT_ERROR_RETRY_ATTEMPTS; attempt++) { try { // eslint-disable-next-line no-await-in-loop - return await fetch(url, { timeout: FETCH_TIMEOUT_MS, ...createInit() }); + return await fetch(url, { timeout: timeoutMs, ...createInit() }); } catch (error) { const isLastAttempt = attempt === TRANSPORT_ERROR_RETRY_ATTEMPTS; if (!retryOnTransportError || !isTransientNetworkError(error) || isLastAttempt) { @@ -163,7 +165,7 @@ const fetchWithTransportRetry = async ( module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { on('task', { async httpCall(options: HttpCallOptions): Promise { - const { method, url, headers, body, failOnStatusCode, allowTransportError } = options; + const { method, url, headers, body, failOnStatusCode, allowTransportError, timeoutMs } = options; try { const response: Response = await fetchWithJsonRetry( @@ -174,7 +176,8 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) body: body ?? undefined, agent: sharedHttpsAgent }), - !allowTransportError + !allowTransportError, + timeoutMs ); if (!response.ok && failOnStatusCode) { @@ -394,6 +397,10 @@ interface HttpCallOptions { failOnStatusCode?: boolean; // For endpoints that restart the server they answer from, so the reply is lost by design. allowTransportError?: boolean; + // Overrides FETCH_TIMEOUT_MS for endpoints that are legitimately slower than a transport-error + // check needs to be, e.g. Kibana's sample-data install (creates an index, then bulk-inserts + // thousands of documents into it). + timeoutMs?: number; } interface FileToUpload { diff --git a/e2e-tests/cypress/support/commands.ts b/e2e-tests/cypress/support/commands.ts index 1ddecf5a..e02613d2 100644 --- a/e2e-tests/cypress/support/commands.ts +++ b/e2e-tests/cypress/support/commands.ts @@ -6,15 +6,20 @@ import { capture as clipboardCapture } from './clipboardCapture'; Cypress.Commands.add( 'kbnPost', - ({ endpoint, credentials, payload, currentGroupHeader, impersonating, headers }, ...args) => + ( + { endpoint, credentials, payload, currentGroupHeader, impersonating, failOnStatusCode, headers, timeoutMs }, + ...args + ) => cy.kbnRequest({ method: 'POST', endpoint, credentials, payload, currentGroupHeader, + failOnStatusCode, headers, - impersonating + impersonating, + timeoutMs }) as Cypress.Chainable ); @@ -67,12 +72,14 @@ Cypress.Commands.add( }) as Cypress.Chainable ); -Cypress.Commands.add('esGet', ({ endpoint, credentials }, ...args) => - cy.esRequest({ - method: 'GET', - endpoint, - credentials - }) as Cypress.Chainable +Cypress.Commands.add( + 'esGet', + ({ endpoint, credentials }, ...args) => + cy.esRequest({ + method: 'GET', + endpoint, + credentials + }) as Cypress.Chainable ); Cypress.Commands.add( @@ -101,7 +108,17 @@ Cypress.Commands.add( Cypress.Commands.add( 'kbnRequest', - ({ method, endpoint, credentials, payload, currentGroupHeader, impersonating, failOnStatusCode, headers }) => { + ({ + method, + endpoint, + credentials, + payload, + currentGroupHeader, + impersonating, + failOnStatusCode, + headers, + timeoutMs + }) => { const customHeaders: { [key: string]: string } = { 'kbn-xsrf': 'true', ...headers }; if (currentGroupHeader) { customHeaders['x-ror-tenancy-id'] = currentGroupHeader; @@ -111,7 +128,15 @@ Cypress.Commands.add( customHeaders['x-ror-impersonating'] = impersonating; } - httpCall(method, `${Cypress.config().baseUrl}/${endpoint}`, credentials, payload, customHeaders, failOnStatusCode); + httpCall( + method, + `${Cypress.config().baseUrl}/${endpoint}`, + credentials, + payload, + customHeaders, + failOnStatusCode, + timeoutMs + ); } ); @@ -125,7 +150,8 @@ function httpCall( credentials: string, payload?: string | object, headers?: { [key: string]: string }, - failOnStatusCode = true + failOnStatusCode = true, + timeoutMs?: number ): Cypress.Chainable { const options = { method, @@ -136,7 +162,8 @@ function httpCall( ...headers }, body: payload ? (typeof payload === 'string' ? payload : JSON.stringify(payload)) : null, - failOnStatusCode + failOnStatusCode, + timeoutMs }; return cy.task('httpCall', options); @@ -208,11 +235,13 @@ Cypress.Commands.add('getValueFromClipboard', () => cy.wrap(clipboardCapture, { // Cypress 15 types cy.wait's alias parameter as `@${string}`; mirroring it here means a // forgotten '@' prefix is a compile error instead of a silent numeric-wait. -Cypress.Commands.add('waitForResponse', (alias: `@${string}`) => - cy.wait(alias).then(({ response }) => { - if (!response) throw new Error(`Expected a response for ${alias}`); - return response; - }) as unknown as Cypress.Chainable<{ statusCode: number }> +Cypress.Commands.add( + 'waitForResponse', + (alias: `@${string}`) => + cy.wait(alias).then(({ response }) => { + if (!response) throw new Error(`Expected a response for ${alias}`); + return response; + }) as unknown as Cypress.Chainable<{ statusCode: number }> ); Cypress.on('uncaught:exception', (err, runnable, promise) => { diff --git a/e2e-tests/cypress/support/e2e.ts b/e2e-tests/cypress/support/e2e.ts index e8a6ac82..25af3949 100644 --- a/e2e-tests/cypress/support/e2e.ts +++ b/e2e-tests/cypress/support/e2e.ts @@ -38,7 +38,8 @@ declare global { payload, currentGroupHeader, failOnStatusCode, - headers + headers, + timeoutMs }: { method: string; endpoint: string; @@ -48,6 +49,7 @@ declare global { impersonating?: string; failOnStatusCode?: boolean; headers?: { [key: string]: string }; + timeoutMs?: number; }): Chainable; kbnGet({ endpoint, @@ -68,14 +70,18 @@ declare global { credentials, payload, currentGroupHeader, - headers + failOnStatusCode, + headers, + timeoutMs }: { endpoint: string; credentials: string; payload?: Payload; currentGroupHeader?: string; impersonating?: string; + failOnStatusCode?: boolean; headers?: { [key: string]: string }; + timeoutMs?: number; }): Chainable; kbnPut({ endpoint, diff --git a/e2e-tests/cypress/support/helpers/KbnApiClient.ts b/e2e-tests/cypress/support/helpers/KbnApiClient.ts index caffbb3b..70d30f6b 100644 --- a/e2e-tests/cypress/support/helpers/KbnApiClient.ts +++ b/e2e-tests/cypress/support/helpers/KbnApiClient.ts @@ -1,3 +1,5 @@ +import { recurse } from 'cypress-recurse'; + export class KbnApiClient { public getDataViews(credentials: string, group?: string): Cypress.Chainable { return cy.kbnGet({ @@ -51,12 +53,37 @@ export class KbnApiClient { }); } - public loadSampleData(sampleDatasetName: string, credentials: string, group?: string): void { - cy.kbnPost({ - endpoint: `api/sample_data/${sampleDatasetName}`, - credentials, - currentGroupHeader: group - }); + /** + * Kibana's sample-data installer deletes the previous index and recreates it in one request; + * those two steps occasionally race each other (resource_already_exists_exception -> 500), and + * the bulk-insert that follows a successful create is too slow for the shared httpCall timeout. + * Give this call more room per attempt and retry with backoff so the race gets to resolve + * itself instead of failing the test. + */ + public loadSampleData( + sampleDatasetName: string, + credentials: string, + group?: string, + timeout = 90000, + interval = 5000 + ): Cypress.Chainable<{ statusCode?: number }> { + return recurse( + () => + cy.kbnPost<{ statusCode?: number }>({ + endpoint: `api/sample_data/${sampleDatasetName}`, + credentials, + currentGroupHeader: group, + failOnStatusCode: false, + timeoutMs: 30000 + }), + response => !response?.statusCode, + { + timeout, + delay: interval, + log: response => cy.log(`Load sample data "${sampleDatasetName}" response: ${JSON.stringify(response)}`), + error: `Timed out loading sample data "${sampleDatasetName}"` + } + ); } public deleteSampleData(sampleDatasetName: string, credentials: string, group?: string): void { diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index 733d3f88..867d2385 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -1,4 +1,5 @@ import * as yaml from 'js-yaml'; +import { recurse } from 'cypress-recurse'; import { rorApiClient } from '../helpers/RorApiClient'; import { RorMenu } from './RorMenu'; @@ -6,6 +7,9 @@ import { SecuritySettings } from './SecuritySettings'; import { parseKbnSettings } from '../helpers/parseKibanaSettings'; export class Settings { + private static readonly SAVE_MODAL_SETTLE_MS = 300; + private static readonly SAVE_MODAL_RETRY_ATTEMPTS = 3; + static open() { cy.log('Open settings'); RorMenu.openRorMenu(); @@ -44,12 +48,37 @@ export class Settings { static confirmSaveModal() { cy.log('Confirm settings save modal'); cy.intercept('POST', '/pkp/api/settings*').as('confirmSaveSettings'); + // The save click can occasionally land while the settings iframe is still catching up with a + // just-established session (a transient "Forbidden" flashes and the confirmation modal never + // mounts), so retry the Save click until "Save anyway" actually shows up instead of failing + // after a single 20s wait. + Settings.clickSaveButtonUntilModalAppears(); SecuritySettings.getIframeBody().contains('Save anyway').click(); cy.waitForResponse('@confirmSaveSettings').then(response => { expect(response.statusCode).to.eq(200); }); } + private static clickSaveButtonUntilModalAppears() { + recurse( + () => + cy + .then(() => Settings.clickSaveButton()) + .then(() => cy.wait(Settings.SAVE_MODAL_SETTLE_MS, { log: false })) + .then(() => SecuritySettings.getIframeBody()), + $body => ($body as JQuery).find(':contains("Save anyway")').length > 0, + { + limit: Settings.SAVE_MODAL_RETRY_ATTEMPTS, + delay: 0, + timeout: 20000, + // confirmSaveModal() asserts on the modal text right after this, so failing here would + // only replace that message with a less specific one. + doNotFail: true, + log: false + } + ); + } + static closeToastMessages() { cy.log('Close toast message'); return SecuritySettings.getIframeBody() From 03174e99ec909e60833457dce3858ce7a9e2cad4 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 14 Sep 2026 18:01:23 +0200 Subject: [PATCH 09/40] Improve e2e test reliability and flake handling --- e2e-tests/cypress.config.ts | 4 ++-- e2e-tests/cypress/e2e/Tenancy.cy.ts | 2 ++ e2e-tests/cypress/support/commands.ts | 6 +++++- e2e-tests/cypress/support/page-objects/Settings.ts | 11 ++++++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress.config.ts b/e2e-tests/cypress.config.ts index 0ac6c244..8a043a85 100644 --- a/e2e-tests/cypress.config.ts +++ b/e2e-tests/cypress.config.ts @@ -29,8 +29,8 @@ export default defineConfig({ pageLoadTimeout: 20000, taskTimeout: 20000, retries: { - openMode: 0, - runMode: 0 + openMode: 2, + runMode: 2 }, e2e: { // We've imported your old cypress plugins here. diff --git a/e2e-tests/cypress/e2e/Tenancy.cy.ts b/e2e-tests/cypress/e2e/Tenancy.cy.ts index 2e5da140..c375c1ae 100644 --- a/e2e-tests/cypress/e2e/Tenancy.cy.ts +++ b/e2e-tests/cypress/e2e/Tenancy.cy.ts @@ -51,6 +51,8 @@ describe('Tenancy', () => { cy.window().then(win => { openedWindow = win.open(urlWithInfosecTenancyId, '_blank'); }); + cy.log('let the second tab reach its login page before the main tab starts logging in'); + cy.wait(3000); }; beforeEach(() => { diff --git a/e2e-tests/cypress/support/commands.ts b/e2e-tests/cypress/support/commands.ts index e02613d2..5fa10c49 100644 --- a/e2e-tests/cypress/support/commands.ts +++ b/e2e-tests/cypress/support/commands.ts @@ -166,7 +166,11 @@ function httpCall( timeoutMs }; - return cy.task('httpCall', options); + // cy.task()'s own default timeout (taskTimeout in cypress.config.ts) is sized for the default + // FETCH_TIMEOUT_MS budget (see plugins/index.ts) - a caller overriding timeoutMs upward (e.g. + // loadSampleData's slower bulk-insert) needs a matching override here, or Cypress kills the + // whole task at the unmodified default before the fetch's own longer timeout is ever reached. + return timeoutMs ? cy.task('httpCall', options, { timeout: timeoutMs + 5000 }) : cy.task('httpCall', options); } function uploadFile( diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index 867d2385..debcab2d 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -63,7 +63,16 @@ export class Settings { recurse( () => cy - .then(() => Settings.clickSaveButton()) + .then(() => SecuritySettings.getIframeBody()) + .then($body => { + // Once the modal is opening, an overlay mask covers the Save button; clicking again + // would report the button as hidden instead of giving the modal time to finish + // mounting. Only re-click while nothing has opened yet. + const hasOverlay = ($body as JQuery).hasClass('euiBody-hasOverlayMask'); + if (!hasOverlay) { + Settings.clickSaveButton(); + } + }) .then(() => cy.wait(Settings.SAVE_MODAL_SETTLE_MS, { log: false })) .then(() => SecuritySettings.getIframeBody()), $body => ($body as JQuery).find(':contains("Save anyway")').length > 0, From 38e839ab0ec5eb298cb5a3da7c000a02ed8f0677 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 15 Sep 2026 06:33:55 +0200 Subject: [PATCH 10/40] Increase pageLoadTimeout in hidden apps test --- e2e-tests/cypress/e2e/Hide_apps.cy.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e-tests/cypress/e2e/Hide_apps.cy.ts b/e2e-tests/cypress/e2e/Hide_apps.cy.ts index 7b7c4f55..ed7afc6d 100644 --- a/e2e-tests/cypress/e2e/Hide_apps.cy.ts +++ b/e2e-tests/cypress/e2e/Hide_apps.cy.ts @@ -66,7 +66,10 @@ describe('hidden apps', () => { }); }); - context('Kibana global search', () => { + // Restoring the default config from "all apps hidden" makes Kibana rebuild its app registry + // (150+ entries) before the SPA-initiated reload fires its `load` event; that routinely exceeds + // the global 20s pageLoadTimeout in the afterEach cleanup. + context('Kibana global search', { pageLoadTimeout: 60000 }, () => { beforeEach(() => { Settings.setSettingsData('hiddenAllAppsSettings.yaml'); Login.initialization(); From 0340371e399f1aea220bd6b02029a82b055766b7 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 15 Sep 2026 06:41:47 +0200 Subject: [PATCH 11/40] Increase pageLoadTimeout for hidden apps test suite --- e2e-tests/cypress/e2e/Hide_apps.cy.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/e2e-tests/cypress/e2e/Hide_apps.cy.ts b/e2e-tests/cypress/e2e/Hide_apps.cy.ts index ed7afc6d..ee9e3563 100644 --- a/e2e-tests/cypress/e2e/Hide_apps.cy.ts +++ b/e2e-tests/cypress/e2e/Hide_apps.cy.ts @@ -10,7 +10,7 @@ import { SearchApps } from '../support/page-objects/SearchApps'; import { Loader } from '../support/page-objects/Loader'; import { Home } from '../support/page-objects/Home'; -describe('hidden apps', () => { +describe('hidden apps', { pageLoadTimeout: 60000 }, () => { afterEach(() => { Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); }); @@ -66,10 +66,7 @@ describe('hidden apps', () => { }); }); - // Restoring the default config from "all apps hidden" makes Kibana rebuild its app registry - // (150+ entries) before the SPA-initiated reload fires its `load` event; that routinely exceeds - // the global 20s pageLoadTimeout in the afterEach cleanup. - context('Kibana global search', { pageLoadTimeout: 60000 }, () => { + context('Kibana global search', () => { beforeEach(() => { Settings.setSettingsData('hiddenAllAppsSettings.yaml'); Login.initialization(); From 71fea4a8518350f4bb08bdef7117cea57696c451 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 15 Sep 2026 07:05:24 +0200 Subject: [PATCH 12/40] Add APM server system account to observability visible settings --- e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml index 48dc3517..b7d6b44a 100644 --- a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml +++ b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml @@ -26,6 +26,10 @@ readonlyrest: verbosity: error auth_key: kibana:kibana + - name: 'APM server - system account' + verbosity: error + auth_key: apm:test + - name: JWT_AUTH jwt_auth: name: 'jwt1' From 50552577adc836317458ebc595db406f3bf3a17d Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 16 Sep 2026 05:37:06 +0200 Subject: [PATCH 13/40] Add ECK internal probe and service account rules to fixtures --- .../fixtures/allowedApiPathsSettings.yaml | 16 +++++++++++++++ .../defaultReadonlyRestEsAndKbnSettings.yaml | 16 +++++++++++++++ .../defaultReadonlyRestEsSettings.yaml | 16 +++++++++++++++ .../fixtures/hiddenAllAppsSettings.yaml | 16 +++++++++++++++ .../fixtures/hiddenHomePageSettings.yaml | 16 +++++++++++++++ .../hiddenSpaceManagementSettings.yaml | 16 +++++++++++++++ .../observabilityVisibleSettings.yaml | 16 +++++++++++++++ .../cypress/fixtures/reportingSettings.yaml | 20 +++++++++++++++++-- e2e-tests/cypress/fixtures/roSettings.yaml | 20 +++++++++++++++++-- .../cypress/fixtures/roStrictSettings.yaml | 18 ++++++++++++++++- e2e-tests/cypress/fixtures/testSettings.yaml | 20 +++++++++++++++++++ 11 files changed, 185 insertions(+), 5 deletions(-) diff --git a/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml b/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml index b248a99e..15b6ebf2 100644 --- a/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml +++ b/e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml @@ -5,10 +5,26 @@ readonlyrest: keystore_pass: readonlyrest key_pass: readonlyrest access_control_rules: + # <-- related to ECK environment --> + - name: 'Kibana service account - token' + verbosity: error + token_authentication: + token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' + username: service_account + - name: 'Kibana service account - user/pass' verbosity: error auth_key: kibana:kibana + - name: 'PROBE' + verbosity: error + auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' + + - name: 'ELASTIC-INTERNAL' + verbosity: error + auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' + # + # api_only user restricted to a single exact Kibana API path. - name: API_ONLY_RESTRICTED auth_key: api_only_restricted_user:dev diff --git a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml index 37a5e906..d28c5775 100644 --- a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml +++ b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml @@ -22,10 +22,26 @@ readonlyrest: index_template: "'readonlyrest_audit_'yyyy-MM-dd" access_control_rules: + # <-- related to ECK environment --> + - name: 'Kibana service account - token' + verbosity: error + token_authentication: + token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' + username: service_account + - name: 'Kibana service account - user/pass' verbosity: error auth_key: kibana:kibana + - name: 'PROBE' + verbosity: error + auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' + + - name: 'ELASTIC-INTERNAL' + verbosity: error + auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' + # + - name: JWT_AUTH jwt_auth: name: 'jwt1' diff --git a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsSettings.yaml b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsSettings.yaml index 40d30e2e..35222ef4 100644 --- a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsSettings.yaml +++ b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsSettings.yaml @@ -22,10 +22,26 @@ readonlyrest: index_template: "'readonlyrest_audit_'yyyy-MM-dd" access_control_rules: + # <-- related to ECK environment --> + - name: 'Kibana service account - token' + verbosity: error + token_authentication: + token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' + username: service_account + - name: 'Kibana service account - user/pass' verbosity: error auth_key: kibana:kibana + - name: 'PROBE' + verbosity: error + auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' + + - name: 'ELASTIC-INTERNAL' + verbosity: error + auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' + # + - name: JWT_AUTH jwt_auth: name: 'jwt1' diff --git a/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml b/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml index ab593443..0352c11f 100644 --- a/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml @@ -1,10 +1,26 @@ readonlyrest: access_control_rules: + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + - name: "Kibana service account - user/pass" verbosity: error auth_key: kibana:kibana + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: ADMIN_GRP groups: [admins_group] kibana_hide_apps: [ "Enterprise Search", "Observability", "Elasticsearch", "Analytics", "Management"] diff --git a/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml b/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml index a9e3d23e..c677c906 100644 --- a/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml @@ -1,10 +1,26 @@ readonlyrest: access_control_rules: + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + - name: "Kibana service account - user/pass" verbosity: error auth_key: kibana:kibana + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: ADMIN_GRP groups: [admins_group] kibana_hide_apps: [ "Home"] diff --git a/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml b/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml index a320a23c..6786ff5a 100644 --- a/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml +++ b/e2e-tests/cypress/fixtures/hiddenSpaceManagementSettings.yaml @@ -1,9 +1,25 @@ readonlyrest: access_control_rules: + # <-- related to ECK environment --> + - name: 'Kibana service account - token' + verbosity: error + token_authentication: + token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' + username: service_account + - name: 'Kibana service account - user/pass' verbosity: error auth_key: kibana:kibana + - name: 'PROBE' + verbosity: error + auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' + + - name: 'ELASTIC-INTERNAL' + verbosity: error + auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' + # + - name: ADMIN_GRP groups: [admins_group] kibana_hide_apps: ['Management|Stack Management'] diff --git a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml index b7d6b44a..e31b8ca2 100644 --- a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml +++ b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml @@ -22,10 +22,26 @@ readonlyrest: index_template: "'readonlyrest_audit_'yyyy-MM-dd" access_control_rules: + # <-- related to ECK environment --> + - name: 'Kibana service account - token' + verbosity: error + token_authentication: + token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' + username: service_account + - name: 'Kibana service account - user/pass' verbosity: error auth_key: kibana:kibana + - name: 'PROBE' + verbosity: error + auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' + + - name: 'ELASTIC-INTERNAL' + verbosity: error + auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' + # + - name: 'APM server - system account' verbosity: error auth_key: apm:test diff --git a/e2e-tests/cypress/fixtures/reportingSettings.yaml b/e2e-tests/cypress/fixtures/reportingSettings.yaml index 5bfb0d93..ecaf38bf 100644 --- a/e2e-tests/cypress/fixtures/reportingSettings.yaml +++ b/e2e-tests/cypress/fixtures/reportingSettings.yaml @@ -22,11 +22,27 @@ readonlyrest: index_template: "'xxx.reporting-'YYYY-MM" access_control_rules: - + + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + - name: "Kibana service account - user/pass" verbosity: error auth_key: kibana:kibana - + + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: PERSONAL_GRP groups: [personal_group] <<: *common-rules diff --git a/e2e-tests/cypress/fixtures/roSettings.yaml b/e2e-tests/cypress/fixtures/roSettings.yaml index c5473a2e..79fb672e 100644 --- a/e2e-tests/cypress/fixtures/roSettings.yaml +++ b/e2e-tests/cypress/fixtures/roSettings.yaml @@ -22,11 +22,27 @@ readonlyrest: index_template: "'readonlyrest_audit_'yyyy-MM-dd" access_control_rules: - + + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + - name: "Kibana service account - user/pass" verbosity: error auth_key: kibana:kibana - + + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: PERSONAL_GRP groups: [personal_group] <<: *common-rules diff --git a/e2e-tests/cypress/fixtures/roStrictSettings.yaml b/e2e-tests/cypress/fixtures/roStrictSettings.yaml index a6b364bc..4452c895 100644 --- a/e2e-tests/cypress/fixtures/roStrictSettings.yaml +++ b/e2e-tests/cypress/fixtures/roStrictSettings.yaml @@ -22,11 +22,27 @@ readonlyrest: index_template: "'readonlyrest_audit_'yyyy-MM-dd" access_control_rules: - + + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + - name: "Kibana service account - user/pass" verbosity: error auth_key: kibana:kibana + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: PERSONAL_GRP groups: [personal_group] <<: *common-rules diff --git a/e2e-tests/cypress/fixtures/testSettings.yaml b/e2e-tests/cypress/fixtures/testSettings.yaml index 4e48c6e2..caf5b298 100644 --- a/e2e-tests/cypress/fixtures/testSettings.yaml +++ b/e2e-tests/cypress/fixtures/testSettings.yaml @@ -7,6 +7,26 @@ readonlyrest: access_control_rules: + # <-- related to ECK environment --> + - name: "Kibana service account - token" + verbosity: error + token_authentication: + token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}" + username: service_account + + - name: "Kibana service account - user/pass" + verbosity: error + auth_key: kibana:kibana + + - name: "PROBE" + verbosity: error + auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}" + + - name: "ELASTIC-INTERNAL" + verbosity: error + auth_key: "elastic-internal:${INTERNAL_USR_PASS}" + # + - name: "::Tweets1::" methods: [GET, POST] indices: ["twitter", ".kibana"] From af0c442c292106d7685df3dbabbdb54438bfe562 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Thu, 17 Sep 2026 05:54:56 +0200 Subject: [PATCH 14/40] Fix user indices in impersonation e2e test --- e2e-tests/cypress/e2e/Impersonate.cy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/e2e/Impersonate.cy.ts b/e2e-tests/cypress/e2e/Impersonate.cy.ts index 8b4ee11d..d7256588 100644 --- a/e2e-tests/cypress/e2e/Impersonate.cy.ts +++ b/e2e-tests/cypress/e2e/Impersonate.cy.ts @@ -80,7 +80,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(3, 'Local users'); Impersonate.assertServiceType(3, 'local'); Impersonate.assertServiceColumns(3, ['Username']); - Impersonate.assertUser(3, 0, 'kibana'); + Impersonate.assertUser(3, 2, 'kibana'); }; createLdapUsers(); @@ -102,7 +102,7 @@ describe('impersonate', () => { cy.log('should impersonate localUser'); Impersonate.open(); - Impersonate.impersonateUserFromTheList(3, 2, 'new_user'); + Impersonate.impersonateUserFromTheList(3, 3, 'new_user'); Impersonate.finishImpersonation(); Impersonate.verifyFinishedImpersonation(); From 91e0faa6dc371625f34f0174d9d68193cb59d80f Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Fri, 18 Sep 2026 05:52:18 +0200 Subject: [PATCH 15/40] Fix flaky kibanaIndexTemplate test in docker environments --- .../cypress/e2e/Readonlyrest-settings.cy.ts | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts index 2e007086..efa1d498 100644 --- a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts +++ b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts @@ -29,13 +29,10 @@ describe('Readonlyrest-settings', () => { }); // The docker env (elk-ror) runs 2 kbn-ror replicas behind kbn-proxy's round robin (see - // base.docker-compose.yml). resetKibanaIndexToTemplate is only applied once, at tenant-index - // creation time (TenantIndexBasedOnTemplateApplier, called from abstractIndexCreator.ts) - unlike - // the CSS/JS/middleware injections elsewhere in this spec, which re-evaluate on every request and - // so self-correct if an early request lands on a stale replica. If the one request that creates - // .kibana_admins_group lands on a replica that has not yet picked up the settings POSTed above, - // the reset never happens and nothing later can retrigger it. The eck-* environments run a single - // Kibana node (kind-cluster/ror/base/kbn.yml: count: 1) and are unaffected. + // base.docker-compose.yml), so a request of this test can land on either of them, and the two + // do not share the per-session state that decides whether the tenant index gets reset from the + // template. That made the test flaky there. The eck-* environments run a single Kibana node + // (kind-cluster/ror/base/kbn.yml: count: 1), where every request hits the same state. (Cypress.env().envName === 'elk-ror' ? it.skip : it)('should verify kibanaIndexTemplate functionality', () => { Settings.setReadonlyRestKbnSettings(` kibanaIndexTemplate: ".kibana_template_group" @@ -63,7 +60,15 @@ describe('Readonlyrest-settings', () => { currentGroupHeader: 'admins_group' }); - cy.reload(); + // A reload is not enough to retrigger the reset: kbn-ror only re-runs the tenant index + // creation (and with it the reindex from the template) once per session id per index, and + // remembers that for 2 minutes in the memory of the node that served the request. A login + // is not subject to that - it always runs the creation for the session it opens - so clear + // the cookies and sign in again to get a session id the node has not seen yet. + cy.clearCookies(); + cy.clearLocalStorage(); + Login.initialization(); + Dashboard.openDashboard(); Dashboard.verifyDashboardNotExist('Look at my dashboard'); }); From 084de352c7c827c57305b4d40e395c232fb3bba2 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 15:36:30 +0200 Subject: [PATCH 16/40] Use Settings.restoreDefaultSettingsData in tests --- e2e-tests/cypress/e2e/Hide_apps.cy.ts | 2 +- e2e-tests/cypress/e2e/Reporting-index.cy.ts | 2 +- e2e-tests/cypress/support/page-objects/Settings.ts | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/e2e/Hide_apps.cy.ts b/e2e-tests/cypress/e2e/Hide_apps.cy.ts index ee9e3563..a6de03e9 100644 --- a/e2e-tests/cypress/e2e/Hide_apps.cy.ts +++ b/e2e-tests/cypress/e2e/Hide_apps.cy.ts @@ -12,7 +12,7 @@ import { Home } from '../support/page-objects/Home'; describe('hidden apps', { pageLoadTimeout: 60000 }, () => { afterEach(() => { - Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); + Settings.restoreDefaultSettingsData(); }); context('Stack Management navigation', () => { diff --git a/e2e-tests/cypress/e2e/Reporting-index.cy.ts b/e2e-tests/cypress/e2e/Reporting-index.cy.ts index 56c431e6..5a7fdb2b 100644 --- a/e2e-tests/cypress/e2e/Reporting-index.cy.ts +++ b/e2e-tests/cypress/e2e/Reporting-index.cy.ts @@ -21,7 +21,7 @@ describe('Reporting index', () => { if (semver.gte(getKibanaVersion(), '8.0.0')) { kbnApiAdvancedClient.deleteDataViews(admin, 'infosec_group'); } - Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); + Settings.restoreDefaultSettingsData(); }); it('should correctly match index pattern when audit index_template contains .reporting', () => { diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index debcab2d..a4a58629 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -137,6 +137,16 @@ export class Settings { rorApiClient.configureRorIndexMainSettingsFromFixture(fixtureYamlSettingsFileName); } + // An open Kibana page keeps sending requests with the tenancy of the old settings. When the new + // settings do not match that tenancy, ES forbids the requests and Kibana stops with a fatal error. + // Cypress then fails the hook that runs. Unloading the page first means no request uses the old tenancy. + static restoreDefaultSettingsData() { + cy.window({ log: false }).then(win => { + win.location.href = 'about:blank'; + }); + Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); + } + static setReadonlyRestKbnSettings(readonlyRestKbnSettings = '') { cy.fixture('defaultReadonlyRestEsSettings.yaml').then(esYamlSettings => { const merged = { From 116a7ac2b5542fe927fc4c38f3de19e641d94796 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:50:51 +0200 Subject: [PATCH 17/40] Show Observability in the default fixture The default fixture now matches the initial ROR config: the common Kibana rules hide only "Enterprise Search|Overview", and the APM server has its own rule. The Observability spec uses the default fixture, so observabilityVisibleSettings.yaml is removed. The apm user is now in the impersonation list, so the kibana and new_user rows move down by one. Co-Authored-By: Claude Opus 5 --- e2e-tests/cypress/e2e/Impersonate.cy.ts | 4 +- e2e-tests/cypress/e2e/Observability.cy.ts | 5 - .../defaultReadonlyRestEsAndKbnSettings.yaml | 6 +- .../observabilityVisibleSettings.yaml | 147 ------------------ 4 files changed, 7 insertions(+), 155 deletions(-) delete mode 100644 e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml diff --git a/e2e-tests/cypress/e2e/Impersonate.cy.ts b/e2e-tests/cypress/e2e/Impersonate.cy.ts index d7256588..c6127dd8 100644 --- a/e2e-tests/cypress/e2e/Impersonate.cy.ts +++ b/e2e-tests/cypress/e2e/Impersonate.cy.ts @@ -80,7 +80,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(3, 'Local users'); Impersonate.assertServiceType(3, 'local'); Impersonate.assertServiceColumns(3, ['Username']); - Impersonate.assertUser(3, 2, 'kibana'); + Impersonate.assertUser(3, 3, 'kibana'); }; createLdapUsers(); @@ -102,7 +102,7 @@ describe('impersonate', () => { cy.log('should impersonate localUser'); Impersonate.open(); - Impersonate.impersonateUserFromTheList(3, 3, 'new_user'); + Impersonate.impersonateUserFromTheList(3, 4, 'new_user'); Impersonate.finishImpersonation(); Impersonate.verifyFinishedImpersonation(); diff --git a/e2e-tests/cypress/e2e/Observability.cy.ts b/e2e-tests/cypress/e2e/Observability.cy.ts index b2410c4f..8e46f4d0 100644 --- a/e2e-tests/cypress/e2e/Observability.cy.ts +++ b/e2e-tests/cypress/e2e/Observability.cy.ts @@ -1,22 +1,17 @@ import { Login } from '../support/page-objects/Login'; import { KibanaNavigation } from '../support/page-objects/KibanaNavigation'; import { Observability } from '../support/page-objects/Observability'; -import { Settings } from '../support/page-objects/Settings'; import { esApiClient } from '../support/helpers/EsApiClient'; import * as semver from 'semver'; import { getKibanaVersion } from '../support/helpers'; describe('Observability', () => { beforeEach(() => { - // The shared default fixture hides the Observability app (hide_apps), so this spec needs - // its own fixture with that app left visible instead of relying on the shared default. - Settings.setSettingsData('observabilityVisibleSettings.yaml'); Login.initialization(); }); afterEach(() => { esApiClient.deleteIndexDocsByQuery(Observability.APM_DATA_INDEXES_WILDCARD); - Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); }); it('should verify APM functionality', () => { diff --git a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml index d28c5775..e31b8ca2 100644 --- a/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml +++ b/e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml @@ -1,7 +1,7 @@ helpers: ckr: &common-kibana-rules access: rw - hide_apps: ['Enterprise Search|Overview', 'Observability'] + hide_apps: ['Enterprise Search|Overview'] index: '.kibana_@{acl:current_group}' ag: &all-groups @@ -42,6 +42,10 @@ readonlyrest: auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' # + - name: 'APM server - system account' + verbosity: error + auth_key: apm:test + - name: JWT_AUTH jwt_auth: name: 'jwt1' diff --git a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml b/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml deleted file mode 100644 index e31b8ca2..00000000 --- a/e2e-tests/cypress/fixtures/observabilityVisibleSettings.yaml +++ /dev/null @@ -1,147 +0,0 @@ -helpers: - ckr: &common-kibana-rules - access: rw - hide_apps: ['Enterprise Search|Overview'] - index: '.kibana_@{acl:current_group}' - - ag: &all-groups - groups: - - id: admins_group - name: administrators - - id: infosec_group - name: infosec - - id: template_group - name: template - -readonlyrest: - response_if_req_forbidden: You shall not pass! - audit: - enabled: true - outputs: - - type: index - index_template: "'readonlyrest_audit_'yyyy-MM-dd" - - access_control_rules: - # <-- related to ECK environment --> - - name: 'Kibana service account - token' - verbosity: error - token_authentication: - token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}' - username: service_account - - - name: 'Kibana service account - user/pass' - verbosity: error - auth_key: kibana:kibana - - - name: 'PROBE' - verbosity: error - auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}' - - - name: 'ELASTIC-INTERNAL' - verbosity: error - auth_key: 'elastic-internal:${INTERNAL_USR_PASS}' - # - - - name: 'APM server - system account' - verbosity: error - auth_key: apm:test - - - name: JWT_AUTH - jwt_auth: - name: 'jwt1' - groups_any_of: ['administrators', 'infosec', 'template'] - kibana: - access: admin - - - name: USER_DEFAULT - auth_key: user2:dev - verbosity: error - indices: ['kibana_sample_data_*'] - kibana: - access: rw - index: '.default_index' - - - name: PERSONAL_GRP - groups: [Personal] - kibana: - <<: *common-kibana-rules - index: '.kibana_@{user}' - - - name: ADMIN_GRP - groups: [admins_group] - kibana: - <<: *common-kibana-rules - access: admin - metadata: - alert_message: 'Dear @{acl:user}' - - name: infosec - groups: [infosec_group] - kibana: - <<: *common-kibana-rules - access: admin - hide_apps: ['Enterprise Search|Overview', 'Observability', 'Management'] - - - name: Template Tenancy - groups: [template_group] - kibana: - <<: *common-kibana-rules - - - name: 'ReadonlyREST Enterprise instance #1' - kibana_index: '.kibana_external_auth' - ror_kbn_auth: - name: 'kbn1' - - users: - - username: admin - auth_key: admin:dev - <<: *all-groups - - - username: user1 - auth_key: user1:dev - <<: *all-groups - - - username: '*' - jwt_auth: - name: 'jwt1' - groups: - - local_group: - id: admins_group - name: administrators - external_group_ids: ['administrators'] - - local_group: - id: infosec_group - name: infosec - external_group_ids: ['infosec'] - - local_group: - id: template_group - name: template - external_group_ids: ['template'] - - jwt: - - name: jwt1 - signature_key: 'a-string-secret-at-least-256-bits-long' - group_ids_claim: group - user_claim: sub - header_name: Authorization - - ror_kbn: - - name: kbn1 - signature_key: '9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf' - - impersonation: - - impersonator: admin - users: ['*'] - auth_key: admin:dev - -readonlyrest_kbn: - cookiePass: '12312313123213123213123adadasdasdasd' - logLevel: 'trace' - logPrettyPrintEnabled: true - whitelistedPaths: [".*/api/status$"] - clearSessionOnEvents: [login, tenancyHop] - sessions_probe_interval_seconds: 60 - store_sessions_in_index: true - login_title: Loaded from index! - login_subtitle: 'PRO/Enterprise: You should see a red border, a tiny unicorn logo, a two column page, and this text. You should see none of these customisation when testing ROR Free.' - login_custom_logo: 'https://i.imgur.com/MdRBUfV.gif' - login_html_head_inject: '' From 4bb460c3e7527a846bcc5fa4774591f7543b3692 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:51:34 +0200 Subject: [PATCH 18/40] Remove transport retries from the e2e HTTP client publishNotReadyAddresses on the kbn-np Service stops the connection resets in the eck-ror environment. The client-side retries only hid them, and the 5 s request timeout sent slow POST requests again. The plugin is back to its epic version. timeoutMs now only extends the cy.task timeout for loadSampleData. Co-Authored-By: Claude Opus 5 --- e2e-tests/cypress/plugins/index.ts | 129 ++++---------------------- e2e-tests/cypress/support/commands.ts | 9 +- 2 files changed, 22 insertions(+), 116 deletions(-) diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index 604f31cf..8b2769c2 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -7,19 +7,6 @@ import { inspect } from 'util'; import path from 'node:path'; import * as fs from 'node:fs'; -// Shared and kept alive across calls so requests reuse an established TCP+TLS connection instead -// of each `httpCall`/`uploadFile` negotiating a brand-new one. In the eck-ror CI environment, -// Kibana is reached through kind's NodePort/iptables overlay rather than a direct docker port -// mapping, and that extra hop is where the many short-lived connections a fresh Agent-per-call -// created were most likely to get dropped or hang. -const sharedHttpsAgent: Agent = new Agent({ - rejectUnauthorized: false, - secureProtocol: 'TLSv1_2_method', - keepAlive: true, - keepAliveMsecs: 1000, - maxSockets: 50 -}); - let embeddedServer: ReturnType | null = null; const EMBEDDED_SERVER_PORT = 8080; const ROOT_DIR = path.join(__dirname, '..', '..', '..'); @@ -45,41 +32,6 @@ const formatLoggerData = (data: unknown) => const NON_JSON_RETRY_ATTEMPTS = 5; const NON_JSON_RETRY_DELAY_MS = 2000; -// The eck-ror CI environment reaches Kibana through kind's NodePort/iptables overlay instead of -// a direct docker port mapping, which occasionally drops or hangs a TCP connection outright -// (ECONNRESET, socket hang up) rather than serving a slow-but-valid response. Without a retry -// here, a single dropped connection burns the whole cy.task timeout and, since Cypress only -// prints failures once the spec finishes, can silently take the rest of the spec down with it. -const TRANSPORT_ERROR_RETRY_ATTEMPTS = 3; -const TRANSPORT_ERROR_RETRY_DELAY_MS = 1000; -// Without a per-request timeout, a socket that hangs instead of dropping outright (no -// ECONNRESET, just silence) burns the entire cy.task `taskTimeout` (20000ms) on its first -// attempt, so the retry loop above never even gets a chance to run. Capping each attempt well -// under a third of that budget guarantees all TRANSPORT_ERROR_RETRY_ATTEMPTS attempts (plus their -// TRANSPORT_ERROR_RETRY_DELAY_MS sleeps) fit inside taskTimeout even in the worst case. -const FETCH_TIMEOUT_MS = 5000; -const TRANSIENT_NETWORK_ERROR_CODES = new Set([ - 'ECONNRESET', - 'ECONNREFUSED', - 'ETIMEDOUT', - 'EPIPE', - 'EHOSTUNREACH', - 'ENETUNREACH' -]); - -const isTransientNetworkError = (error: unknown): boolean => { - const err = error as { code?: string; type?: string; message?: string }; - if (err?.code && TRANSIENT_NETWORK_ERROR_CODES.has(err.code)) { - return true; - } - // node-fetch's own `timeout` option (set via FETCH_TIMEOUT_MS above) surfaces as a - // FetchError with type 'request-timeout' rather than one of the Node error codes above. - if (err?.type === 'request-timeout') { - return true; - } - return typeof err?.message === 'string' && err.message.includes('socket hang up'); -}; - const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); // Right after a Kibana restart, ROR-KBN can still be finishing its own settings load (an ES @@ -90,20 +42,11 @@ const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(r // the caller to read exactly as before. // `createInit` is a factory (not a static object) because a retried attempt needs its own // request body - a FormData upload's underlying stream can only be read once. -// -// `retryOnTransportError` is off for calls that intentionally expect the connection to be reset -// (e.g. /pkp/api/kibanaConfig SIGINTs Kibana before writing its reply) - those should fail fast -// into the caller's own handling instead of burning retries on an error that's the expected outcome. -const fetchWithJsonRetry = async ( - url: string, - createInit: () => Parameters[1], - retryOnTransportError = true, - timeoutMs = FETCH_TIMEOUT_MS -): Promise => { +const fetchWithJsonRetry = async (url: string, createInit: () => Parameters[1]): Promise => { let response: Response; for (let attempt = 1; attempt <= NON_JSON_RETRY_ATTEMPTS; attempt++) { // eslint-disable-next-line no-await-in-loop - response = await fetchWithTransportRetry(url, createInit, retryOnTransportError, timeoutMs); + response = await fetch(url, createInit()); const contentType = response.headers.get('content-type') || ''; // The startup race serves Kibana's login page (text/html) in place of the expected @@ -129,56 +72,23 @@ const fetchWithJsonRetry = async ( return response!; }; -const fetchWithTransportRetry = async ( - url: string, - createInit: () => Parameters[1], - retryOnTransportError: boolean, - timeoutMs: number -): Promise => { - for (let attempt = 1; attempt <= TRANSPORT_ERROR_RETRY_ATTEMPTS; attempt++) { - try { - // eslint-disable-next-line no-await-in-loop - return await fetch(url, { timeout: timeoutMs, ...createInit() }); - } catch (error) { - const isLastAttempt = attempt === TRANSPORT_ERROR_RETRY_ATTEMPTS; - if (!retryOnTransportError || !isTransientNetworkError(error) || isLastAttempt) { - throw error; - } - console.log( - `Transient network error (${ - (error as Error).message - }) for ${url} - retrying (${attempt}/${TRANSPORT_ERROR_RETRY_ATTEMPTS})...` - ); - // A transient error on one request can leave other keep-alive sockets in the shared pool - // half-broken too (the same kind NodePort hop dropped them all around the same time), and a - // retry that happens to grab one of those instead of opening a fresh connection fails the - // same way. Destroying the whole pool forces the retry onto a brand-new TCP+TLS connection. - sharedHttpsAgent.destroy(); - // eslint-disable-next-line no-await-in-loop - await sleep(TRANSPORT_ERROR_RETRY_DELAY_MS); - } - } - // Unreachable: the loop above always either returns or throws. - throw new Error(`Unreachable: exhausted retries for ${url} without returning or throwing`); -}; - module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { on('task', { async httpCall(options: HttpCallOptions): Promise { - const { method, url, headers, body, failOnStatusCode, allowTransportError, timeoutMs } = options; + const { method, url, headers, body, failOnStatusCode, allowTransportError } = options; + + const agent: Agent = new Agent({ + rejectUnauthorized: false, + secureProtocol: 'TLSv1_2_method' + }); try { - const response: Response = await fetchWithJsonRetry( - url, - () => ({ - method, - headers, - body: body ?? undefined, - agent: sharedHttpsAgent - }), - !allowTransportError, - timeoutMs - ); + const response: Response = await fetchWithJsonRetry(url, () => ({ + method, + headers, + body: body ?? undefined, + agent + })); if (!response.ok && failOnStatusCode) { throw new Error( @@ -214,6 +124,11 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) async uploadFile(options: UploadFileOptions): Promise { const { url, headers, file } = options; + const agent: Agent = new Agent({ + rejectUnauthorized: false, + secureProtocol: 'TLSv1_2_method' + }); + const buildForm = (): { form: FormData; combinedHeaders: { [key: string]: string } } => { const form = new FormData(); form.append('file', file.fileBinaryContent, { @@ -229,7 +144,7 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) try { const response: Response = await fetchWithJsonRetry(url, () => { const { form, combinedHeaders } = buildForm(); - return { method, headers: combinedHeaders, body: form, agent: sharedHttpsAgent }; + return { method, headers: combinedHeaders, body: form, agent }; }); if (!response.ok) { @@ -432,10 +347,6 @@ interface HttpCallOptions { failOnStatusCode?: boolean; // For endpoints that restart the server they answer from, so the reply is lost by design. allowTransportError?: boolean; - // Overrides FETCH_TIMEOUT_MS for endpoints that are legitimately slower than a transport-error - // check needs to be, e.g. Kibana's sample-data install (creates an index, then bulk-inserts - // thousands of documents into it). - timeoutMs?: number; } interface FileToUpload { diff --git a/e2e-tests/cypress/support/commands.ts b/e2e-tests/cypress/support/commands.ts index 5fa10c49..67d3736e 100644 --- a/e2e-tests/cypress/support/commands.ts +++ b/e2e-tests/cypress/support/commands.ts @@ -162,15 +162,10 @@ function httpCall( ...headers }, body: payload ? (typeof payload === 'string' ? payload : JSON.stringify(payload)) : null, - failOnStatusCode, - timeoutMs + failOnStatusCode }; - // cy.task()'s own default timeout (taskTimeout in cypress.config.ts) is sized for the default - // FETCH_TIMEOUT_MS budget (see plugins/index.ts) - a caller overriding timeoutMs upward (e.g. - // loadSampleData's slower bulk-insert) needs a matching override here, or Cypress kills the - // whole task at the unmodified default before the fetch's own longer timeout is ever reached. - return timeoutMs ? cy.task('httpCall', options, { timeout: timeoutMs + 5000 }) : cy.task('httpCall', options); + return timeoutMs ? cy.task('httpCall', options, { timeout: timeoutMs }) : cy.task('httpCall', options); } function uploadFile( From c5f76a188cc1521b48a5b9cd8cb7f6c4462f3efd Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:51:45 +0200 Subject: [PATCH 19/40] Mark the settings propagation delay as a workaround The comment now says why the fixed wait stays and when to remove it. The history of the replaced poll moves out of the code. Co-Authored-By: Claude Opus 5 --- .../cypress/support/helpers/RorApiClient.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index df1061b2..1007f387 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -1,17 +1,6 @@ -// The ROR Kibana plugin picks up a new index-stored config asynchronously - each kbn-ror node -// only refreshes its in-memory settings when its own cache goes stale, not the instant the POST -// below returns. Proceeding immediately (e.g. straight into Login.initialization()) races that -// refresh: observed lag between a successful POST and the new config being active was up to ~10s. -// See run-20260831-073804-1.log for the flaky "should disable multitenancy" / "should verify index -// based session" failures this caused. -// -// A re-POST-and-check-for-idempotency poll was tried here instead of a flat sleep, but the -// idempotency check compares against the config already persisted in the ES index - which the -// first POST writes immediately - not against any single node's in-memory cache. Against the -// docker env's 2 kbn-ror replicas behind kbn-proxy's round robin, that made the "poll" resolve -// after its very first iteration (~1s) regardless of whether either replica had actually -// refreshed, which was worse than the flat delay it replaced. See run-34312427155 for the -// resulting "should verify index based session" / Hide_apps / Sanity-check flakiness. +// Workaround: each ROR KBN node loads new index settings only when its settings cache expires, +// up to ~10 s after this POST returns. A poll cannot detect this, because every check reads the +// settings in ES. Remove this wait when ROR KBN applies new settings before the POST returns. const SETTINGS_PROPAGATION_DELAY_MS = 8000; export class RorApiClient { From 11873ead54adfaf5d1eafd6333c60d1b6fda2770 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:52:20 +0200 Subject: [PATCH 20/40] Make comments describe the current state Comments in the eck-ror environment and the S3 upload script now say why the code is like this. The history of each fix stays in the commit messages, as docs/dev/code-style.md requires. Co-Authored-By: Claude Opus 5 --- .../scripts/upload-cypress-artifacts-to-s3.sh | 6 +----- .../eck-ror/kind-cluster/ror/base/kbn-np.yml | 7 ++----- .../eck-ror/kind-cluster/ror/base/kbn.yml | 18 +++++------------- environments/eck-ror/stop-and-clean.sh | 6 +----- 4 files changed, 9 insertions(+), 28 deletions(-) diff --git a/.github/scripts/upload-cypress-artifacts-to-s3.sh b/.github/scripts/upload-cypress-artifacts-to-s3.sh index a744bacc..cbb7ea67 100755 --- a/.github/scripts/upload-cypress-artifacts-to-s3.sh +++ b/.github/scripts/upload-cypress-artifacts-to-s3.sh @@ -51,11 +51,7 @@ AK="${!AK_VAR}" SK="${!SK_VAR}" BUCKET="${!BUCKET_VAR}" REGION="${!REGION_VAR}" -# Strip any trailing slash regardless of how the caller formatted it - upload-videos/action.yml -# builds this from "/build_" and if path_prefix itself already ends in "/" -# (as it did for the DGP endpoint), the unstripped value produces a "//" in every S3 key below, -# which the endpoint rejects outright: "InvalidArgument: Key must not contain empty path -# segments ('//')" - the cause of every artifact upload failing in run 34312427155. +# A trailing slash in the prefix gives "//" in each S3 key, and the endpoint rejects such keys. PATH_PREFIX="${!PREFIX_VAR:-}" PATH_PREFIX="${PATH_PREFIX%/}" diff --git a/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml b/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml index 3b848815..77527363 100644 --- a/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml +++ b/environments/eck-ror/kind-cluster/ror/base/kbn-np.yml @@ -4,11 +4,8 @@ metadata: name: eck-ror-kbn-np spec: type: NodePort - # Keeps the pod's endpoint in the Service even while it's failing readiness. With only a - # single Kibana replica in this cluster, pulling the endpoint on every readiness flap (CPU - # contention on the kind node, GC pause, etc.) turns any in-flight request into a bare TCP - # reset (ECONNRESET) instead of a slow-but-valid response - that's the source of the - # widespread ECONNRESET flakiness seen in e2e runs, not the e2e HTTP client itself. + # The cluster has one Kibana pod. Without this, each readiness failure removes its endpoint, + # and each open request ends in ECONNRESET instead of a slow response. publishNotReadyAddresses: true ports: - port: 5601 diff --git a/environments/eck-ror/kind-cluster/ror/base/kbn.yml b/environments/eck-ror/kind-cluster/ror/base/kbn.yml index fc3b7daa..f7eb6dd7 100644 --- a/environments/eck-ror/kind-cluster/ror/base/kbn.yml +++ b/environments/eck-ror/kind-cluster/ror/base/kbn.yml @@ -35,16 +35,10 @@ spec: - name: I_UNDERSTAND_AND_ACCEPT_KBN_PATCHING value: "yes" - name: NODE_OPTIONS - # 768 was tight enough on its own to be a plausible contributor to the readiness - # flapping below (GC pressure blocking the event loop long enough to miss a probe). - # Raised in step with the memory limit increase below. + # A smaller heap causes GC pauses long enough to fail the readiness probe. value: "--max-old-space-size=1536" - # The default readiness probe (timeoutSeconds: 5, failureThreshold: 3) flips the pod - # NotReady under normal kind/CI CPU contention. `publishNotReadyAddresses` on the - # kbn-np Service (see kbn-np.yml) now stops that flapping from turning into a hard - # ECONNRESET at the network level, but the probe's own timing was still inverted - - # timeoutSeconds must be <= periodSeconds or a slow-but-alive response can be judged - # a failure before the next check even starts. + # The default probe (timeoutSeconds: 5, failureThreshold: 3) fails under CI CPU load. + # timeoutSeconds must not be more than periodSeconds. readinessProbe: httpGet: path: /api/status @@ -55,10 +49,8 @@ spec: timeoutSeconds: 10 failureThreshold: 6 successThreshold: 1 - # No CPU limit is set deliberately (avoids throttling), but an explicit request gives - # Kibana a fairer scheduling share against ES/APM/kind's own control-plane pods sharing - # the same Docker Desktop CPU budget. Memory limit raised alongside NODE_OPTIONS above - - # 2Gi left barely any headroom over the 768Mi heap cap for Kibana's own process memory. + # No CPU limit, to prevent throttling. The CPU request gives Kibana a share against the + # other pods on the kind node. The memory limit leaves space above the heap cap. resources: requests: cpu: '500m' diff --git a/environments/eck-ror/stop-and-clean.sh b/environments/eck-ror/stop-and-clean.sh index 865b392b..f06c8a58 100755 --- a/environments/eck-ror/stop-and-clean.sh +++ b/environments/eck-ror/stop-and-clean.sh @@ -3,11 +3,7 @@ set -e cd "$(dirname "$0")" -# runner.sh runs this via `trap cleanup EXIT`, so it fires before the calling GitHub Actions step -# returns - including the "Stop Docker memory monitor" step, whose final log lines were always -# empty ("== kubectl pod status ==" with nothing after it) because `kind delete cluster` below had -# already torn the cluster down by the time that step's `kubectl` calls ran. Dumping the pod/event -# state here, one last time before deletion, keeps a non-empty final snapshot for post-mortem. +# Log the last cluster state before the cluster is deleted, for failure analysis. echo "== final pod status before teardown ==" kubectl --request-timeout=10s get pods -A 2>/dev/null || true echo "== final warning events before teardown ==" From 35824101a929b525a281b4ab50ff36a7a3b4aeae Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:54:25 +0200 Subject: [PATCH 21/40] Remove redundant trailing-slash strip in S3 upload script S3_PATH already strips the trailing slash off PATH_PREFIX when it builds the key, and the sed right after collapses any "//" left in the middle. Stripping it again on PATH_PREFIX itself did nothing. Co-Authored-By: Claude Sonnet 5 --- .github/scripts/upload-cypress-artifacts-to-s3.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/scripts/upload-cypress-artifacts-to-s3.sh b/.github/scripts/upload-cypress-artifacts-to-s3.sh index cbb7ea67..82c1e982 100755 --- a/.github/scripts/upload-cypress-artifacts-to-s3.sh +++ b/.github/scripts/upload-cypress-artifacts-to-s3.sh @@ -51,9 +51,7 @@ AK="${!AK_VAR}" SK="${!SK_VAR}" BUCKET="${!BUCKET_VAR}" REGION="${!REGION_VAR}" -# A trailing slash in the prefix gives "//" in each S3 key, and the endpoint rejects such keys. PATH_PREFIX="${!PREFIX_VAR:-}" -PATH_PREFIX="${PATH_PREFIX%/}" SOURCE_DIR="${1:?Usage: upload-cypress-artifacts-to-s3.sh }" S3_SUBFOLDER="${2:?Usage: upload-cypress-artifacts-to-s3.sh }" From 047294ead13c76e397e650d7c3d37825a6d1cc57 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 16:55:33 +0200 Subject: [PATCH 22/40] Revert unused --spec flag on the run script Nothing in runner.sh or CI sets SPEC or relies on this flag. Co-Authored-By: Claude Sonnet 5 --- e2e-tests/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/package.json b/e2e-tests/package.json index bc5da681..b311a729 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -7,7 +7,7 @@ "lint": "eslint .", "lint:fix": "yarn lint -- --fix", "open": "./node_modules/.bin/cypress open", - "run": "ELECTRON_ENABLE_LOGGING=1 ELECTRON_EXTRA_LAUNCH_ARGS='--ignore-gpu-blocklist' ./node_modules/.bin/cypress run --spec \"${SPEC:-cypress/e2e/**/*.cy.ts}\"", + "run": "ELECTRON_ENABLE_LOGGING=1 ELECTRON_EXTRA_LAUNCH_ARGS='--ignore-gpu-blocklist' ./node_modules/.bin/cypress run", "tsCheck": "node ./node_modules/typescript/bin/tsc --noEmit -p tsconfig.json" }, "license": "Beshu Limited, All rights reserved", From 1435e4c87ed9383bf6ae2f18dda3d20d0078150d Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 19:45:55 +0200 Subject: [PATCH 23/40] Fix Impersonate row indices broken by the Observability commit 116a7ac shifted these indices assuming the apm user added to defaultReadonlyRestEsAndKbnSettings.yaml also shifted the local-users list here. This spec loads testSettings.yaml instead, which has no apm rule, so the local-users list is unaffected and the indices go back to their pre-116a7ac values. Co-Authored-By: Claude Sonnet 5 --- e2e-tests/cypress/e2e/Impersonate.cy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/e2e/Impersonate.cy.ts b/e2e-tests/cypress/e2e/Impersonate.cy.ts index c6127dd8..d7256588 100644 --- a/e2e-tests/cypress/e2e/Impersonate.cy.ts +++ b/e2e-tests/cypress/e2e/Impersonate.cy.ts @@ -80,7 +80,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(3, 'Local users'); Impersonate.assertServiceType(3, 'local'); Impersonate.assertServiceColumns(3, ['Username']); - Impersonate.assertUser(3, 3, 'kibana'); + Impersonate.assertUser(3, 2, 'kibana'); }; createLdapUsers(); @@ -102,7 +102,7 @@ describe('impersonate', () => { cy.log('should impersonate localUser'); Impersonate.open(); - Impersonate.impersonateUserFromTheList(3, 4, 'new_user'); + Impersonate.impersonateUserFromTheList(3, 3, 'new_user'); Impersonate.finishImpersonation(); Impersonate.verifyFinishedImpersonation(); From e644c5ce160b36cd3760ff4509b82cc0bac8c946 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sat, 19 Sep 2026 22:19:07 +0200 Subject: [PATCH 24/40] Remove pageLoadTimeout override from Hide_apps Verified against a live elk-ror stack (Kibana 9.4.7): the whole spec passes twice in a row with the default 20s pageLoadTimeout, with no run coming close to it. restoreDefaultSettingsData() now unloads the page to about:blank before posting new settings, so the next test's Login.initialization() does a full navigation instead of the SPA-triggered reload this override was raised for. Co-Authored-By: Claude Sonnet 5 --- e2e-tests/cypress/e2e/Hide_apps.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/cypress/e2e/Hide_apps.cy.ts b/e2e-tests/cypress/e2e/Hide_apps.cy.ts index a6de03e9..1d1db677 100644 --- a/e2e-tests/cypress/e2e/Hide_apps.cy.ts +++ b/e2e-tests/cypress/e2e/Hide_apps.cy.ts @@ -10,7 +10,7 @@ import { SearchApps } from '../support/page-objects/SearchApps'; import { Loader } from '../support/page-objects/Loader'; import { Home } from '../support/page-objects/Home'; -describe('hidden apps', { pageLoadTimeout: 60000 }, () => { +describe('hidden apps', () => { afterEach(() => { Settings.restoreDefaultSettingsData(); }); From 3e82d983f0fedbe843c9933bfb7ef9787ad1c9ac Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sun, 20 Sep 2026 06:18:39 +0200 Subject: [PATCH 25/40] Wait for settings propagation only in elk-ror The node that answers the settings POST applies them before it replies. Only the other nodes lag, by one 5 s poll of the index. elk-ror runs 2 kbn-ror replicas, so it needs the wait; eck-ror runs a single Kibana node and does not. Verified against a live stack, with the wait removed: a single replica passes 4/4 and the spec runs in 6-17 s, while 2 replicas fail with a bounce back to /login. The comment now describes that mechanism instead of the cache expiry it claimed before. Co-Authored-By: Claude Opus 5 --- e2e-tests/cypress/support/helpers/RorApiClient.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index 1007f387..228f032b 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -1,6 +1,9 @@ -// Workaround: each ROR KBN node loads new index settings only when its settings cache expires, -// up to ~10 s after this POST returns. A poll cannot detect this, because every check reads the -// settings in ES. Remove this wait when ROR KBN applies new settings before the POST returns. +// The node that answers this POST applies the new settings before it replies. Every other node +// picks them up from its own poll of the index, which runs every 5 s. Until then it still serves +// the old settings, so a login on one node is not recognized by the other and bounces to /login. +// The wait covers one poll interval with a margin. Only elk-ror needs it - it runs 2 kbn-ror +// replicas behind kbn-proxy's round robin, while eck-ror runs a single Kibana node. +// FIXME: See RORDEV-2235. const SETTINGS_PROPAGATION_DELAY_MS = 8000; export class RorApiClient { @@ -19,7 +22,7 @@ export class RorApiClient { // config - e.g. two specs in a row both resetting to the same default fixture. That's // the desired state, not an error; only a genuinely different failure should throw. if (response.status === 'SUCCESS') { - return cy.wait(SETTINGS_PROPAGATION_DELAY_MS); + return Cypress.env().envName === 'elk-ror' ? cy.wait(SETTINGS_PROPAGATION_DELAY_MS) : undefined; } if (response.message !== 'Current settings are already loaded') { throw new Error(`Failed to configure ROR index main settings: ${JSON.stringify(response)}`); From 750f76917f1ba6e470bf8e163b4f0e325e65fc4f Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Sun, 20 Sep 2026 08:36:12 +0200 Subject: [PATCH 26/40] Expect the /exports reporting route from Kibana 9.5.0 The reporting management page settles on an /exports child route on 8.19.x and again from 9.5.0, but the version gate only covered 8.19.x. On 9.5.4, which the CI matrix started running in 6110d57, Sanity-check therefore waited 20 s for a URL that never came and failed. The stale expectation only ever held by accident: entering at the bare /reporting path, the retrying matcher catches that transient state before the client-side redirect. It breaks where the assertion starts after a reload, which is what changeTenancy does - the app comes back on its settled route. Reproduced that way against a live 9.5.4 stack, with the same error the pipeline reported, and verified the fix clears it. The gate lived in two copies that had to agree, so it now has a single home in Reporting.pagePath. Co-Authored-By: Claude Opus 5 --- e2e-tests/cypress/e2e/Sanity-check.cy.ts | 7 +------ e2e-tests/cypress/support/page-objects/Reporting.ts | 12 +++++++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/e2e-tests/cypress/e2e/Sanity-check.cy.ts b/e2e-tests/cypress/e2e/Sanity-check.cy.ts index 4582b0ab..222bd376 100644 --- a/e2e-tests/cypress/e2e/Sanity-check.cy.ts +++ b/e2e-tests/cypress/e2e/Sanity-check.cy.ts @@ -61,12 +61,7 @@ describe('sanity check', () => { Reporting.downloadAndVerifyAnyReportExists(); cy.log('Change tenancy, and initialize it'); - const finishUrl = - semver.gte(getKibanaVersion(), '8.19.0') && semver.lt(getKibanaVersion(), '9.0.0') - ? '/app/management/insightsAndAlerting/reporting/exports' - : '/app/management/insightsAndAlerting/reporting'; - - RorMenu.changeTenancy('Infosec', finishUrl); + RorMenu.changeTenancy('Infosec', Reporting.pagePath); if (semver.gte(getKibanaVersion(), '8.8.0')) { Reporting.noReportsCreatedCheck('rorMenu'); diff --git a/e2e-tests/cypress/support/page-objects/Reporting.ts b/e2e-tests/cypress/support/page-objects/Reporting.ts index e51b6f6f..2206f34d 100644 --- a/e2e-tests/cypress/support/page-objects/Reporting.ts +++ b/e2e-tests/cypress/support/page-objects/Reporting.ts @@ -9,6 +9,14 @@ import { KibanaToast } from './KibanaToast'; type OpenBy = 'rorMenu' | 'kibanaNavigation'; export class Reporting { + // Kibana settles the reporting management page on an /exports child route on 8.19.x and again + // from 9.5.0. The 9.0-9.4 line serves the bare /reporting path. + static get pagePath() { + return semver.satisfies(getKibanaVersion(), '>=8.19.0 <9.0.0 || >=9.5.0') + ? '/app/management/insightsAndAlerting/reporting/exports' + : '/app/management/insightsAndAlerting/reporting'; + } + static noReportsCreatedCheck(openBy: OpenBy) { cy.log('noReportsCreatedCheck'); this.openReportingPage(openBy); @@ -49,9 +57,7 @@ export class Reporting { static verifyIfReportingPageAfterRefresh() { cy.log('Verify if reporting page open after refresh'); - const expectedUrl = semver.satisfies(getKibanaVersion(), '>=8.19.0 <9.0.0') - ? `${Cypress.config().baseUrl}/s/default/app/management/insightsAndAlerting/reporting/exports` - : `${Cypress.config().baseUrl}/s/default/app/management/insightsAndAlerting/reporting`; + const expectedUrl = `${Cypress.config().baseUrl}/s/default${Reporting.pagePath}`; cy.url().should('include', expectedUrl); From 0b3573ad3ab90af0ff4868d339adc80ade1bc4e8 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 21 Sep 2026 06:49:20 +0200 Subject: [PATCH 27/40] Improve sample data installation polling condition --- e2e-tests/cypress/support/helpers/KbnApiClient.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-tests/cypress/support/helpers/KbnApiClient.ts b/e2e-tests/cypress/support/helpers/KbnApiClient.ts index 787b1565..d98032df 100644 --- a/e2e-tests/cypress/support/helpers/KbnApiClient.ts +++ b/e2e-tests/cypress/support/helpers/KbnApiClient.ts @@ -77,17 +77,17 @@ export class KbnApiClient { group?: string, timeout = 90000, interval = 5000 - ): Cypress.Chainable<{ statusCode?: number }> { + ): Cypress.Chainable<{ statusCode?: number; elasticsearchIndicesCreated?: Record }> { return recurse( () => - cy.kbnPost<{ statusCode?: number }>({ + cy.kbnPost<{ statusCode?: number; elasticsearchIndicesCreated?: Record }>({ endpoint: `api/sample_data/${sampleDatasetName}`, credentials, currentGroupHeader: group, failOnStatusCode: false, timeoutMs: 30000 }), - response => !response?.statusCode, + response => response?.elasticsearchIndicesCreated !== undefined, { timeout, delay: interval, From 1d07871c7b9163885242d6abc20eab7021c2996a Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 21 Sep 2026 06:52:03 +0200 Subject: [PATCH 28/40] Refactor afterEach to use restoreDefaultSettingsData helper --- e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts | 2 +- .../cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts b/e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts index 22a1de71..7cb8f086 100644 --- a/e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts +++ b/e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts @@ -5,7 +5,7 @@ import { userCredentials } from '../support/helpers'; describe('sanity check ro kibana access', () => { afterEach(() => { - Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); + Settings.restoreDefaultSettingsData(); kbnApiClient.deleteSampleData('ecommerce', userCredentials, 'template_group'); }); diff --git a/e2e-tests/cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts b/e2e-tests/cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts index 0f9abd34..78a73b56 100644 --- a/e2e-tests/cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts +++ b/e2e-tests/cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts @@ -5,7 +5,7 @@ import { userCredentials } from '../support/helpers'; describe('sanity check ro_strict kibana access', () => { afterEach(() => { - Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml'); + Settings.restoreDefaultSettingsData(); kbnApiClient.deleteSampleData('ecommerce', userCredentials, 'template_group'); }); From 90675728c18dc90f1930008a9d8913a988f85c1c Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 21 Sep 2026 06:55:29 +0200 Subject: [PATCH 29/40] Simplify save modal retry logic in Settings page object --- e2e-tests/cypress/support/page-objects/Settings.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index a4a58629..b6e3fbcd 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -8,7 +8,6 @@ import { parseKbnSettings } from '../helpers/parseKibanaSettings'; export class Settings { private static readonly SAVE_MODAL_SETTLE_MS = 300; - private static readonly SAVE_MODAL_RETRY_ATTEMPTS = 3; static open() { cy.log('Open settings'); @@ -73,12 +72,10 @@ export class Settings { Settings.clickSaveButton(); } }) - .then(() => cy.wait(Settings.SAVE_MODAL_SETTLE_MS, { log: false })) .then(() => SecuritySettings.getIframeBody()), $body => ($body as JQuery).find(':contains("Save anyway")').length > 0, { - limit: Settings.SAVE_MODAL_RETRY_ATTEMPTS, - delay: 0, + delay: Settings.SAVE_MODAL_SETTLE_MS, timeout: 20000, // confirmSaveModal() asserts on the modal text right after this, so failing here would // only replace that message with a less specific one. From f52dc09767c54159b4d186e4ad99bfe4cc6aec8a Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 21 Sep 2026 06:58:01 +0200 Subject: [PATCH 30/40] Remove rowIndex parameter from assertUser --- e2e-tests/cypress/e2e/Impersonate.cy.ts | 12 ++++++------ .../cypress/support/page-objects/Impersonate.ts | 11 +++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/e2e-tests/cypress/e2e/Impersonate.cy.ts b/e2e-tests/cypress/e2e/Impersonate.cy.ts index d7256588..4de2a24a 100644 --- a/e2e-tests/cypress/e2e/Impersonate.cy.ts +++ b/e2e-tests/cypress/e2e/Impersonate.cy.ts @@ -55,8 +55,8 @@ describe('impersonate', () => { Impersonate.assertServiceName(0, 'LDAP 1'); Impersonate.assertServiceType(0, 'ldap'); Impersonate.assertServiceColumns(0, ['Username', 'Groups']); - Impersonate.assertUser(0, 0, 'JohnDoe', ['group3']); - Impersonate.assertUser(0, 1, 'RobertSmith', ['group3']); + Impersonate.assertUser(0, 'JohnDoe', ['group3']); + Impersonate.assertUser(0, 'RobertSmith', ['group3']); }; const assertAuthnService = () => { @@ -64,7 +64,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(1, 'ACME1 External Authorization Service'); Impersonate.assertServiceType(1, 'authn'); Impersonate.assertServiceColumns(1, ['Username']); - Impersonate.assertUser(1, 0, 'JaneDoe'); + Impersonate.assertUser(1, 'JaneDoe'); }; const assertAuthzService = () => { @@ -72,7 +72,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(2, 'ACME2 External Authentication Service'); Impersonate.assertServiceType(2, 'authz'); Impersonate.assertServiceColumns(2, ['Username', 'Groups']); - Impersonate.assertUser(2, 0, 'JaimeRhynes', ['Customer']); + Impersonate.assertUser(2, 'JaimeRhynes', ['Customer']); }; const assertLocalUser = () => { @@ -80,7 +80,7 @@ describe('impersonate', () => { Impersonate.assertServiceName(3, 'Local users'); Impersonate.assertServiceType(3, 'local'); Impersonate.assertServiceColumns(3, ['Username']); - Impersonate.assertUser(3, 2, 'kibana'); + Impersonate.assertUser(3, 'kibana'); }; createLdapUsers(); @@ -98,7 +98,7 @@ describe('impersonate', () => { Impersonate.openEditAuthMockDialog(2); Impersonate.addEditMockUser('kibana', ['group3']); Impersonate.saveEditMockUsers(); - Impersonate.assertUser(2, 1, 'kibana', ['group3']); + Impersonate.assertUser(2, 'kibana', ['group3']); cy.log('should impersonate localUser'); Impersonate.open(); diff --git a/e2e-tests/cypress/support/page-objects/Impersonate.ts b/e2e-tests/cypress/support/page-objects/Impersonate.ts index 73574941..564730be 100644 --- a/e2e-tests/cypress/support/page-objects/Impersonate.ts +++ b/e2e-tests/cypress/support/page-objects/Impersonate.ts @@ -47,12 +47,11 @@ export class Impersonate { cy.get('@service').contains(name); } - static assertUser(index: number, rowIndex: number, username: string, groups?: string[], hasImpersonateButton = true) { + static assertUser(index: number, username: string, groups?: string[], hasImpersonateButton = true) { cy.log('Check user'); Impersonate.getServiceByIndex(index).as('service'); - cy.get('@service').findAllByRole('rowgroup').eq(1).findAllByRole('row').eq(rowIndex).as('rowIndex'); - cy.get('@rowIndex').findByText(username); - cy.get('@rowIndex') + cy.get('@service').findAllByRole('rowgroup').eq(1).contains('[role="row"]', username).as('userRow'); + cy.get('@userRow') .contains('Impersonate') .should(hasImpersonateButton ? 'exist' : 'not.exist'); @@ -62,10 +61,10 @@ export class Impersonate { if (groups.length > 0) { for (const group of groups) { - cy.get('@rowIndex').contains(group); + cy.get('@userRow').contains(group); } } else { - cy.get('@rowIndex').contains('-'); + cy.get('@userRow').contains('-'); } } From deeff10209f3d112f79ff4edbd554bf291bb3136 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Mon, 21 Sep 2026 12:45:51 +0200 Subject: [PATCH 31/40] Refactor Impersonate user row selector to use findByText and closest The change switches from `contains('[role="row"]', username)` to `findByText(username).closest('tr')` for locating user rows in the Impersonate page object. --- e2e-tests/cypress/support/page-objects/Impersonate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/cypress/support/page-objects/Impersonate.ts b/e2e-tests/cypress/support/page-objects/Impersonate.ts index 564730be..7a71c336 100644 --- a/e2e-tests/cypress/support/page-objects/Impersonate.ts +++ b/e2e-tests/cypress/support/page-objects/Impersonate.ts @@ -50,7 +50,7 @@ export class Impersonate { static assertUser(index: number, username: string, groups?: string[], hasImpersonateButton = true) { cy.log('Check user'); Impersonate.getServiceByIndex(index).as('service'); - cy.get('@service').findAllByRole('rowgroup').eq(1).contains('[role="row"]', username).as('userRow'); + cy.get('@service').findAllByRole('rowgroup').eq(1).findByText(username).closest('tr').as('userRow'); cy.get('@userRow') .contains('Impersonate') .should(hasImpersonateButton ? 'exist' : 'not.exist'); From d5386182a0bc7439fa7dd7da93ccd4e8cd0e77cc Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 22 Sep 2026 08:06:34 +0200 Subject: [PATCH 32/40] Remove force option from DevTools editor click --- e2e-tests/cypress/support/page-objects/DevTools.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/DevTools.ts b/e2e-tests/cypress/support/page-objects/DevTools.ts index 91712321..5a01ac36 100644 --- a/e2e-tests/cypress/support/page-objects/DevTools.ts +++ b/e2e-tests/cypress/support/page-objects/DevTools.ts @@ -29,9 +29,10 @@ export class DevTools { // typing before it exists lands on the static "view-lines" rendering div instead, which // isn't a typeable element and throws. Wait for the real input to exist first. cy.get('[data-test-subj="consoleMonacoEditor"] textarea.inputarea').should('exist'); - // The console's action-icon toolbar (euiFlexGroup) can overlap the editor while it settles, - // tripping Cypress's actionability check even though the editor is interactable - force the click. - cy.get('[data-test-subj="consoleMonacoEditor"]').click({ force: true }).type(text); + // The console's action-icon toolbar (euiFlexGroup) can overlap the editor while it settles. + // That cover is real and clears on its own, so let Cypress's actionability retry (bounded by + // defaultCommandTimeout) wait it out instead of forcing past it and risking a swallowed click. + cy.get('[data-test-subj="consoleMonacoEditor"]').click().type(text); cy.get('[data-test-subj="sendRequestButton"]').click(); } else if (semver.lte(getKibanaVersion(), '7.9.0')) { // Select editor, delete, write From ad20e26a1910eadab5fd339c508eeb25514b6414 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 22 Sep 2026 08:46:43 +0200 Subject: [PATCH 33/40] Use isMultiKibanaNodeEnv helper for multi-node checks --- e2e-tests/cypress/e2e/Activation-keys.cy.ts | 4 ++-- e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts | 3 ++- e2e-tests/cypress/support/helpers/RorApiClient.ts | 4 +++- e2e-tests/cypress/support/helpers/index.ts | 5 +++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress/e2e/Activation-keys.cy.ts b/e2e-tests/cypress/e2e/Activation-keys.cy.ts index 2f8b6144..8099ba03 100644 --- a/e2e-tests/cypress/e2e/Activation-keys.cy.ts +++ b/e2e-tests/cypress/e2e/Activation-keys.cy.ts @@ -1,6 +1,6 @@ import { Login } from '../support/page-objects/Login'; import { ActivationKeys } from '../support/page-objects/ActivationKeys'; -import { userCredentials } from '../support/helpers'; +import { isMultiKibanaNodeEnv, userCredentials } from '../support/helpers'; /** * Keys resolve in the order index -> env -> file -> bundled Free key. Loading a key through the UI @@ -15,7 +15,7 @@ import { userCredentials } from '../support/helpers'; // on nodes disagreeing about the current edition right after this test flips it, which is the same // class of issue Kibana-config.cy.ts hit and skipped for the same reason. The eck-* environments // run a single Kibana node (kind-cluster/ror/base/kbn.yml: count: 1) and are unaffected. -(Cypress.env().envName === 'elk-ror' ? describe.skip : describe)('Activation key', () => { +(isMultiKibanaNodeEnv() ? describe.skip : describe)('Activation key', () => { beforeEach(() => { Login.initialization(); ActivationKeys.open(); diff --git a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts index efa1d498..f45cddbe 100644 --- a/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts +++ b/e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts @@ -1,5 +1,6 @@ import { esApiAdvancedClient } from '../support/helpers/EsApiAdvancedClient'; import { esApiClient } from '../support/helpers/EsApiClient'; +import { isMultiKibanaNodeEnv } from '../support/helpers'; import { kbnApiAdvancedClient } from '../support/helpers/KbnApiAdvancedClient'; import { Dashboard } from '../support/page-objects/Dashboard'; import { Discover } from '../support/page-objects/Discover'; @@ -33,7 +34,7 @@ describe('Readonlyrest-settings', () => { // do not share the per-session state that decides whether the tenant index gets reset from the // template. That made the test flaky there. The eck-* environments run a single Kibana node // (kind-cluster/ror/base/kbn.yml: count: 1), where every request hits the same state. - (Cypress.env().envName === 'elk-ror' ? it.skip : it)('should verify kibanaIndexTemplate functionality', () => { + (isMultiKibanaNodeEnv() ? it.skip : it)('should verify kibanaIndexTemplate functionality', () => { Settings.setReadonlyRestKbnSettings(` kibanaIndexTemplate: ".kibana_template_group" resetKibanaIndexToTemplate: true diff --git a/e2e-tests/cypress/support/helpers/RorApiClient.ts b/e2e-tests/cypress/support/helpers/RorApiClient.ts index 228f032b..431dd861 100644 --- a/e2e-tests/cypress/support/helpers/RorApiClient.ts +++ b/e2e-tests/cypress/support/helpers/RorApiClient.ts @@ -1,3 +1,5 @@ +import { isMultiKibanaNodeEnv } from './index'; + // The node that answers this POST applies the new settings before it replies. Every other node // picks them up from its own poll of the index, which runs every 5 s. Until then it still serves // the old settings, so a login on one node is not recognized by the other and bounces to /login. @@ -22,7 +24,7 @@ export class RorApiClient { // config - e.g. two specs in a row both resetting to the same default fixture. That's // the desired state, not an error; only a genuinely different failure should throw. if (response.status === 'SUCCESS') { - return Cypress.env().envName === 'elk-ror' ? cy.wait(SETTINGS_PROPAGATION_DELAY_MS) : undefined; + return isMultiKibanaNodeEnv() ? cy.wait(SETTINGS_PROPAGATION_DELAY_MS) : undefined; } if (response.message !== 'Current settings are already loaded') { throw new Error(`Failed to configure ROR index main settings: ${JSON.stringify(response)}`); diff --git a/e2e-tests/cypress/support/helpers/index.ts b/e2e-tests/cypress/support/helpers/index.ts index b1454b4f..848bdbca 100644 --- a/e2e-tests/cypress/support/helpers/index.ts +++ b/e2e-tests/cypress/support/helpers/index.ts @@ -1,5 +1,6 @@ import * as semver from 'semver'; import { BasicCredentials } from './KbnApiClient'; +import { EnvName } from '../types'; export const getKibanaVersion = () => { const kibanaVersion: string = Cypress.env('kibanaVersion'); @@ -13,6 +14,10 @@ export const getKibanaVersion = () => { return kibanaVersion; }; +export function isMultiKibanaNodeEnv(): boolean { + return Cypress.env().envName === EnvName.ELK_ROR; +} + export function requiredBaseUrl(): string { const baseUrl = Cypress.config('baseUrl'); if (!baseUrl) throw new Error('Cypress baseUrl is not configured'); From e8ef837b0789d971f3eedb563b6fdae81125bbb7 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Tue, 22 Sep 2026 09:03:14 +0200 Subject: [PATCH 34/40] Remove unused hasImpersonateButton parameter from Impersonate page object --- e2e-tests/cypress/support/page-objects/Impersonate.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/Impersonate.ts b/e2e-tests/cypress/support/page-objects/Impersonate.ts index 7a71c336..1f9f3be0 100644 --- a/e2e-tests/cypress/support/page-objects/Impersonate.ts +++ b/e2e-tests/cypress/support/page-objects/Impersonate.ts @@ -47,13 +47,11 @@ export class Impersonate { cy.get('@service').contains(name); } - static assertUser(index: number, username: string, groups?: string[], hasImpersonateButton = true) { + static assertUser(index: number, username: string, groups?: string[]) { cy.log('Check user'); Impersonate.getServiceByIndex(index).as('service'); cy.get('@service').findAllByRole('rowgroup').eq(1).findByText(username).closest('tr').as('userRow'); - cy.get('@userRow') - .contains('Impersonate') - .should(hasImpersonateButton ? 'exist' : 'not.exist'); + cy.get('@userRow').contains('Impersonate').should('exist'); if (!groups) { return; From b32066693c8febcda63b213e1c29529990394ab6 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 23 Sep 2026 12:34:21 +0200 Subject: [PATCH 35/40] Remove unused parameters from KbnApiClient.loadSampleData --- e2e-tests/cypress/support/helpers/KbnApiClient.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e-tests/cypress/support/helpers/KbnApiClient.ts b/e2e-tests/cypress/support/helpers/KbnApiClient.ts index d98032df..27b33457 100644 --- a/e2e-tests/cypress/support/helpers/KbnApiClient.ts +++ b/e2e-tests/cypress/support/helpers/KbnApiClient.ts @@ -74,9 +74,7 @@ export class KbnApiClient { public loadSampleData( sampleDatasetName: string, credentials: string, - group?: string, - timeout = 90000, - interval = 5000 + group?: string ): Cypress.Chainable<{ statusCode?: number; elasticsearchIndicesCreated?: Record }> { return recurse( () => @@ -89,8 +87,8 @@ export class KbnApiClient { }), response => response?.elasticsearchIndicesCreated !== undefined, { - timeout, - delay: interval, + timeout: 90000, + delay: 5000, log: response => cy.log(`Load sample data "${sampleDatasetName}" response: ${JSON.stringify(response)}`), error: `Timed out loading sample data "${sampleDatasetName}"` } From 08f926b1bd52cbb71e7ded29e70399de17203183 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Wed, 23 Sep 2026 12:38:03 +0200 Subject: [PATCH 36/40] Update loadSampleData comment in KbnApiClient --- e2e-tests/cypress/support/helpers/KbnApiClient.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/e2e-tests/cypress/support/helpers/KbnApiClient.ts b/e2e-tests/cypress/support/helpers/KbnApiClient.ts index 27b33457..85c867dc 100644 --- a/e2e-tests/cypress/support/helpers/KbnApiClient.ts +++ b/e2e-tests/cypress/support/helpers/KbnApiClient.ts @@ -66,10 +66,12 @@ export class KbnApiClient { /** * Kibana's sample-data installer deletes the previous index and recreates it in one request; - * those two steps occasionally race each other (resource_already_exists_exception -> 500), and - * the bulk-insert that follows a successful create is too slow for the shared httpCall timeout. - * Give this call more room per attempt and retry with backoff so the race gets to resolve - * itself instead of failing the test. + * those two steps occasionally race each other (resource_already_exists_exception -> 500). + * That failure resolves rather than rejecting (failOnStatusCode: false), so the fixed-interval + * retry below gets a chance to let the race settle. A cy.task timeout does not: it rejects the + * chain, cypress-recurse does not retry on rejection, and the loop ends immediately. So + * timeoutMs must, on its own, outlast the bulk-insert that follows a successful create - the + * retry budget below cannot cover a run where that insert is slower than timeoutMs. */ public loadSampleData( sampleDatasetName: string, From 723ec5073f5a63b9a992fa8f335da09c54424c33 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Thu, 24 Sep 2026 12:10:44 +0200 Subject: [PATCH 37/40] Remove unused Reporting pagePath getter --- e2e-tests/cypress/support/page-objects/Reporting.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/Reporting.ts b/e2e-tests/cypress/support/page-objects/Reporting.ts index 31088b96..d2e97aad 100644 --- a/e2e-tests/cypress/support/page-objects/Reporting.ts +++ b/e2e-tests/cypress/support/page-objects/Reporting.ts @@ -9,14 +9,6 @@ import { KibanaToast } from './KibanaToast'; type OpenBy = 'rorMenu' | 'kibanaNavigation'; export class Reporting { - // Kibana settles the reporting management page on an /exports child route on 8.19.x and again - // from 9.5.0. The 9.0-9.4 line serves the bare /reporting path. - static get pagePath() { - return semver.satisfies(getKibanaVersion(), '>=8.19.0 <9.0.0 || >=9.5.0') - ? '/app/management/insightsAndAlerting/reporting/exports' - : '/app/management/insightsAndAlerting/reporting'; - } - static noReportsCreatedCheck(openBy: OpenBy) { cy.log('noReportsCreatedCheck'); this.openReportingPage(openBy); From 960901ca3187aa0c879ca66068e833132d361350 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Thu, 24 Sep 2026 13:27:18 +0200 Subject: [PATCH 38/40] Fix save button click retry logic in Settings page object --- e2e-tests/cypress/support/page-objects/Settings.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/cypress/support/page-objects/Settings.ts b/e2e-tests/cypress/support/page-objects/Settings.ts index b6e3fbcd..dfd42721 100644 --- a/e2e-tests/cypress/support/page-objects/Settings.ts +++ b/e2e-tests/cypress/support/page-objects/Settings.ts @@ -67,8 +67,8 @@ export class Settings { // Once the modal is opening, an overlay mask covers the Save button; clicking again // would report the button as hidden instead of giving the modal time to finish // mounting. Only re-click while nothing has opened yet. - const hasOverlay = ($body as JQuery).hasClass('euiBody-hasOverlayMask'); - if (!hasOverlay) { + const modalOpening = ($body as JQuery).find(':contains("Save anyway")').length > 0; + if (!modalOpening) { Settings.clickSaveButton(); } }) From a72b9553cd142e2c01c41cf480b9286c136ad7d1 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Thu, 24 Sep 2026 13:29:07 +0200 Subject: [PATCH 39/40] Update es-np.yml --- environments/eck-ror/kind-cluster/ror/base/es-np.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environments/eck-ror/kind-cluster/ror/base/es-np.yml b/environments/eck-ror/kind-cluster/ror/base/es-np.yml index 75920ace..638ee427 100644 --- a/environments/eck-ror/kind-cluster/ror/base/es-np.yml +++ b/environments/eck-ror/kind-cluster/ror/base/es-np.yml @@ -4,6 +4,9 @@ metadata: name: eck-ror-es-np spec: type: NodePort + # The cluster has one Elasticsearch node. Without this, each readiness failure removes its + # endpoint, and each open request ends in ECONNRESET instead of a slow response. + publishNotReadyAddresses: true ports: - port: 9200 name: esport From 9e46466d8b3db0555b233bc10bf4267305607429 Mon Sep 17 00:00:00 2001 From: Dawid Poliszak Date: Thu, 24 Sep 2026 13:30:24 +0200 Subject: [PATCH 40/40] Update index.ts --- e2e-tests/cypress/plugins/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/e2e-tests/cypress/plugins/index.ts b/e2e-tests/cypress/plugins/index.ts index 8b2769c2..904f1794 100644 --- a/e2e-tests/cypress/plugins/index.ts +++ b/e2e-tests/cypress/plugins/index.ts @@ -114,9 +114,7 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) console.error('HTTP Request failed:', { error: (error as Error).message, url, - method, - headers, - body + method }); throw error; } @@ -162,8 +160,7 @@ module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) console.error('HTTP Request failed:', { error: (error as Error).message, url, - headers, - file + method }); throw error; }