[RORDEV-2219] Await SessionManagerProvider readiness before PKP/PEP start accepting requests

e2e-tests/cypress/e2e/Activation-keys.cy.ts

@@ -1,6 +1,6 @@
 import { Login } from '../support/page-objects/Login';
 import { ActivationKeys } from '../support/page-objects/ActivationKeys';
-import { userCredentials } from '../support/helpers';
+import { isMultiKibanaNodeEnv, userCredentials } from '../support/helpers';
 
 /**
  * Keys resolve in the order index -> env -> file -> bundled Free key. Loading a key through the UI
@@ -15,7 +15,7 @@ import { userCredentials } from '../support/helpers';
 // on nodes disagreeing about the current edition right after this test flips it, which is the same
 // class of issue Kibana-config.cy.ts hit and skipped for the same reason. The eck-* environments
 // run a single Kibana node (kind-cluster/ror/base/kbn.yml: count: 1) and are unaffected.
-(Cypress.env().envName === 'elk-ror' ? describe.skip : describe)('Activation key', () => {
+(isMultiKibanaNodeEnv() ? describe.skip : describe)('Activation key', () => {
   beforeEach(() => {
     Login.initialization();
     ActivationKeys.open();

e2e-tests/cypress/e2e/Allowed-api-paths.cy.ts

@@ -4,7 +4,6 @@
  * Written by Beshu Limited <info@readonlyrest.com> in London, UK
  */
 
-import { Login } from '../support/page-objects/Login';
 import { rorApiClient } from '../support/helpers/RorApiClient';
 
 // api_only users — allowed_api_paths enforcement is active

e2e-tests/cypress/e2e/Hide_apps.cy.ts

@@ -12,7 +12,7 @@ import { Home } from '../support/page-objects/Home';
 
 describe('hidden apps', () => {
   afterEach(() => {
-    Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml');
+    Settings.restoreDefaultSettingsData();
   });
 
   context('Stack Management navigation', () => {

e2e-tests/cypress/e2e/Impersonate.cy.ts

@@ -55,32 +55,32 @@ describe('impersonate', () => {
       Impersonate.assertServiceName(0, 'LDAP 1');
       Impersonate.assertServiceType(0, 'ldap');
       Impersonate.assertServiceColumns(0, ['Username', 'Groups']);
-      Impersonate.assertUser(0, 0, 'JohnDoe', ['group3']);
-      Impersonate.assertUser(0, 1, 'RobertSmith', ['group3']);
+      Impersonate.assertUser(0, 'JohnDoe', ['group3']);
+      Impersonate.assertUser(0, 'RobertSmith', ['group3']);
     };
 
     const assertAuthnService = () => {
       cy.log('should assert authn service');
       Impersonate.assertServiceName(1, 'ACME1 External Authorization Service');
       Impersonate.assertServiceType(1, 'authn');
       Impersonate.assertServiceColumns(1, ['Username']);
-      Impersonate.assertUser(1, 0, 'JaneDoe');
+      Impersonate.assertUser(1, 'JaneDoe');
     };
 
     const assertAuthzService = () => {
       cy.log('should assert authz service');
       Impersonate.assertServiceName(2, 'ACME2 External Authentication Service');
       Impersonate.assertServiceType(2, 'authz');
       Impersonate.assertServiceColumns(2, ['Username', 'Groups']);
-      Impersonate.assertUser(2, 0, 'JaimeRhynes', ['Customer']);
+      Impersonate.assertUser(2, 'JaimeRhynes', ['Customer']);
     };
 
     const assertLocalUser = () => {
       cy.log('should assert local user');
       Impersonate.assertServiceName(3, 'Local users');
       Impersonate.assertServiceType(3, 'local');
       Impersonate.assertServiceColumns(3, ['Username']);
-      Impersonate.assertUser(3, 0, 'kibana');
+      Impersonate.assertUser(3, 'kibana');
     };
 
     createLdapUsers();
@@ -98,11 +98,11 @@ describe('impersonate', () => {
     Impersonate.openEditAuthMockDialog(2);
     Impersonate.addEditMockUser('kibana', ['group3']);
     Impersonate.saveEditMockUsers();
-    Impersonate.assertUser(2, 1, 'kibana', ['group3']);
+    Impersonate.assertUser(2, 'kibana', ['group3']);
 
     cy.log('should impersonate localUser');
     Impersonate.open();
-    Impersonate.impersonateUserFromTheList(3, 2, 'new_user');
+    Impersonate.impersonateUserFromTheList(3, 3, 'new_user');
     Impersonate.finishImpersonation();
     Impersonate.verifyFinishedImpersonation();
 

e2e-tests/cypress/e2e/Readonlyrest-settings.cy.ts

@@ -1,5 +1,6 @@
 import { esApiAdvancedClient } from '../support/helpers/EsApiAdvancedClient';
 import { esApiClient } from '../support/helpers/EsApiClient';
+import { isMultiKibanaNodeEnv } from '../support/helpers';
 import { kbnApiAdvancedClient } from '../support/helpers/KbnApiAdvancedClient';
 import { Dashboard } from '../support/page-objects/Dashboard';
 import { Discover } from '../support/page-objects/Discover';
@@ -29,14 +30,11 @@ describe('Readonlyrest-settings', () => {
   });
 
   // The docker env (elk-ror) runs 2 kbn-ror replicas behind kbn-proxy's round robin (see
-  // base.docker-compose.yml). resetKibanaIndexToTemplate is only applied once, at tenant-index
-  // creation time (TenantIndexBasedOnTemplateApplier, called from abstractIndexCreator.ts) - unlike
-  // the CSS/JS/middleware injections elsewhere in this spec, which re-evaluate on every request and
-  // so self-correct if an early request lands on a stale replica. If the one request that creates
-  // .kibana_admins_group lands on a replica that has not yet picked up the settings POSTed above,
-  // the reset never happens and nothing later can retrigger it. The eck-* environments run a single
-  // Kibana node (kind-cluster/ror/base/kbn.yml: count: 1) and are unaffected.
-  (Cypress.env().envName === 'elk-ror' ? it.skip : it)('should verify kibanaIndexTemplate functionality', () => {
+  // base.docker-compose.yml), so a request of this test can land on either of them, and the two
+  // do not share the per-session state that decides whether the tenant index gets reset from the
+  // template. That made the test flaky there. The eck-* environments run a single Kibana node
+  // (kind-cluster/ror/base/kbn.yml: count: 1), where every request hits the same state.
+  (isMultiKibanaNodeEnv() ? it.skip : it)('should verify kibanaIndexTemplate functionality', () => {
     Settings.setReadonlyRestKbnSettings(`
   kibanaIndexTemplate: ".kibana_template_group"
   resetKibanaIndexToTemplate: true
@@ -63,7 +61,15 @@ describe('Readonlyrest-settings', () => {
       currentGroupHeader: 'admins_group'
     });
 
-    cy.reload();
+    // A reload is not enough to retrigger the reset: kbn-ror only re-runs the tenant index
+    // creation (and with it the reindex from the template) once per session id per index, and
+    // remembers that for 2 minutes in the memory of the node that served the request. A login
+    // is not subject to that - it always runs the creation for the session it opens - so clear
+    // the cookies and sign in again to get a session id the node has not seen yet.
+    cy.clearCookies();
+    cy.clearLocalStorage();
+    Login.initialization();
+
     Dashboard.openDashboard();
     Dashboard.verifyDashboardNotExist('Look at my dashboard');
   });
@@ -87,7 +93,7 @@ describe('Readonlyrest-settings', () => {
 
     cy.reload();
 
-    cy.get('h1').shouldHaveStyle('color', 'rgb(0,128,0)');
+    cy.get('h1', { timeout: 30000 }).shouldHaveStyle('color', 'rgb(0,128,0)');
   });
 
   it('should verify custom Kibana JS', () => {
@@ -104,7 +110,7 @@ describe('Readonlyrest-settings', () => {
 
     cy.reload();
 
-    cy.get('[data-testid="metadata-alert-message"]')
+    cy.get('[data-testid="metadata-alert-message"]', { timeout: 30000 })
       .should('exist')
       .then($el => {
         cy.log(`Alert message: ${$el.text()}`);
@@ -139,7 +145,7 @@ describe('Readonlyrest-settings', () => {
 
     cy.reload();
 
-    cy.get('[data-testid="metadata-enriched-data"]')
+    cy.get('[data-testid="metadata-enriched-data"]', { timeout: 30000 })
       .should('exist')
       .then($el => {
         cy.log(`Entiched data: ${$el.text()}`);

e2e-tests/cypress/e2e/Reporting-index.cy.ts

@@ -21,7 +21,7 @@ describe('Reporting index', () => {
     if (semver.gte(getKibanaVersion(), '8.0.0')) {
       kbnApiAdvancedClient.deleteDataViews(admin, 'infosec_group');
     }
-    Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml');
+    Settings.restoreDefaultSettingsData();
   });
 
   it('should correctly match index pattern when audit index_template contains .reporting', () => {

e2e-tests/cypress/e2e/Sanity-check-ro-kibana-access.cy.ts

@@ -5,7 +5,7 @@ import { userCredentials } from '../support/helpers';
 
 describe('sanity check ro kibana access', () => {
   afterEach(() => {
-    Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml');
+    Settings.restoreDefaultSettingsData();
     kbnApiClient.deleteSampleData('ecommerce', userCredentials, 'template_group');
   });
 

e2e-tests/cypress/e2e/Sanity-check-ro_strict-kibana-access.cy.ts

@@ -5,7 +5,7 @@ import { userCredentials } from '../support/helpers';
 
 describe('sanity check ro_strict kibana access', () => {
   afterEach(() => {
-    Settings.setSettingsData('defaultReadonlyRestEsAndKbnSettings.yaml');
+    Settings.restoreDefaultSettingsData();
     kbnApiClient.deleteSampleData('ecommerce', userCredentials, 'template_group');
   });
 

e2e-tests/cypress/e2e/Tenancy.cy.ts

@@ -41,17 +41,30 @@ describe('Tenancy', () => {
   describe('should run tests when tenancy switched in a different tab', () => {
     const urlWithInfosecTenancyId = `/s/default/app/discover?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedInfosecGroup}`;
 
+    // Retries re-run the whole test body, including this callback, so a failed attempt opens
+    // another popup on top of whatever the previous attempt left behind. Left uncollected, they
+    // pile up (each one polling Kibana unauthenticated) until a later attempt's login hangs
+    // waiting for a Kibana request queue the orphaned tabs are still contending for.
+    let openedWindow: Window | null = null;
+
     const openAnotherTabs = () => {
       cy.window().then(win => {
-        win.open(urlWithInfosecTenancyId, '_blank');
+        openedWindow = win.open(urlWithInfosecTenancyId, '_blank');
       });
+      cy.log('let the second tab reach its login page before the main tab starts logging in');
+      cy.wait(3000);
     };
 
     beforeEach(() => {
       cy.clearCookies();
       cy.clearLocalStorage();
     });
 
+    afterEach(() => {
+      openedWindow?.close();
+      openedWindow = null;
+    });
+
     // eslint-disable-next-line no-use-before-define
     runTests({ callbackBeforeLogin: openAnotherTabs });
   });

e2e-tests/cypress/fixtures/allowedApiPathsSettings.yaml

@@ -5,10 +5,26 @@ readonlyrest:
         keystore_pass: readonlyrest
         key_pass: readonlyrest
     access_control_rules:
+        # <-- related to ECK environment -->
+        - name: 'Kibana service account - token'
+          verbosity: error
+          token_authentication:
+              token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}'
+              username: service_account
+
         - name: 'Kibana service account - user/pass'
           verbosity: error
           auth_key: kibana:kibana
 
+        - name: 'PROBE'
+          verbosity: error
+          auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}'
+
+        - name: 'ELASTIC-INTERNAL'
+          verbosity: error
+          auth_key: 'elastic-internal:${INTERNAL_USR_PASS}'
+        # <!-- related to ECK environment -->
+
         # api_only user restricted to a single exact Kibana API path.
         - name: API_ONLY_RESTRICTED
           auth_key: api_only_restricted_user:dev
@@ -60,3 +76,4 @@ readonlyrest:
                     http_path: '^/api/ror/user/tenants$'
 readonlyrest_kbn:
     cookiePass: '12312313123213123213123adadasdasdasd'
+    store_sessions_in_index: true

e2e-tests/cypress/fixtures/defaultReadonlyRestEsAndKbnSettings.yaml

@@ -1,7 +1,7 @@
 helpers:
     ckr: &common-kibana-rules
         access: rw
-        hide_apps: ['Enterprise Search|Overview', 'Observability']
+        hide_apps: ['Enterprise Search|Overview']
         index: '.kibana_@{acl:current_group}'
 
     ag: &all-groups
@@ -22,10 +22,30 @@ readonlyrest:
               index_template: "'readonlyrest_audit_'yyyy-MM-dd"
 
     access_control_rules:
+        # <-- related to ECK environment -->
+        - name: 'Kibana service account - token'
+          verbosity: error
+          token_authentication:
+              token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}'
+              username: service_account
+
         - name: 'Kibana service account - user/pass'
           verbosity: error
           auth_key: kibana:kibana
 
+        - name: 'PROBE'
+          verbosity: error
+          auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}'
+
+        - name: 'ELASTIC-INTERNAL'
+          verbosity: error
+          auth_key: 'elastic-internal:${INTERNAL_USR_PASS}'
+        # <!-- related to ECK environment -->
+
+        - name: 'APM server - system account'
+          verbosity: error
+          auth_key: apm:test
+
         - name: JWT_AUTH
           jwt_auth:
               name: 'jwt1'
@@ -52,7 +72,6 @@ readonlyrest:
           kibana:
               <<: *common-kibana-rules
               access: admin
-              hide_apps: ['Enterprise Search|Overview', 'Management']
               metadata:
                   alert_message: 'Dear @{acl:user}'
         - name: infosec

e2e-tests/cypress/fixtures/defaultReadonlyRestEsSettings.yaml

@@ -22,10 +22,26 @@ readonlyrest:
               index_template: "'readonlyrest_audit_'yyyy-MM-dd"
 
     access_control_rules:
+        # <-- related to ECK environment -->
+        - name: 'Kibana service account - token'
+          verbosity: error
+          token_authentication:
+              token: 'Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}'
+              username: service_account
+
         - name: 'Kibana service account - user/pass'
           verbosity: error
           auth_key: kibana:kibana
 
+        - name: 'PROBE'
+          verbosity: error
+          auth_key: 'elastic-internal-probe:${INTERNAL_PROBE_PASS}'
+
+        - name: 'ELASTIC-INTERNAL'
+          verbosity: error
+          auth_key: 'elastic-internal:${INTERNAL_USR_PASS}'
+        # <!-- related to ECK environment -->
+
         - name: JWT_AUTH
           jwt_auth:
               name: 'jwt1'

e2e-tests/cypress/fixtures/hiddenAllAppsSettings.yaml

@@ -1,10 +1,26 @@
 readonlyrest:
     access_control_rules:
 
+        # <-- related to ECK environment -->
+        - name: "Kibana service account - token"
+          verbosity: error
+          token_authentication:
+              token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}"
+              username: service_account
+
         - name: "Kibana service account - user/pass"
           verbosity: error
           auth_key: kibana:kibana
 
+        - name: "PROBE"
+          verbosity: error
+          auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}"
+
+        - name: "ELASTIC-INTERNAL"
+          verbosity: error
+          auth_key: "elastic-internal:${INTERNAL_USR_PASS}"
+        # <!-- related to ECK environment -->
+
         - name: ADMIN_GRP
           groups: [admins_group]
           kibana_hide_apps: [ "Enterprise Search", "Observability", "Elasticsearch", "Analytics", "Management"]
@@ -18,3 +34,4 @@ readonlyrest:
                   name: administrators
 readonlyrest_kbn:
     cookiePass: '12312313123213123213123adadasdasdasd'
+    store_sessions_in_index: true

e2e-tests/cypress/fixtures/hiddenHomePageSettings.yaml

@@ -1,10 +1,26 @@
 readonlyrest:
     access_control_rules:
 
+        # <-- related to ECK environment -->
+        - name: "Kibana service account - token"
+          verbosity: error
+          token_authentication:
+              token: "Bearer ${KIBANA_SERVICE_ACCOUNT_TOKEN}"
+              username: service_account
+
         - name: "Kibana service account - user/pass"
           verbosity: error
           auth_key: kibana:kibana
 
+        - name: "PROBE"
+          verbosity: error
+          auth_key: "elastic-internal-probe:${INTERNAL_PROBE_PASS}"
+
+        - name: "ELASTIC-INTERNAL"
+          verbosity: error
+          auth_key: "elastic-internal:${INTERNAL_USR_PASS}"
+        # <!-- related to ECK environment -->
+
         - name: ADMIN_GRP
           groups: [admins_group]
           kibana_hide_apps: [ "Home"]
@@ -18,3 +34,4 @@ readonlyrest:
                   name: administrators
 readonlyrest_kbn:
     cookiePass: '12312313123213123213123adadasdasdasd'
+    store_sessions_in_index: true