diff --git a/.github/cleanup-disk-space/action.yml b/.github/cleanup-disk-space/action.yml index 14c0886d..1e13b02f 100644 --- a/.github/cleanup-disk-space/action.yml +++ b/.github/cleanup-disk-space/action.yml @@ -1,16 +1,52 @@ name: 'Cleanup disk space' -description: 'Cleanup runner disk space' +description: 'Frees runner disk space, but only when the job needs more than the runner already has free.' + +inputs: + required_gb: + description: > + Free space on / the job needs, in GB. The cleanup is skipped when the runner already has + this much: deleting the preinstalled toolchains costs about 80 seconds. + required: false + default: '25' runs: using: 'composite' steps: - name: Cleanup disk space shell: bash + env: + REQUIRED_GB: ${{ inputs.required_gb }} run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo rm -rf /usr/local/share/boost /usr/local/share/boost-build + set -euo pipefail + + FREE_GB=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') + echo "Free space on /: ${FREE_GB}G. This job asks for ${REQUIRED_GB}G." + + if [ "$FREE_GB" -ge "$REQUIRED_GB" ]; then + echo "Enough free space already โ€” skipping the cleanup." + exit 0 + fi + + # Independent trees, deleted in parallel. Each pid is waited on by name: a bare `wait` + # returns 0 whatever the jobs did, and a failed deletion must fail this step. + pids=() + sudo rm -rf /usr/share/dotnet & pids+=("$!") + sudo rm -rf /usr/local/lib/android & pids+=("$!") + sudo rm -rf /opt/ghc & pids+=("$!") + sudo rm -rf /usr/local/share/boost /usr/local/share/boost-build & pids+=("$!") + + failed=0 + for pid in "${pids[@]}"; do + wait "$pid" || failed=1 + done + if [ "$failed" -ne 0 ]; then + echo "At least one cleanup deletion failed. Disk space below:" + df -h / + exit 1 + fi + sudo apt-get clean sudo rm -rf /var/lib/apt/lists/* - + echo "Available disk space after cleanup:" - df -h + df -h / diff --git a/.github/cypress-cache/action.yml b/.github/cypress-cache/action.yml new file mode 100644 index 00000000..964f98cf --- /dev/null +++ b/.github/cypress-cache/action.yml @@ -0,0 +1,21 @@ +name: 'Cypress cache' +description: 'Restores and saves the yarn download cache, node_modules and the Cypress binary for the e2e suite.' + +runs: + using: 'composite' + steps: + # Both caches are keyed on yarn.lock, the file that decides their content, so a stale entry + # cannot be served. The Cypress binary version is pinned by the `cypress` entry in it, so one + # key covers the binary too. + - name: Cache the yarn download cache and node_modules + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: | + ~/.cache/yarn + e2e-tests/node_modules + key: e2e-yarn-${{ runner.os }}-${{ hashFiles('e2e-tests/yarn.lock') }} + - name: Cache the Cypress binary + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: ~/.cache/Cypress + key: e2e-cypress-${{ runner.os }}-${{ hashFiles('e2e-tests/yarn.lock') }} diff --git a/.github/run-e2e-tests/action.yml b/.github/run-e2e-tests/action.yml index 0ab8c404..c7f09350 100644 --- a/.github/run-e2e-tests/action.yml +++ b/.github/run-e2e-tests/action.yml @@ -35,6 +35,8 @@ inputs: runs: using: 'composite' steps: + - name: Restore the yarn and Cypress caches + uses: ./.github/cypress-cache - name: Run E2E tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 with: diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index a00c6155..26976577 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -4,37 +4,76 @@ on: workflow_dispatch: {} schedule: - cron: '0 0 * * *' - # Only the two long-lived branches. Feature branches are covered by their pull request, and - # listing them here would run everything twice for every push to an open PR. + # Feature branches are covered by their pull request. Prose cannot change what the suite does. push: branches: [master, develop] + paths-ignore: ['docs/**', '**/*.md', 'LICENSE'] + # ready_for_review: a draft runs docker only, so the ECK legs are due when the draft ends. pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: ['docs/**', '**/*.md', 'LICENSE'] + +# A second push to a pull request cancels the run in progress. Every other run is grouped by its +# run id, so those runs neither cancel nor queue behind each other. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: APPLY_RESOURCE_LIMITS: "auto" + # Newest first: a failure on the newest line is the one to see first in the run UI. + ELK_VERSIONS: "9.5.3 9.4.6 8.19.21 7.17.29" + ECK_3X: eck-3.5.0 + ECK_2X: eck-2.16.1 jobs: + # ========================================== + # PLAN + # ========================================== + # Which plugin images to test, and on which environments. prod images are the released plugins. + # dev images are built from the branch under test, or from develop on the cron. + plan: + name: "๐Ÿงฎ Plan" + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + images: ${{ steps.plan.outputs.images }} + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - name: Choose the images and the environments + id: plan + env: + EVENT: ${{ github.event_name }} + # A pull request tests the plugins of the branch it targets. + BRANCH: ${{ github.base_ref || github.ref_name }} + DRAFT: ${{ github.event.pull_request.draft == true }} + # A fork PR gets no secrets, and the dev image pre-build needs them. + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + run: | + set -euo pipefail + if [ "$EVENT" = schedule ] || [ "$EVENT" = workflow_dispatch ]; then IMAGES="prod dev"; ENVS="docker $ECK_2X $ECK_3X" + elif [ "$FORK" = true ]; then IMAGES=""; ENVS="" + elif [ "$BRANCH" = master ] && [ "$DRAFT" = true ]; then IMAGES="prod"; ENVS="docker" + elif [ "$BRANCH" = master ]; then IMAGES="prod"; ENVS="docker $ECK_3X" + elif [ "$DRAFT" = true ]; then IMAGES="dev"; ENVS="docker" + else IMAGES="dev"; ENVS="docker $ECK_3X" + fi + { + echo "images=$IMAGES" + echo "matrix=$(jq -cn --arg v "$ELK_VERSIONS" --arg e "$ENVS" '{version: ($v | split(" ")), env: ($e | split(" "))}')" + } >> "$GITHUB_OUTPUT" + # ========================================== # E2E TESTS - RELEASED (PROD) PLUGIN IMAGES # ========================================== - # Runs against the RELEASED plugin images (`--mode prod`, ror-latest). The signal is "the shipped - # plugins still pass the suite": master itself, PRs targeting it, and the nightly schedule. - # - # Keep the matrix in sync with ELK_VERSIONS in prepare-dev-images. prod-e2e-tests: name: "๐Ÿ”ฌ E2E Tests (released plugins)" - if: > - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref == 'refs/heads/master') || - (github.event_name == 'pull_request' && github.base_ref == 'master' && github.event.pull_request.head.repo.fork == false) + needs: plan + if: contains(needs.plan.outputs.images, 'prod') runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - version: ["9.5.3", "9.4.6", "8.19.21", "7.17.29"] - env: [docker, eck-2.16.1, eck-3.5.0] + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -78,75 +117,17 @@ jobs: path_prefix: ${{ vars.ROR_S3_PATH_E2E_REPORTS }} # ========================================== - # BOOTSTRAP TESTS - # ========================================== - # Same condition as prod-e2e-tests, which this gates on โ€” kept identical so the two cannot drift. - # Bootstrap uses released images (no --mode), as it did before. - master-bootstrap-tests: - name: "๐Ÿš€ Bootstrap Tests" - if: > - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && github.ref == 'refs/heads/master') || - (github.event_name == 'pull_request' && github.base_ref == 'master' && github.event.pull_request.head.repo.fork == false) - needs: prod-e2e-tests - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # 7.10.0 was temporarily removed due to this issue: https://github.com/sscarduzio/elasticsearch-readonlyrest-plugin/pull/1362. Add it again when the fix is released. - version: ["9.5.0", "9.4.0", "9.3.0", "9.2.0", "9.1.0", "9.0.0", "8.19.0", "8.18.0", "8.17.0", "8.16.0", "8.15.0", "8.14.0", "8.13.0", "8.12.0", "8.11.0", "8.10.1", "8.9.0", "8.8.0", "8.7.0", "8.6.0", "8.5.0", "8.4.0", "8.3.0", "8.2.0", "8.1.0", "8.0.0", "7.17.0", "7.16.0", "7.15.0", "7.14.0", "7.13.0", "7.12.0", "7.11.2", "7.9.0"] - env: [docker, eck-3.5.0] - steps: - - name: Checkout code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - - name: Configure the Docker Hub mirror - run: source .github/scripts/docker-hub-mirror.sh - - name: Clean up disk space - uses: ./.github/cleanup-disk-space - - name: Start Docker memory monitor - uses: ./.github/docker-memory-monitor - with: - action: start - # Retry mechanism to handle transient infrastructure issues: - # - npm error 429 Too Many Requests from registry.npmjs.org - # - Docker image pull failures (e.g., beshultd/kibana-readonlyrest:*-ror-latest not found) - # These errors are typically temporary and resolve with a simple retry. - - name: Run bootstrap tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_on: any - command: | - ./runner.sh --run bootstrap --env ${{ matrix.env }} --elk ${{ matrix.version }} - env: - ROR_ACTIVATION_KEY: ${{ secrets.ROR_ENT_ACTIVATION_TOKEN }} - - name: Stop Docker memory monitor - if: always() - uses: ./.github/docker-memory-monitor - with: - action: stop - - # ========================================== - # DEV IMAGE PREPARATION - DEVELOP AND NON-MASTER PRs + # DEV IMAGE PREPARATION # ========================================== - # Builds branch-matched dev images of both plugins for dev-e2e-tests, tagged per run. One dispatch - # per plugin covers the whole matrix, since both pre-build workflows accept a version list. - # `target_branch` is passed verbatim; both fall back to `develop` if the branch is not there. - # Fork PRs are excluded: the dispatch needs secrets GitHub does not expose to them. + # Builds dev images of both plugins for every version, tagged per run. prepare-dev-images: name: "๐Ÿ—๏ธ Prepare dev images" - if: > - (github.event_name == 'push' && github.ref == 'refs/heads/develop') || - (github.event_name == 'pull_request' && github.base_ref != 'master' && github.event.pull_request.head.repo.fork == false) + needs: plan + if: contains(needs.plan.outputs.images, 'dev') runs-on: ubuntu-latest timeout-minutes: 90 outputs: run_tag: ${{ steps.prepare.outputs.run_tag }} - versions: ${{ steps.prepare.outputs.versions }} steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -155,12 +136,10 @@ jobs: - name: Dispatch and await ROR plugin pre-builds id: prepare env: - # Mirrors the prod-e2e-tests matrix โ€” keep both in sync. - ELK_VERSIONS: "9.5.3 9.4.6 8.19.21 7.17.29" - # On a PR, head_ref is the source branch โ€” the plugins must be built from the PR head, not - # from the merge ref github.ref points at. On a push, head_ref is empty and ref_name is - # the branch that was pushed (develop). - TARGET_BRANCH: ${{ github.head_ref || github.ref_name }} + # The cron builds develop. On a PR, head_ref is the source branch: the plugins must be + # built from the PR head, not from the merge ref github.ref points at. On a push or a + # manual dispatch, head_ref is empty and ref_name is the branch. + TARGET_BRANCH: ${{ github.event_name == 'schedule' && 'develop' || github.head_ref || github.ref_name }} ROR_GH_TOKEN: ${{ secrets.ROR_GH_TOKEN }} run: | @@ -169,10 +148,7 @@ jobs: # Unique per attempt: a re-run must not silently reuse the previous attempt's images. RUN_TAG="run-${{ github.run_id }}-${{ github.run_attempt }}" - { - echo "run_tag=$RUN_TAG" - echo "versions=$(printf '%s\n' $ELK_VERSIONS | jq -Rcn '[inputs]')" - } >> "$GITHUB_OUTPUT" + echo "run_tag=$RUN_TAG" >> "$GITHUB_OUTPUT" dispatch_prebuild_images "$ELK_VERSIONS" "$TARGET_BRANCH" "$RUN_TAG" wait_for_prebuild_images "$ELK_VERSIONS" "$RUN_TAG" @@ -180,18 +156,14 @@ jobs: # ========================================== # E2E TESTS - PRE-BUILD (DEV) PLUGIN IMAGES # ========================================== - # Every push to develop and every non-fork pull request that does not target master, against the - # per-run, branch-matched dev images produced by prepare-dev-images. Skipped automatically when - # that job is skipped. + # Skipped when prepare-dev-images is skipped. dev-e2e-tests: name: "๐Ÿงช E2E Tests (pre-build plugins)" - needs: prepare-dev-images + needs: [plan, prepare-dev-images] runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - version: ${{ fromJSON(needs.prepare-dev-images.outputs.versions) }} - env: [docker, eck-2.16.1, eck-3.5.0] + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} env: ROR_IMAGE_TAG: ${{ needs.prepare-dev-images.outputs.run_tag }} steps: diff --git a/.github/workflows/bootstrap-tests.yml b/.github/workflows/bootstrap-tests.yml new file mode 100644 index 00000000..f5125361 --- /dev/null +++ b/.github/workflows/bootstrap-tests.yml @@ -0,0 +1,190 @@ +name: Bootstrap tests + +# Boots each released ELK version against the plugin images and stops. `--run bootstrap` runs no +# tests: the only assertion is that the stack came up. What it can catch is "a plugin image does +# not start on this old version". +# +# trigger images +# Sunday cron prod, the released plugins +# Wednesday cron dev, built from develop +# manual dispatch both +# +# Only docker. This job starts a stack and does not exercise the Kibana proxy, which is where +# docker and ECK differ. +permissions: + contents: read + +on: + workflow_dispatch: {} + schedule: + - cron: '0 1 * * 0' + - cron: '0 1 * * 3' + +env: + APPLY_RESOURCE_LIMITS: "auto" + # 7.10.0 is out of the list until the fix in + # https://github.com/sscarduzio/elasticsearch-readonlyrest-plugin/pull/1362 is released. + ELK_VERSIONS: "9.5.0 9.4.0 9.3.0 9.2.0 9.1.0 9.0.0 8.19.0 8.18.0 8.17.0 8.16.0 8.15.0 8.14.0 8.13.0 8.12.0 8.11.0 8.10.1 8.9.0 8.8.0 8.7.0 8.6.0 8.5.0 8.4.0 8.3.0 8.2.0 8.1.0 8.0.0 7.17.0 7.16.0 7.15.0 7.14.0 7.13.0 7.12.0 7.11.2 7.9.0" + +jobs: + # ========================================== + # PLAN + # ========================================== + plan: + name: "๐Ÿงฎ Plan" + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + images: ${{ steps.plan.outputs.images }} + versions: ${{ steps.plan.outputs.versions }} + steps: + - name: Choose the images and list the versions + id: plan + env: + EVENT: ${{ github.event_name }} + CRON: ${{ github.event.schedule }} + run: | + set -euo pipefail + case "$EVENT:$CRON" in + workflow_dispatch:*) IMAGES="prod dev" ;; + "schedule:0 1 * * 0") IMAGES=prod ;; + "schedule:0 1 * * 3") IMAGES=dev ;; + *) echo "No plan for $EVENT $CRON" >&2; exit 1 ;; + esac + { + echo "images=$IMAGES" + echo "versions=$(jq -cn --arg v "$ELK_VERSIONS" '$v | split(" ")')" + } >> "$GITHUB_OUTPUT" + + # ========================================== + # BOOTSTRAP TESTS - RELEASED (PROD) PLUGIN IMAGES + # ========================================== + prod-bootstrap-tests: + name: "๐Ÿš€ Bootstrap Tests (released plugins)" + needs: plan + if: contains(needs.plan.outputs.images, 'prod') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.versions) }} + env: [docker] + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - name: Configure the Docker Hub mirror + run: source .github/scripts/docker-hub-mirror.sh + - name: Clean up disk space + uses: ./.github/cleanup-disk-space + with: + # A bootstrap leg pulls two images and writes no video, screenshot or Cypress cache. + required_gb: '8' + - name: Start Docker memory monitor + uses: ./.github/docker-memory-monitor + with: + action: start + # For transient infrastructure: an HTTP 429 from the npm registry, or an image pull that + # fails once. + - name: Run bootstrap tests + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 + with: + max_attempts: 3 + timeout_minutes: 30 + retry_on: any + command: | + ./runner.sh --run bootstrap --mode prod --env ${{ matrix.env }} --elk ${{ matrix.version }} + env: + ROR_ACTIVATION_KEY: ${{ secrets.ROR_ENT_ACTIVATION_TOKEN }} + - name: Stop Docker memory monitor + if: always() + uses: ./.github/docker-memory-monitor + with: + action: stop + + # ========================================== + # DEV IMAGE PREPARATION + # ========================================== + # Builds dev images of both plugins for every version in the list, tagged per run. The cron + # builds develop. A manual dispatch builds the branch it runs on; both pre-build workflows fall + # back to `develop` if the branch is not there. + prepare-dev-images: + name: "๐Ÿ—๏ธ Prepare dev images" + needs: plan + if: contains(needs.plan.outputs.images, 'dev') + runs-on: ubuntu-latest + # The pre-build runs build the whole list on one self-hosted runner, one version after the + # other. The e2e list has 4 versions; this one has 34. + timeout-minutes: 300 + outputs: + run_tag: ${{ steps.prepare.outputs.run_tag }} + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - name: Dispatch and await ROR plugin pre-builds + id: prepare + env: + TARGET_BRANCH: ${{ github.event_name == 'schedule' && 'develop' || github.ref_name }} + ROR_GH_TOKEN: ${{ secrets.ROR_GH_TOKEN }} + ROR_ES_WAIT_TIMEOUT_SECONDS: 14400 + ROR_KBN_WAIT_TIMEOUT_SECONDS: 14400 + run: | + set -euo pipefail + . ci/prebuild-images-lib.sh + + # Unique per attempt: a re-run must not silently reuse the previous attempt's images. + RUN_TAG="run-${{ github.run_id }}-${{ github.run_attempt }}" + echo "run_tag=$RUN_TAG" >> "$GITHUB_OUTPUT" + + dispatch_prebuild_images "$ELK_VERSIONS" "$TARGET_BRANCH" "$RUN_TAG" + wait_for_prebuild_images "$ELK_VERSIONS" "$RUN_TAG" + + # ========================================== + # BOOTSTRAP TESTS - PRE-BUILD (DEV) PLUGIN IMAGES + # ========================================== + # Skipped when prepare-dev-images is skipped. + dev-bootstrap-tests: + name: "๐Ÿš€ Bootstrap Tests (pre-build plugins)" + needs: [plan, prepare-dev-images] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.versions) }} + env: [docker] + env: + ROR_IMAGE_TAG: ${{ needs.prepare-dev-images.outputs.run_tag }} + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - name: Configure the Docker Hub mirror + run: source .github/scripts/docker-hub-mirror.sh + - name: Clean up disk space + uses: ./.github/cleanup-disk-space + with: + # A bootstrap leg pulls two images and writes no video, screenshot or Cypress cache. + required_gb: '8' + - name: Start Docker memory monitor + uses: ./.github/docker-memory-monitor + with: + action: start + - name: Run bootstrap tests + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 + with: + max_attempts: 3 + timeout_minutes: 30 + retry_on: any + command: | + ./runner.sh --run bootstrap --mode dev --env ${{ matrix.env }} --elk ${{ matrix.version }} --ror-es "$ROR_IMAGE_TAG" --ror-kbn "$ROR_IMAGE_TAG" + env: + ROR_ACTIVATION_KEY: ${{ secrets.ROR_ENT_ACTIVATION_TOKEN }} + - name: Stop Docker memory monitor + if: always() + uses: ./.github/docker-memory-monitor + with: + action: stop diff --git a/CLAUDE.md b/CLAUDE.md index 0f824c91..c0eae191 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ End-to-end tests for the ReadonlyREST plugins for Elasticsearch and Kibana. Cypr - `e2e-tests/cypress/e2e` โ€” the tests. `e2e-tests/cypress/support/page-objects` โ€” the page objects they drive. - `environments/` โ€” the tested stacks: `elk-ror` for Docker Compose, `eck-ror` for Kind (the ECK version is a `start.sh` flag), and `common` for the parts both share. -- `ci/`, `.github/workflows/` โ€” the pipeline. `all-e2e-tests.yml` runs the full matrix, `targeted-e2e-tests.yml` one version on demand. +- `ci/`, `.github/workflows/` โ€” the pipeline. `all-e2e-tests.yml` runs the suite, `bootstrap-tests.yml` the scheduled start-up sweep over every ELK version, `targeted-e2e-tests.yml` one version on demand. - `runner.sh` โ€” bootstraps an environment and runs the suite in one shot. - `README.md` โ€” how to run the tests: the `runner.sh` flags, the environments, the Docker-based dev env, and the Docker Hub mirror. diff --git a/docs/dev/branching.md b/docs/dev/branching.md index 6840bb09..074c8743 100644 --- a/docs/dev/branching.md +++ b/docs/dev/branching.md @@ -2,8 +2,8 @@ ## The two long-lived branches -- `master` tests the released plugins. Its CI runs the suite against the published images (`--mode prod`, `ror-latest`) on every push to `master`, on every non-fork pull request that targets `master`, and on the nightly schedule. -- `develop` tests the plugin code that is not released yet. Its CI builds branch-matched dev images of the ES and the Kibana plugin, then runs the suite against them. +- `master` tests the released plugins. Its CI runs the suite against the published images (`--mode prod`, `ror-latest`) on every push to `master`, on every non-fork pull request that targets `master`, and on the daily schedule. +- `develop` tests the plugin code that is not released yet. Its CI builds branch-matched dev images of the ES and the Kibana plugin, then runs the suite against them. The daily schedule runs this set too, with images built from `develop`. The two branches hold nearly the same tests, and the branch decides which plugin build they run against. They differ where `develop` already covers behaviour that is not released yet; the merge into `master` carries those tests over once the release is out. `.github/workflows/all-e2e-tests.yml` holds the two job sets. diff --git a/e2e-tests/cypress/e2e/Kibana-config.cy.ts b/e2e-tests/cypress/e2e/Kibana-config.cy.ts deleted file mode 100644 index 9fb25fb9..00000000 --- a/e2e-tests/cypress/e2e/Kibana-config.cy.ts +++ /dev/null @@ -1,221 +0,0 @@ -import * as semver from 'semver'; -import { rorApiInternalKbnClient } from '../support/helpers/RorApiInternalKbnClient'; -import { Login } from '../support/page-objects/Login'; -import { kbnApiAdvancedClient } from '../support/helpers/KbnApiAdvancedClient'; -import { RorMenu } from '../support/page-objects/RorMenu'; -import { getKibanaVersion, requiredBaseUrl } from '../support/helpers'; -import { Discover } from '../support/page-objects/Discover'; -import { Dashboard } from '../support/page-objects/Dashboard'; -import { Reporting } from '../support/page-objects/Reporting'; -import { SampleData } from '../support/helpers/SampleData'; -import { esApiClient } from '../support/helpers/EsApiClient'; -import { esApiAdvancedClient } from '../support/helpers/EsApiAdvancedClient'; -import { Tenancy } from '../support/page-objects/Tenancy'; - -const customKibanaIndexName = '.kibana_custom'; - -// rorApiInternalKbnClient.changeKibanaConfig rewrites kibana.yml on disk and restarts Kibana, which -// this suite relies on for every nested describe. Two environments can't support that: -// - docker env (elk-ror) runs 2 kbn-ror replicas behind kbn-proxy's round robin (see -// base.docker-compose.yml). The config reload only updates the node that handles the request; the -// ROR Kibana plugin does not propagate config changes across instances, so the other replica keeps -// serving the stale config. A single client's requests can then land on nodes disagreeing about the -// active config, which is the same class of issue Activation-keys.cy.ts hit and skipped for the -// same reason. -// - eck envs run Kibana under Kubernetes, where kibana.yml is mounted read-only from a -// ConfigMap/Secret. The rewrite always 500s with EROFS, so the custom config never applies and -// every test here fails predictably. -describe.skip('Kibana-config', () => { - after(() => { - rorApiInternalKbnClient.changeKibanaConfig('defaultKibanaConfig.yml'); - kbnApiAdvancedClient.waitForKibanaHealth(requiredBaseUrl()); - esApiAdvancedClient.deleteIndicesByPattern(customKibanaIndexName); - esApiAdvancedClient.deleteDataStreamsByPattern(customKibanaIndexName); - }); - - describe('Custom kibana config', () => { - const adminCredentials = 'admin:dev'; - const customSessionIndex = `test_index`; - - before(() => { - rorApiInternalKbnClient.changeKibanaConfig('customKibanaConfig.yml'); - kbnApiAdvancedClient.waitForKibanaHealth(requiredBaseUrl()); - }); - - afterEach(() => { - kbnApiAdvancedClient.deleteSavedObjects(adminCredentials, 'template_group'); - - // deleteSavedObjects will return 404 error because, thanks to resetKibanaIndexToTemplate: true, ROR KBN plugin will reset all data to template_group deleted above, first - kbnApiAdvancedClient.getSavedObjects(adminCredentials); - - esApiClient.deleteIndex(customSessionIndex); - }); - - it('should verify kibanaIndexTemplate functionality', () => { - cy.kbnImport({ - endpoint: 'api/saved_objects/_import?overwrite=true', - credentials: adminCredentials, - fixtureFilename: 'audit_dashboard.ndjson', - currentGroupHeader: 'template_group' - }); - - Login.initialization(); - Discover.openDataViewPage(); - Discover.verifyIndexPatternSwitchLink('readonlyrest_audit-*'); - Dashboard.openDashboard(); - Dashboard.verifyDashboardExists('ReadonlyREST Audit Dashboard'); - - // Verify that the index is reset to the template - cy.kbnImport({ - endpoint: 'api/saved_objects/_import?overwrite=true', - credentials: adminCredentials, - fixtureFilename: 'file.ndjson', - currentGroupHeader: 'admins_group' - }); - - cy.reload(); - Dashboard.verifyDashboardExists('Look at my dashboard'); - RorMenu.openRorMenu(); - - RorMenu.pressLogoutButton(); - // Logging out keeps the current location as nextUrl, so this login lands back on the - // dashboards list rather than on the home page Loader.finish expects by default. - Login.initialization({ finishUrl: '/app/dashboards' }); - Dashboard.openDashboard(); - Dashboard.verifyDashboardNotExist('Look at my dashboard'); - }); - - it('should verify index based session', () => { - Login.initialization(); - esApiAdvancedClient.waitForDocsCount(customSessionIndex, 1).then(() => { - // Backdate the session instead of waiting out the 1-minute timeout: the cleanup task - // deletes documents whose expiresAt has passed, and runs every second in this stack. - // Repeated, because live Kibana traffic rolls expiresAt forward and can rescue the doc. - esApiAdvancedClient.expireAllSessionsUntilSwept(customSessionIndex); - esApiAdvancedClient.waitForDocsCount(customSessionIndex, 0, 15000); - }); - }); - - it('should verify custom Kibana CSS', () => { - Login.initialization(); - cy.get('h1').shouldHaveStyle('color', 'rgb(0,128,0)'); - }); - - it('should verify custom Kibana JS', () => { - Login.initialization(); - cy.get('[data-testid="metadata-alert-message"]') - .should('exist') - .then($el => { - cy.log(`Alert message: ${$el.text()}`); - - cy.wrap($el).should('contain', 'Dear admin'); - }); - }); - - it('should verify custom middleware', () => { - Login.initialization(); - cy.get('[data-testid="metadata-enriched-data"]') - .should('exist') - .then($el => { - cy.log(`Entiched data: ${$el.text()}`); - - cy.wrap($el).should('contain', 'custom enriched data'); - }); - }); - - it('should verify whitelisted Urls', () => { - cy.request(`${Cypress.config().baseUrl}/api/index_management/indices`).then(response => { - expect(response.status).to.equal(200); - }); - - cy.request({ url: `${Cypress.config().baseUrl}/api/spaces/space`, failOnStatusCode: false }).then(response => { - expect(response.status).to.equal(403); - expect(response.body.error).to.equal('Unauthorized'); - }); - }); - }); - - describe('Default tenant middleware', () => { - before(() => { - rorApiInternalKbnClient.changeKibanaConfig('customMiddlewareDefaultTenantKibanaConfig.yml'); - kbnApiAdvancedClient.waitForKibanaHealth(requiredBaseUrl()); - }); - - // FIXME: flaky, about 2 runs in 16 on 8.19.19. When it fails the badge reads 'administrators', - // the normal first group, so the middleware's reorder of availableGroups on /pkp/api/info did - // not take โ€” and it then fails all three retries, so it is settled state and not a slow page. - // It behaves the same with clearSessionOnEvents set and unset, so it is not that. The other - // eight tests here are steady, so this is skipped rather than left to erode the signal. - it.skip('should open correct tenancy after login when custom middleware sets defaultGroup', () => { - Login.initialization(); - - Tenancy.checkTenancyNameInBadge('infosec', 'a'); - }); - }); - - describe('Custom kibana config multitenancy disabled', () => { - before(() => { - rorApiInternalKbnClient.changeKibanaConfig('customKibanaConfigMultitenancyDisabled.yml'); - kbnApiAdvancedClient.waitForKibanaHealth(requiredBaseUrl()); - }); - - it('should verify disabled multiTenancy', () => { - // With multitenancy off there is no tenancy query string, so the default finish URL of - // Loader.finish ('/app/home?tenancy=*') never matches. - Login.initialization({ finishUrl: '/app/home' }); - RorMenu.openRorMenu(); - RorMenu.verifyNoTenantAvailable(); - }); - - it('should verify custom Kibana index', () => { - const customIndex = `${customKibanaIndexName}_${getKibanaVersion()}_001`; - esApiClient.findIndicesByPattern(customIndex).then(result => { - const foundIndex = result.find(({ index }) => index === customIndex); - if (!foundIndex) throw new Error(`Expected to find an index matching ${customIndex}`); - expect(foundIndex.index).to.equal(customIndex); - expect(foundIndex.health).to.equal('green'); - expect(Number.parseInt(foundIndex['docs.count'], 10)).to.be.greaterThan(0); - }); - }); - }); - // xpack.reporting.index was removed in Kibana 8.0, so this only applies to the 7.x leg. - if (semver.lt(getKibanaVersion(), '8.0.0')) { - describe('Custom kibana config custom xpack.reporting.index', () => { - before(() => { - rorApiInternalKbnClient.changeKibanaConfig('customKibanaConfigXpackReportingIndex.yml'); - kbnApiAdvancedClient.waitForKibanaHealth(requiredBaseUrl()); - }); - - it('should verify custom reporting index', () => { - const docsIndex = 'sample_index'; - - SampleData.createSampleData(docsIndex, 1); - Login.initialization(); - - Discover.openDataViewPage(); - Discover.createIndexPattern('sample_index'); - Discover.saveReport('admin_search'); - Discover.exportToCsv(); - Reporting.openReportingPage('kibanaNavigation'); - Reporting.verifySavedReport(['admin_search']); - esApiAdvancedClient.getAllReportingIndices().then(results => { - expect(results).to.be.length(1); - const xpackReportingCustomIndex = results.find(index => index.index.startsWith('.reporting-test-index')); - /* eslint-disable no-unused-expressions */ - expect(xpackReportingCustomIndex).to.exist; - if (!xpackReportingCustomIndex) throw new Error('Expected to find a custom reporting index'); - expect(xpackReportingCustomIndex.health).to.equal('green'); - expect(Number.parseInt(xpackReportingCustomIndex['docs.count'], 10)).to.equal(1); - }); - - esApiClient.deleteIndex(docsIndex); - esApiAdvancedClient.pruneAllReportingIndices(); - kbnApiAdvancedClient.deleteSavedObjects('admin:dev'); - }); - }); - } else { - describe.skip('Custom kibana config custom xpack.reporting.index', () => { - // Tests are skipped - }); - } -}); diff --git a/e2e-tests/cypress/e2e/Settings.cy.ts b/e2e-tests/cypress/e2e/Settings.cy.ts index 9728507e..0349b199 100644 --- a/e2e-tests/cypress/e2e/Settings.cy.ts +++ b/e2e-tests/cypress/e2e/Settings.cy.ts @@ -31,11 +31,6 @@ describe('settings', () => { cy.log('should check save changes functionality when no changes provided'); // Settings.currentSettingsAlreadyLoadedToast().should('be.visible'); - cy.log('should check save changes functionality when malformed settings provided'); - Editor.changeConfig('readonlyrest:'); - Settings.clickSaveButton(); - // Settings.malformedSavedConfigurationToast().should('be.visible'); - cy.log('should check save changes functionality when success'); Editor.replaceValues('PERSONAL_GRP', `PERSONAL_GRP${Cypress._.random(0, 1e6)}`); Settings.clickSaveButton(); diff --git a/e2e-tests/cypress/e2e/Tenancy.cy.ts b/e2e-tests/cypress/e2e/Tenancy.cy.ts index 5d6c2afb..c7713210 100644 --- a/e2e-tests/cypress/e2e/Tenancy.cy.ts +++ b/e2e-tests/cypress/e2e/Tenancy.cy.ts @@ -56,6 +56,101 @@ describe('Tenancy', () => { runTests({ callbackBeforeLogin: openAnotherTabs }); }); + // Outside runTests: the share panel does not depend on how the session was opened, and the first + // test in runTests already checks that the tenancy survives each variant. This is the most + // expensive test in the suite, because it installs the ecommerce sample data. + describe('share link', () => { + afterEach(() => { + kbnApiClient.deleteSampleData('ecommerce', userCredentials); + }); + + it('should copy link to specific visualization with tenancy information', () => { + const urlWithTenancyId = `/s/default/app/management/data/index_management/indices?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedTenancyWithTemplateGroup}`; + Login.initialization({ + visitedUrl: urlWithTenancyId, + finishUrl: urlWithTenancyId, + spacePrefix: '' + }); + + kbnApiClient.loadSampleData('ecommerce', userCredentials, 'template_group'); + cy.waitForNetworkIdle('*', 500, { timeout: 10000 }); + KibanaNavigation.openPage('Discover'); + if (semver.gte(getKibanaVersion(), '8.0.0')) { + cy.get('[data-test-subj="discover-dataView-switch-link"]', { timeout: 30000 }).should('exist'); + } else { + cy.get('[data-test-subj="indexPattern-switch-link"]', { timeout: 30000 }).should('exist'); + } + Discover.openShareDiscover(); + Discover.clickCopyLinkButton('admin'); + if (semver.gte(getKibanaVersion(), '8.0.0')) { + cy.getValueFromClipboard() + .should('contain', 'https://localhost:5601/s/default/app/r/s') + .should('contain', `?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedTenancyWithTemplateGroup}`); + } else { + cy.getValueFromClipboard().should( + 'contain', + `https://localhost:5601/s/default/app/discover?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedTenancyWithTemplateGroup}#` + ); + } + + Dashboard.openDashboard(); + Dashboard.openItem(0); + Dashboard.openShareDashboard(); + Dashboard.clickCopyLinkButton(); + + if (semver.gte(getKibanaVersion(), '8.0.0')) { + cy.getValueFromClipboard() + .should('contain', 'https://localhost:5601/s/default/app/r/s') + .should('contain', `?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedTenancyWithTemplateGroup}`); + } else { + cy.getValueFromClipboard().should( + 'contain', + `https://localhost:5601/s/default/app/dashboards?${TENANCY_QUERY_STRING_KEY}=${Tenancy.encryptedTenancyWithTemplateGroup}#/` + ); + } + if (semver.lt(getKibanaVersion(), '8.0.0')) { + Dashboard.backToShareDashboard(); + } + Dashboard.clickEmbedTab(); + Dashboard.clickCopyEmbedCodeButton(); + + if (semver.gte(getKibanaVersion(), '8.0.0')) { + cy.getValueFromClipboard().should( + 'contain', + `