From f59a4ed6655a851b72c9b89d254ca6f30b6df2b0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 27 May 2026 23:33:05 +0300 Subject: [PATCH 01/12] chore(ci): add e2e shellcheck and actionlint tasks --- .github/actionlint.yaml | 4 +++ .../bash/e2e/configure-virtualization.sh | 3 ++ .shellcheckrc | 3 ++ Taskfile.yaml | 28 +++++++++++++++++++ api/scripts/update-codegen.sh | 3 +- images/virtualization-artifact/hack/args.sh | 2 +- images/virtualization-artifact/hack/dlv.sh | 1 + .../virtualization-artifact/hack/pyroscope.sh | 1 + 8 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .shellcheckrc diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..c2ec8d2d86 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - large +config-variables: null diff --git a/.github/scripts/bash/e2e/configure-virtualization.sh b/.github/scripts/bash/e2e/configure-virtualization.sh index efa9215815..db9a9d908d 100644 --- a/.github/scripts/bash/e2e/configure-virtualization.sh +++ b/.github/scripts/bash/e2e/configure-virtualization.sh @@ -11,6 +11,9 @@ source "${SCRIPT_DIR}/deckhouse.sh" require_env DEV_REGISTRY_DOCKER_CFG require_env NESTED_STORAGE_CLASS_NAME require_env VIRTUALIZATION_TAG +: "${DEV_REGISTRY_DOCKER_CFG:?}" +: "${NESTED_STORAGE_CLASS_NAME:?}" +: "${VIRTUALIZATION_TAG:?}" kubectl_apply_with_retry() { local count=20 diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000000..72c1748832 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,3 @@ +shell=bash +external-sources=true +enable=quote-safe-variables,deprecate-which,check-unassigned-uppercase diff --git a/Taskfile.yaml b/Taskfile.yaml index e698b063ed..3090c30af9 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -163,9 +163,37 @@ tasks: cmds: - task: lint:doc-ru - task: lint:prettier:yaml + - task: lint:shellcheck + - task: lint:actionlint - task: virtualization-controller:dvcr:lint - task: virtualization-controller:lint + lint:shellcheck: + desc: "Run shellcheck for CI shell scripts." + cmds: + - | + docker run --rm \ + -v "$PWD:/mnt" \ + -w /mnt \ + koalaman/shellcheck-alpine:v0.10.0 \ + .github/scripts/bash/e2e/*.sh \ + api/scripts/update-codegen.sh \ + images/virtualization-artifact/hack/args.sh \ + images/virtualization-artifact/hack/dlv.sh \ + images/virtualization-artifact/hack/pyroscope.sh + + lint:actionlint: + desc: "Run actionlint for E2E GitHub workflows." + cmds: + - | + docker run --rm \ + -v "$PWD:/repo" \ + -w /repo \ + rhysd/actionlint:1.7.7 \ + -color \ + -shellcheck= \ + .github/workflows/e2e*.yml + lint:doc-ru: desc: "Check the correspondence between description fields in the original crd and the Russian language version" cmds: diff --git a/api/scripts/update-codegen.sh b/api/scripts/update-codegen.sh index fccf4f01da..ab2f593493 100755 --- a/api/scripts/update-codegen.sh +++ b/api/scripts/update-codegen.sh @@ -43,6 +43,7 @@ function source::settings { "NodeUSBDevice" "USBDevice") + # shellcheck disable=SC1091 # kube_codegen.sh is resolved from the Go module cache at runtime. source "${CODEGEN_PKG}/kube_codegen.sh" } @@ -86,7 +87,7 @@ function generate::crds { if ! [[ " ${ALLOWED_RESOURCE_GEN_CRD[*]} " =~ [[:space:]]$(cat "$file" | yq '.spec.names.kind')[[:space:]] ]]; then continue fi - cp "$file" "${ROOT}/crds/$(echo $file | awk -Fio_ '{print $2}')" + cp "$file" "${ROOT}/crds/$(echo "$file" | awk -Fio_ '{print $2}')" done } diff --git a/images/virtualization-artifact/hack/args.sh b/images/virtualization-artifact/hack/args.sh index 481043edc2..98c08cfa03 100644 --- a/images/virtualization-artifact/hack/args.sh +++ b/images/virtualization-artifact/hack/args.sh @@ -36,7 +36,7 @@ function parse_flag() { local DEFAULT="${3:-}" local RESULT="" - for f in ${FLAGS[*]}; do + for f in "${FLAGS[@]}"; do case "${f}" in --${NAME}=*|-${SHORT_NAME}=*) RESULT="${f#*=}" diff --git a/images/virtualization-artifact/hack/dlv.sh b/images/virtualization-artifact/hack/dlv.sh index 9881789165..665a341911 100755 --- a/images/virtualization-artifact/hack/dlv.sh +++ b/images/virtualization-artifact/hack/dlv.sh @@ -109,6 +109,7 @@ DIR="$(dirname "$0")" ROOT="${DIR}/../../../" cd "$ROOT" +# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" diff --git a/images/virtualization-artifact/hack/pyroscope.sh b/images/virtualization-artifact/hack/pyroscope.sh index 80028a3428..af5e1b3e5c 100755 --- a/images/virtualization-artifact/hack/pyroscope.sh +++ b/images/virtualization-artifact/hack/pyroscope.sh @@ -88,6 +88,7 @@ function stop-pyroscope() { docker compose -f "${DOCKER_COMPOSE_FILE_PYROSCOPE_ONLY}" down } +# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" From d20a717f7e806a8e882a02a58397ecec3942b782 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 27 May 2026 23:34:04 +0300 Subject: [PATCH 02/12] chore(ci): rename nightly e2e workflows --- ...sable-pipeline.yml => e2e-nightly-reusable-pipeline.yml} | 2 +- .github/workflows/{e2e-matrix.yml => e2e-nightly.yml} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{e2e-reusable-pipeline.yml => e2e-nightly-reusable-pipeline.yml} (99%) rename .github/workflows/{e2e-matrix.yml => e2e-nightly.yml} (99%) diff --git a/.github/workflows/e2e-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml similarity index 99% rename from .github/workflows/e2e-reusable-pipeline.yml rename to .github/workflows/e2e-nightly-reusable-pipeline.yml index 9a452bb0b0..30c1b0341e 100644 --- a/.github/workflows/e2e-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: E2E Pipeline (Reusable) +name: E2E Nightly Pipeline (Reusable) on: workflow_call: diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-nightly.yml similarity index 99% rename from .github/workflows/e2e-matrix.yml rename to .github/workflows/e2e-nightly.yml index 421ef0cd57..5d8338a680 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-nightly.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: E2E Matrix Tests (nested clusters) +name: E2E Nightly on: workflow_dispatch: @@ -417,7 +417,7 @@ jobs: name: E2E Pipeline (Replicated) needs: - set-vars - uses: ./.github/workflows/e2e-reusable-pipeline.yml + uses: ./.github/workflows/e2e-nightly-reusable-pipeline.yml with: storage_type: replicated pipeline_job_name: "E2E Pipeline (Replicated)" @@ -445,7 +445,7 @@ jobs: name: E2E Pipeline (NFS) needs: - set-vars - uses: ./.github/workflows/e2e-reusable-pipeline.yml + uses: ./.github/workflows/e2e-nightly-reusable-pipeline.yml with: storage_type: nfs pipeline_job_name: "E2E Pipeline (NFS)" From eb028a86a8acd51ad9fed4d5baba7f1445c1df50 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 09:08:42 +0300 Subject: [PATCH 03/12] chore(ci): add e2e composite actions --- .../append-encrypted-artifacts-help/README.md | 3 + .../action.yml | 15 + .github/actions/gen-run-id/README.md | 3 + .github/actions/gen-run-id/action.yml | 18 + .../actions/gpg-encrypt-and-upload/README.md | 5 + .../actions/gpg-encrypt-and-upload/action.yml | 81 +++++ .github/actions/setup-e2e-toolchain/README.md | 5 + .../actions/setup-e2e-toolchain/action.yml | 56 +++ .../actions/use-nested-kubeconfig/README.md | 3 + .../actions/use-nested-kubeconfig/action.yml | 58 +++ .../templates/encrypted-artifacts-help.md | 37 ++ .../e2e-nightly-reusable-pipeline.yml | 313 ++++------------- .github/workflows/e2e-nightly.yml | 8 +- .../e2e-test-releases-reusable-pipeline.yml | 331 +++++------------- .github/workflows/e2e-test-releases.yml | 8 +- 15 files changed, 438 insertions(+), 506 deletions(-) create mode 100644 .github/actions/append-encrypted-artifacts-help/README.md create mode 100644 .github/actions/append-encrypted-artifacts-help/action.yml create mode 100644 .github/actions/gen-run-id/README.md create mode 100644 .github/actions/gen-run-id/action.yml create mode 100644 .github/actions/gpg-encrypt-and-upload/README.md create mode 100644 .github/actions/gpg-encrypt-and-upload/action.yml create mode 100644 .github/actions/setup-e2e-toolchain/README.md create mode 100644 .github/actions/setup-e2e-toolchain/action.yml create mode 100644 .github/actions/use-nested-kubeconfig/README.md create mode 100644 .github/actions/use-nested-kubeconfig/action.yml create mode 100644 .github/scripts/templates/encrypted-artifacts-help.md diff --git a/.github/actions/append-encrypted-artifacts-help/README.md b/.github/actions/append-encrypted-artifacts-help/README.md new file mode 100644 index 0000000000..46ff813227 --- /dev/null +++ b/.github/actions/append-encrypted-artifacts-help/README.md @@ -0,0 +1,3 @@ +# append-encrypted-artifacts-help + +Appends `.github/scripts/templates/encrypted-artifacts-help.md` to `GITHUB_STEP_SUMMARY`. diff --git a/.github/actions/append-encrypted-artifacts-help/action.yml b/.github/actions/append-encrypted-artifacts-help/action.yml new file mode 100644 index 0000000000..4a4040c48f --- /dev/null +++ b/.github/actions/append-encrypted-artifacts-help/action.yml @@ -0,0 +1,15 @@ +name: Append encrypted artifacts help +description: Append encrypted artifact decryption instructions to the job summary. +inputs: + template: + description: Help template path. + required: false + default: .github/scripts/templates/encrypted-artifacts-help.md +runs: + using: composite + steps: + - name: Append encrypted artifacts help + shell: bash + env: + TEMPLATE: ${{ inputs.template }} + run: cat "$TEMPLATE" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/gen-run-id/README.md b/.github/actions/gen-run-id/README.md new file mode 100644 index 0000000000..027e99d795 --- /dev/null +++ b/.github/actions/gen-run-id/README.md @@ -0,0 +1,3 @@ +# gen-run-id + +Generates the `date_start` and `randuuid4c` outputs used to name E2E namespaces and artifacts. diff --git a/.github/actions/gen-run-id/action.yml b/.github/actions/gen-run-id/action.yml new file mode 100644 index 0000000000..76a706b91b --- /dev/null +++ b/.github/actions/gen-run-id/action.yml @@ -0,0 +1,18 @@ +name: Generate E2E run id +description: Generate timestamp and short random suffix for E2E workflow runs. +outputs: + date_start: + description: Timestamp in %Y%m%d-%H%M%S format. + value: ${{ steps.vars.outputs.date-start }} + randuuid4c: + description: Four random hexadecimal characters. + value: ${{ steps.vars.outputs.randuuid4c }} +runs: + using: composite + steps: + - name: Generate run id + id: vars + shell: bash + run: | + echo "date-start=$(date +%Y%m%d-%H%M%S)" >> "$GITHUB_OUTPUT" + echo "randuuid4c=$(openssl rand -hex 2)" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/gpg-encrypt-and-upload/README.md b/.github/actions/gpg-encrypt-and-upload/README.md new file mode 100644 index 0000000000..fa7bd8ec96 --- /dev/null +++ b/.github/actions/gpg-encrypt-and-upload/README.md @@ -0,0 +1,5 @@ +# gpg-encrypt-and-upload + +Encrypts artifacts with GPG symmetric AES256 encryption and uploads the resulting `.gpg` file. + +Set `archive: "true"` for directory or multi-path inputs that should be zipped before encryption. Set `archive: "false"` for a single file such as a kubeconfig. diff --git a/.github/actions/gpg-encrypt-and-upload/action.yml b/.github/actions/gpg-encrypt-and-upload/action.yml new file mode 100644 index 0000000000..0647c82648 --- /dev/null +++ b/.github/actions/gpg-encrypt-and-upload/action.yml @@ -0,0 +1,81 @@ +name: GPG encrypt and upload +description: Encrypt a file or archived paths with GPG and upload the encrypted artifact. +inputs: + path: + description: File path to encrypt, or paths to zip when archive is true. + required: true + passphrase: + description: GPG symmetric encryption passphrase. + required: true + artifact_name: + description: Base artifact name without .gpg suffix. + required: true + working-directory: + description: Directory used for archive path resolution. + required: false + default: "." + archive: + description: Zip the provided paths before encryption. + required: false + default: "true" + retention-days: + description: Artifact retention in days. + required: false + default: "3" + overwrite: + description: Whether to overwrite an existing artifact. + required: false + default: "true" + include-hidden-files: + description: Whether upload-artifact includes hidden files. + required: false + default: "true" +runs: + using: composite + steps: + - name: Encrypt artifact + id: encrypt + shell: bash + env: + ARTIFACT_NAME: ${{ inputs.artifact_name }} + ARCHIVE: ${{ inputs.archive }} + GPG_PASSPHRASE: ${{ inputs.passphrase }} + INPUT_PATH: ${{ inputs.path }} + WORKING_DIRECTORY: ${{ inputs.working-directory }} + run: | + if [ "$ARCHIVE" = "true" ]; then + pushd "$WORKING_DIRECTORY" + # INPUT_PATH intentionally supports a whitespace-separated path list. + zip -r "$RUNNER_TEMP/${ARTIFACT_NAME}.zip" $INPUT_PATH + popd + input_file="$RUNNER_TEMP/${ARTIFACT_NAME}.zip" + encrypted_file="$RUNNER_TEMP/${ARTIFACT_NAME}.zip.gpg" + upload_name="${ARTIFACT_NAME}.zip.gpg" + else + input_file="$INPUT_PATH" + encrypted_file="$RUNNER_TEMP/${ARTIFACT_NAME}.gpg" + upload_name="${ARTIFACT_NAME}.gpg" + fi + + gpg --symmetric --batch --yes --pinentry-mode loopback \ + --passphrase "$GPG_PASSPHRASE" \ + --cipher-algo AES256 \ + --output "$encrypted_file" \ + "$input_file" + + if [ "$ARCHIVE" = "true" ]; then + rm -f "$input_file" + fi + + echo "encrypted_path=$encrypted_file" >> "$GITHUB_OUTPUT" + echo "upload_name=$upload_name" >> "$GITHUB_OUTPUT" + + - name: Upload encrypted artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ steps.encrypt.outputs.upload_name }} + path: ${{ steps.encrypt.outputs.encrypted_path }} + overwrite: ${{ inputs.overwrite }} + include-hidden-files: ${{ inputs.include-hidden-files }} + retention-days: ${{ inputs.retention-days }} + archive: false diff --git a/.github/actions/setup-e2e-toolchain/README.md b/.github/actions/setup-e2e-toolchain/README.md new file mode 100644 index 0000000000..c605b54a3f --- /dev/null +++ b/.github/actions/setup-e2e-toolchain/README.md @@ -0,0 +1,5 @@ +# setup-e2e-toolchain + +Installs the common E2E workflow toolchain: checkout, Task, deckhouse-cli (`d8`), and kubectl. + +Use `checkout: "false"` for jobs that already checked out the repository before calling this action. diff --git a/.github/actions/setup-e2e-toolchain/action.yml b/.github/actions/setup-e2e-toolchain/action.yml new file mode 100644 index 0000000000..e28bd53ee8 --- /dev/null +++ b/.github/actions/setup-e2e-toolchain/action.yml @@ -0,0 +1,56 @@ +name: Setup E2E toolchain +description: Checkout repository and install common E2E CLI tools. +inputs: + checkout: + description: Run actions/checkout before installing tools. + required: false + default: "true" + task-version: + description: go-task version to install. + required: false + default: 3.x + d8-version: + description: deckhouse-cli version to install. + required: false + default: v0.29.24 + install-kubectl: + description: Install kubectl via azure/setup-kubectl. + required: false + default: "true" + github-token: + description: GitHub token passed to go-task/setup-task. + required: false + default: "" +runs: + using: composite + steps: + - name: Checkout + if: inputs.checkout == 'true' + uses: actions/checkout@v6 + + - name: Install Task + uses: go-task/setup-task@v2 + with: + version: ${{ inputs.task-version }} + repo-token: ${{ inputs.github-token }} + + - name: Restore d8 cache + id: d8-cache + uses: actions/cache@v4 + with: + path: /opt/deckhouse/bin/d8 + key: d8-${{ inputs.d8-version }}-${{ runner.os }} + + - name: Setup d8 + if: steps.d8-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/install-d8 + with: + version: ${{ inputs.d8-version }} + + - name: Add d8 to PATH + shell: bash + run: echo "/opt/deckhouse/bin" >> "$GITHUB_PATH" + + - name: Install kubectl CLI + if: inputs.install-kubectl == 'true' + uses: azure/setup-kubectl@v4 diff --git a/.github/actions/use-nested-kubeconfig/README.md b/.github/actions/use-nested-kubeconfig/README.md new file mode 100644 index 0000000000..408e89e26c --- /dev/null +++ b/.github/actions/use-nested-kubeconfig/README.md @@ -0,0 +1,3 @@ +# use-nested-kubeconfig + +Decodes the nested-cluster kubeconfig used by E2E workflows into `~/.kube/config`, fixes permissions, selects the requested context, and can wait for `kubectl get nodes` to succeed. diff --git a/.github/actions/use-nested-kubeconfig/action.yml b/.github/actions/use-nested-kubeconfig/action.yml new file mode 100644 index 0000000000..0447f59f6f --- /dev/null +++ b/.github/actions/use-nested-kubeconfig/action.yml @@ -0,0 +1,58 @@ +name: Use nested kubeconfig +description: Decode a nested-cluster kubeconfig and optionally wait for the API. +inputs: + kubeconfig: + description: Double-base64 encoded kubeconfig. + required: true + context: + description: Context to select after writing kubeconfig. + required: false + default: nested-e2e-nested-sa + check-api: + description: Run kubectl get nodes with retries. + required: false + default: "true" + attempts: + description: Number of kubectl get nodes attempts. + required: false + default: "30" + delay-seconds: + description: Delay between API check attempts. + required: false + default: "10" +runs: + using: composite + steps: + - name: Configure nested kubeconfig + shell: bash + env: + KUBECONFIG_B64: ${{ inputs.kubeconfig }} + KUBECONTEXT: ${{ inputs.context }} + CHECK_API: ${{ inputs.check-api }} + ATTEMPTS: ${{ inputs.attempts }} + DELAY_SECONDS: ${{ inputs.delay-seconds }} + run: | + mkdir -p ~/.kube + echo "[INFO] Configure kubeconfig for nested cluster" + printf '%s' "$KUBECONFIG_B64" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + + if [ -n "$KUBECONTEXT" ]; then + kubectl config use-context "$KUBECONTEXT" + fi + + if [ "$CHECK_API" != "true" ]; then + exit 0 + fi + + for i in $(seq 1 "$ATTEMPTS"); do + echo "[INFO] Check nested kube-api availability ${i}/${ATTEMPTS}" + if kubectl get nodes; then + echo "[SUCCESS] Nested kube-api is available" + exit 0 + fi + sleep "$DELAY_SECONDS" + done + + echo "[ERROR] Nested kube-api is not available" + exit 1 diff --git a/.github/scripts/templates/encrypted-artifacts-help.md b/.github/scripts/templates/encrypted-artifacts-help.md new file mode 100644 index 0000000000..0e13edec02 --- /dev/null +++ b/.github/scripts/templates/encrypted-artifacts-help.md @@ -0,0 +1,37 @@ +## Encrypted artifacts + +Some uploaded artifacts in this workflow are encrypted with GPG symmetric encryption. + +Secret used for decryption passphrase: +- `E2E_ARTIFACTS_GPG_PASSPHRASE` + +Encrypted artifact types: +- `*-generated-files-*.zip.gpg` +- `*-generated-files-ssh-*.zip.gpg` +- `*-generated-files-kubeconfig-*.gpg` +- `*-release-generated-files-*.zip.gpg` +- `*-release-generated-files-ssh-*.zip.gpg` +- `*-release-generated-files-kubeconfig-*.gpg` + +Decrypt examples: + +```bash +# zip.gpg artifact +gpg --decrypt --batch --yes --pinentry-mode loopback \ + --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ + --output artifact.zip \ + artifact.zip.gpg + +unzip -o artifact.zip + +# same, but with simultaneous decryption and extraction of the whole archive +gpg --decrypt --batch --yes --pinentry-mode loopback \ + --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ + artifact.zip.gpg > artifact.zip && unzip -o artifact.zip + +# single-file .gpg artifact +gpg --decrypt --batch --yes --pinentry-mode loopback \ + --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ + --output kube-config \ + artifact.gpg +``` diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index 30c1b0341e..2adf9dfb0e 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -201,16 +201,14 @@ jobs: sudo apt-get update sudo apt-get install -y apache2-utils - - name: Install Task - uses: go-task/setup-task@v2 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v6 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + install-kubectl: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Log in to private registry env: @@ -461,176 +459,58 @@ jobs: yq e '.discovered.registry_auth = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_auth" echo "${{ steps.generate-kubeconfig.outputs.kubeconfig }}" | base64 -d | base64 -d > ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config || echo "kubeconfig not available, skipping" - - name: Encrypt generated files artifact - if: success() || failure() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-generated-files-${{ env.E2E_START_TIME }} - run: | - pushd ${{ env.SETUP_CLUSTER_TYPE_PATH }} - zip -r $RUNNER_TEMP/${ARTIFACT_NAME}.zip tmp values.yaml - popd - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.zip.gpg \ - $RUNNER_TEMP/${ARTIFACT_NAME}.zip - rm -f $RUNNER_TEMP/${ARTIFACT_NAME}.zip - - - name: Upload generated files - uses: actions/upload-artifact@v7 + - name: Encrypt and upload generated files artifact if: success() || failure() + uses: ./.github/actions/gpg-encrypt-and-upload with: - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-generated-files-${{ env.E2E_START_TIME }}.zip.gpg - overwrite: true - include-hidden-files: true + path: tmp values.yaml + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-generated-files-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - - - name: Encrypt ssh config artifact - if: always() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-generated-files-ssh-${{ env.E2E_START_TIME }} - run: | - pushd ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp - zip -r $RUNNER_TEMP/${ARTIFACT_NAME}.zip ssh - popd - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.zip.gpg \ - $RUNNER_TEMP/${ARTIFACT_NAME}.zip - rm -f $RUNNER_TEMP/${ARTIFACT_NAME}.zip - - name: Upload ssh config - uses: actions/upload-artifact@v7 + - name: Encrypt and upload ssh config artifact if: always() + uses: ./.github/actions/gpg-encrypt-and-upload with: - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-generated-files-ssh-${{ env.E2E_START_TIME }}.zip.gpg - overwrite: true - include-hidden-files: true + path: ssh + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-generated-files-ssh-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - - name: Encrypt kubeconfig artifact + - name: Encrypt and upload kubeconfig artifact if: always() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-generated-files-kubeconfig-${{ env.E2E_START_TIME }} - run: | - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.gpg \ - ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config - - - name: Upload kubeconfig - uses: actions/upload-artifact@v7 + uses: ./.github/actions/gpg-encrypt-and-upload with: - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-generated-files-kubeconfig-${{ env.E2E_START_TIME }}.gpg - overwrite: true - include-hidden-files: true + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + archive: "false" + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-generated-files-kubeconfig-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - name: Add encrypted artifacts help to job summary if: always() - run: | - cat >> "$GITHUB_STEP_SUMMARY" <<'EOF' - ## Encrypted artifacts - - Some uploaded artifacts in this workflow are encrypted with GPG symmetric encryption. - - Secret used for decryption passphrase: - - `E2E_ARTIFACTS_GPG_PASSPHRASE` - - Encrypted artifact types: - - `*-generated-files-*.zip.gpg` - - `*-generated-files-ssh-*.zip.gpg` - - `*-generated-files-kubeconfig-*.gpg` - - Decrypt commands: - - ```bash - # zip.gpg artifact - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - --output artifact.zip \ - artifact.zip.gpg - - unzip -o artifact.zip - - # same, but with simultaneous decryption and extraction of the whole archive - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - artifact.zip.gpg > artifact.zip && unzip -o artifact.zip - - # single-file .gpg artifact - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - --output kube-config \ - artifact.gpg - ``` - EOF + uses: ./.github/actions/append-encrypted-artifacts-help configure-sdn: name: Configure SDN runs-on: ubuntu-latest needs: bootstrap steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Install Task - uses: go-task/setup-task@v2 + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - name: Check nested kube-api via generated kubeconfig - run: | - mkdir -p ~/.kube - echo "[INFO] Configure kubeconfig for nested cluster" - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - - echo "[INFO] Show paths and files content" - ls -la ~/.kube - echo "[INFO] Set permissions for kubeconfig" - chmod 600 ~/.kube/config - - echo "[INFO] Show current kubeconfig context" - kubectl config get-contexts - - echo "[INFO] Show nodes in cluster" - # `kubectl get nodes` may return error, so we need to retry. - count=30 - success=false - for i in $(seq 1 $count); do - echo "[INFO] Attempt $i/$count..." - if kubectl get nodes; then - echo "[SUCCESS] Successfully retrieved nodes." - success=true - break - fi - - if [ $i -lt $count ]; then - echo "[INFO] Retrying in 10 seconds..." - sleep 10 - fi - done - - if [ "$success" = false ]; then - echo "[ERROR] Failed to retrieve nodes after $count attempts." - exit 1 - fi + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} - name: Enable SDN run: | # Keep SDN enable and readiness checks in a script because this step needs @@ -736,59 +616,19 @@ jobs: - configure-sdn - bootstrap steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Install Task - uses: go-task/setup-task@v2 + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - name: Check nested kube-api via generated kubeconfig - run: | - mkdir -p ~/.kube - echo "[INFO] Configure kubeconfig for nested cluster" - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - - echo "[INFO] Show paths and files content" - ls -la ~/.kube - echo "[INFO] Set permissions for kubeconfig" - chmod 600 ~/.kube/config - - echo "[INFO] Show current kubeconfig context" - kubectl config get-contexts - - echo "[INFO] Show nodes in cluster" - # `kubectl get nodes` may return error, so we need to retry. - count=30 - success=false - for i in $(seq 1 $count); do - echo "[INFO] Attempt $i/$count..." - if kubectl get nodes; then - echo "[SUCCESS] Successfully retrieved nodes." - success=true - break - fi - - if [ $i -lt $count ]; then - echo "[INFO] Retrying in 10 seconds..." - sleep 10 - fi - done - - if [ "$success" = false ]; then - echo "[ERROR] Failed to retrieve nodes after $count attempts." - exit 1 - fi - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} - name: Configure replicated storage id: storage-replicated-setup if: ${{ inputs.storage_type == 'replicated' }} @@ -1057,22 +897,19 @@ jobs: - bootstrap - configure-storage steps: - - uses: actions/checkout@v4 - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v6 - - name: Check kubeconfig - run: | - echo "[INFO] Configure kube config" - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup kubeconfig + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: Configure Virtualization env: DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} @@ -1286,11 +1123,13 @@ jobs: with: go-version: "${{ env.GO_VERSION }}" - - name: Install Task - uses: go-task/setup-task@v2 + - uses: actions/checkout@v6 + + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ginkgo working-directory: ./test/e2e/ @@ -1298,22 +1137,11 @@ jobs: echo "Install ginkgo" go install tool - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - - name: Setup kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - echo "[INFO] Test cluster connection by showing existing vmclass" - kubectl get vmclass - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: Download dependencies working-directory: ./test/e2e/ run: | @@ -1502,18 +1330,13 @@ jobs: - e2e-test if: cancelled() || success() steps: - - uses: actions/checkout@v4 - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v6 - - name: Install Task - uses: go-task/setup-task@v2 + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download artifacts uses: actions/download-artifact@v8 diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 5d8338a680..49c0600c9f 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -404,14 +404,14 @@ jobs: needs: power-off-vms-for-nested runs-on: ubuntu-latest outputs: - date_start: ${{ steps.vars.outputs.date-start }} + date_start: ${{ steps.vars.outputs.date_start }} randuuid4c: ${{ steps.vars.outputs.randuuid4c }} steps: + - uses: actions/checkout@v6 + - name: Set vars id: vars - run: | - echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT - echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + uses: ./.github/actions/gen-run-id e2e-replicated: name: E2E Pipeline (Replicated) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 6946ab790b..619dc64a8c 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -164,16 +164,14 @@ jobs: sudo apt-get update sudo apt-get install -y apache2-utils - - name: Install Task - uses: go-task/setup-task@v2 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v6 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + install-kubectl: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Log in to private registry env: @@ -424,122 +422,39 @@ jobs: yq e '.discovered.registry_auth = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_auth" echo "${{ steps.generate-kubeconfig.outputs.kubeconfig }}" | base64 -d | base64 -d > ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config || echo "kubeconfig not available, skipping" - - name: Encrypt generated files artifact - if: success() || failure() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-release-generated-files-${{ env.E2E_START_TIME }} - run: | - pushd ${{ env.SETUP_CLUSTER_TYPE_PATH }} - zip -r $RUNNER_TEMP/${ARTIFACT_NAME}.zip tmp values.yaml - popd - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.zip.gpg \ - $RUNNER_TEMP/${ARTIFACT_NAME}.zip - rm -f $RUNNER_TEMP/${ARTIFACT_NAME}.zip - - - name: Upload generated files - uses: actions/upload-artifact@v7 + - name: Encrypt and upload generated files artifact if: success() || failure() + uses: ./.github/actions/gpg-encrypt-and-upload with: - name: ${{ env.STORAGE_TYPE }}-release-generated-files-${{ env.E2E_START_TIME }}.zip.gpg - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-release-generated-files-${{ env.E2E_START_TIME }}.zip.gpg - overwrite: true - include-hidden-files: true + path: tmp values.yaml + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-release-generated-files-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - - name: Encrypt ssh config artifact - if: always() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-release-generated-files-ssh-${{ env.E2E_START_TIME }} - run: | - pushd ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp - zip -r $RUNNER_TEMP/${ARTIFACT_NAME}.zip ssh - popd - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.zip.gpg \ - $RUNNER_TEMP/${ARTIFACT_NAME}.zip - rm -f $RUNNER_TEMP/${ARTIFACT_NAME}.zip - - - name: Upload ssh config - uses: actions/upload-artifact@v7 + - name: Encrypt and upload ssh config artifact if: always() + uses: ./.github/actions/gpg-encrypt-and-upload with: - name: ${{ env.STORAGE_TYPE }}-release-generated-files-ssh-${{ env.E2E_START_TIME }}.zip.gpg - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-release-generated-files-ssh-${{ env.E2E_START_TIME }}.zip.gpg - overwrite: true - include-hidden-files: true + path: ssh + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-release-generated-files-ssh-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - - - name: Encrypt kubeconfig artifact - if: always() - env: - GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} - ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-release-generated-files-kubeconfig-${{ env.E2E_START_TIME }} - run: | - gpg --symmetric --batch --yes --pinentry-mode loopback \ - --passphrase "$GPG_PASSPHRASE" \ - --cipher-algo AES256 \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.gpg \ - ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config - - name: Upload kubeconfig - uses: actions/upload-artifact@v7 + - name: Encrypt and upload kubeconfig artifact if: always() + uses: ./.github/actions/gpg-encrypt-and-upload with: - name: ${{ env.STORAGE_TYPE }}-release-generated-files-kubeconfig-${{ env.E2E_START_TIME }}.gpg - path: ${{ runner.temp }}/${{ env.STORAGE_TYPE }}-release-generated-files-kubeconfig-${{ env.E2E_START_TIME }}.gpg - overwrite: true - include-hidden-files: true + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + archive: "false" + passphrase: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} + artifact_name: ${{ env.STORAGE_TYPE }}-release-generated-files-kubeconfig-${{ env.E2E_START_TIME }} retention-days: 3 - archive: false - name: Add encrypted artifacts help to job summary if: always() - run: | - cat >> "$GITHUB_STEP_SUMMARY" <<'EOF' - ## Encrypted artifacts - - Some uploaded artifacts in this workflow are encrypted with GPG symmetric encryption. - - Secret used for decryption passphrase: - - `E2E_ARTIFACTS_GPG_PASSPHRASE` - - Encrypted artifact types: - - `*-release-generated-files-*.zip.gpg` - - `*-release-generated-files-ssh-*.zip.gpg` - - `*-release-generated-files-kubeconfig-*.gpg` - - Decrypt commands: - - ```bash - # zip.gpg artifact - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - --output artifact.zip \ - artifact.zip.gpg - - unzip -o artifact.zip - - # same, but with simultaneous decryption and extraction of the whole archive - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - artifact.zip.gpg > artifact.zip && unzip -o artifact.zip - - # single-file .gpg artifact - gpg --decrypt --batch --yes --pinentry-mode loopback \ - --passphrase "$E2E_ARTIFACTS_GPG_PASSPHRASE" \ - --output kube-config \ - artifact.gpg - ``` - EOF + uses: ./.github/actions/append-encrypted-artifacts-help configure-sdn: name: Configure SDN @@ -548,56 +463,17 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install Task - uses: go-task/setup-task@v2 + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - name: Check nested kube-api via generated kubeconfig - run: | - mkdir -p ~/.kube - echo "[INFO] Configure kubeconfig for nested cluster" - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - - echo "[INFO] Show paths and files content" - ls -la ~/.kube - echo "[INFO] Set permissions for kubeconfig" - chmod 600 ~/.kube/config - - echo "[INFO] Show current kubeconfig context" - kubectl config get-contexts - - echo "[INFO] Show nodes in cluster" - # `kubectl get nodes` may return error, so we need to retry. - count=30 - success=false - for i in $(seq 1 $count); do - echo "[INFO] Attempt $i/$count..." - if kubectl get nodes; then - echo "[SUCCESS] Successfully retrieved nodes." - success=true - break - fi - - if [ $i -lt $count ]; then - echo "[INFO] Retrying in 10 seconds..." - sleep 10 - fi - done - - if [ "$success" = false ]; then - echo "[ERROR] Failed to retrieve nodes after $count attempts." - exit 1 - fi + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} - name: Enable SDN run: | echo "[INFO] Enable SDN" @@ -709,56 +585,17 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install Task - uses: go-task/setup-task@v2 + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - name: Check nested kube-api via generated kubeconfig - run: | - mkdir -p ~/.kube - echo "[INFO] Configure kubeconfig for nested cluster" - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - - echo "[INFO] Show paths and files content" - ls -la ~/.kube - echo "[INFO] Set permissions for kubeconfig" - chmod 600 ~/.kube/config - - echo "[INFO] Show current kubeconfig context" - kubectl config get-contexts - - echo "[INFO] Show nodes in cluster" - count=30 - success=false - for i in $(seq 1 $count); do - echo "[INFO] Attempt $i/$count..." - if kubectl get nodes; then - echo "[SUCCESS] Successfully retrieved nodes." - success=true - break - fi - - if [ $i -lt $count ]; then - echo "[INFO] Retrying in 10 seconds..." - sleep 10 - fi - done - - if [ "$success" = false ]; then - echo "[ERROR] Failed to retrieve nodes after $count attempts." - exit 1 - fi - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} - name: Configure replicated storage id: storage-replicated-setup if: ${{ inputs.storage_type == 'replicated' || inputs.storage_type == 'mixed' }} @@ -1131,21 +968,18 @@ jobs: - configure-storage steps: - uses: actions/checkout@v6 - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check kubeconfig - run: | - echo "[INFO] Configure kube config" - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup kubeconfig + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: Configure Virtualization run: | REGISTRY=$(base64 -d <<< "${{secrets.DEV_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) @@ -1394,21 +1228,19 @@ jobs: with: go-version: "${{ env.GO_VERSION }}" - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 + - uses: actions/checkout@v6 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: Install ginkgo working-directory: ./test/e2e/ run: | @@ -1500,20 +1332,17 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: Show current MPO state run: | echo "[INFO] Current ModulePullOverride before patching:" @@ -1636,13 +1465,11 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install kubectl CLI - uses: azure/setup-kubectl@v4 - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Go uses: actions/setup-go@v5 @@ -1662,12 +1489,10 @@ jobs: go mod download - name: Setup kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa - + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" - name: "Run E2E tests on new-release" env: CSI: ${{ inputs.storage_type }} diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 63aa7b7f2a..1e49a9664d 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -46,14 +46,14 @@ jobs: name: Set vars runs-on: ubuntu-latest outputs: - date_start: ${{ steps.vars.outputs.date-start }} + date_start: ${{ steps.vars.outputs.date_start }} randuuid4c: ${{ steps.vars.outputs.randuuid4c }} steps: + - uses: actions/checkout@v6 + - name: Set vars id: vars - run: | - echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT - echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + uses: ./.github/actions/gen-run-id resolve-release-inputs: name: Resolve release inputs From 1814cde6522908d7651462872e0238b8b5058e7c Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 15:13:40 +0300 Subject: [PATCH 04/12] chore(ci): extract nested VM power-off script Signed-off-by: Nikita Korolev --- .../scripts/bash/e2e/power-off-nested-vms.sh | 306 ++++++++++++++++++ .github/workflows/e2e-nightly.yml | 293 +---------------- 2 files changed, 309 insertions(+), 290 deletions(-) create mode 100755 .github/scripts/bash/e2e/power-off-nested-vms.sh diff --git a/.github/scripts/bash/e2e/power-off-nested-vms.sh b/.github/scripts/bash/e2e/power-off-nested-vms.sh new file mode 100755 index 0000000000..b27585effa --- /dev/null +++ b/.github/scripts/bash/e2e/power-off-nested-vms.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +# Constants (nested cluster: 1 master + 3 workers x2) +REQUIRED_MEM_GI=86 +REQUIRED_CPU=26 +MIN_MEM_GI_PER_NODE=12 +MIN_CPU_PER_NODE=4 +MIN_NODES_FOR_PLACEMENT=3 +POWER_OFF_POLL_INTERVAL_SEC=10 +POWER_OFF_WAIT_TIMEOUT_SEC=180 + +mem_to_gi() { + local q="$1" q_lower + q_lower=$(echo "$q" | tr '[:upper:]' '[:lower:]') + if [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)gi?$ ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)mi?$ ]]; then + echo "scale=4; ${BASH_REMATCH[1]} / 1024" | bc + elif [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)ki?$ ]]; then + echo "scale=6; ${BASH_REMATCH[1]} / 1024 / 1024" | bc + elif [[ "$q" =~ ^[0-9]+\.?[0-9]*$ ]]; then + echo "scale=6; $q / 1024 / 1024 / 1024" | bc + else + echo "0" + fi +} + +cpu_to_cores() { + local q="${1:-0}" q_lower + q_lower=$(echo "$q" | tr '[:upper:]' '[:lower:]') + if [[ "$q_lower" == *m ]]; then + echo "scale=4; ${q%[mM]} / 1000" | bc + else + echo "${q:-0}" + fi +} + +float_gt() { (($(echo "$1 > $2" | bc))); } +float_le() { (($(echo "$1 <= $2" | bc))); } + +worker_nodes=$(kubectl get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[*].metadata.name}') + +gather_node_resources() { + local available_mem_gi=0 + local available_cpu=0 + local nodes_meeting_min=0 + local node node_json alloc_mem_gi alloc_cpu pods_json requested_mem_gi requested_cpu + local node_free_mem node_free_cpu node_ok_mem node_ok_cpu + + for node in $worker_nodes; do + [[ -n "$node" ]] || continue + node_json=$(kubectl get node "$node" -o json 2>/dev/null) || true + if [[ -z "$node_json" ]]; then + echo "[WARN] Node $node: could not get node spec, skipping" >&2 + continue + fi + + alloc_mem_gi=$(mem_to_gi "$(echo "$node_json" | jq -r '.status.allocatable.memory // "0"')") + alloc_cpu=$(cpu_to_cores "$(echo "$node_json" | jq -r '.status.allocatable.cpu // "0"')") + + pods_json=$(kubectl get pods -A --field-selector spec.nodeName="$node" -o json 2>/dev/null) || true + requested_mem_gi=0 + requested_cpu=0 + if [[ -n "$pods_json" ]]; then + while read -r qty; do + [[ -z "$qty" ]] && continue + requested_mem_gi=$(echo "$requested_mem_gi + $(mem_to_gi "$qty")" | bc) + done < <(echo "$pods_json" | jq -r ' + .items[] + | select(.status.phase == "Running" or .status.phase == "Pending") + | [(.spec.containers[]? | try .resources.requests.memory catch null), (.spec.initContainers[]? | try .resources.requests.memory catch null)] + | .[] | . // "0" + ') + + while read -r qty; do + [[ -z "$qty" ]] && continue + requested_cpu=$(echo "$requested_cpu + $(cpu_to_cores "$qty")" | bc) + done < <(echo "$pods_json" | jq -r ' + .items[] + | select(.status.phase == "Running" or .status.phase == "Pending") + | [(.spec.containers[]? | try .resources.requests.cpu catch null), (.spec.initContainers[]? | try .resources.requests.cpu catch null)] + | .[] | . // "0" + ') + fi + + node_free_mem=$(echo "x = $alloc_mem_gi - $requested_mem_gi; if (x < 0) 0 else x" | bc 2>/dev/null || echo "0") + node_free_cpu=$(echo "x = $alloc_cpu - $requested_cpu; if (x < 0) 0 else x" | bc 2>/dev/null || echo "0") + + available_mem_gi=$(echo "$available_mem_gi + $node_free_mem" | bc) + available_cpu=$(echo "$available_cpu + $node_free_cpu" | bc) + + node_ok_mem=$(echo "$node_free_mem >= $MIN_MEM_GI_PER_NODE" | bc) + node_ok_cpu=$(echo "$node_free_cpu >= $MIN_CPU_PER_NODE" | bc) + if [[ "$node_ok_mem" -eq 1 && "$node_ok_cpu" -eq 1 ]]; then + nodes_meeting_min=$((nodes_meeting_min + 1)) + else + echo "[INFO] Node $node: does not meet placement min - free ${node_free_mem} Gi RAM, ${node_free_cpu} CPU (required: >= ${MIN_MEM_GI_PER_NODE} Gi, >= ${MIN_CPU_PER_NODE} CPU)" >&2 + fi + done + + printf '%s\t%s\t%s\n' "$available_mem_gi" "$available_cpu" "$nodes_meeting_min" +} + +refresh_resource_state() { + IFS=$'\t' read -r available_mem_gi available_cpu nodes_meeting_min < <(gather_node_resources) + deficit_mem=$(echo "$REQUIRED_MEM_GI - $available_mem_gi" | bc 2>/dev/null || echo "$REQUIRED_MEM_GI") + deficit_cpu=$(echo "$REQUIRED_CPU - $available_cpu" | bc 2>/dev/null || echo "$REQUIRED_CPU") + + total_sufficient=false + if float_le "$deficit_mem" 0 && float_le "$deficit_cpu" 0; then + total_sufficient=true + fi + + placement_sufficient=false + if [[ $nodes_meeting_min -ge $MIN_NODES_FOR_PLACEMENT ]]; then + placement_sufficient=true + fi +} + +get_vms_candidates() { + kubectl get vm -A -o json | jq -r ' + .items[] + | select(.metadata.namespace | test("^nightly-e2e-|static-cse") | not) + | select(.metadata.labels | tostring | test("e2e-cluster/do-not-stop-vm-on-e2e-run") | not) + | select(.status.phase != "Stopped") + | [.metadata.namespace, .metadata.name, (.spec.memory.size // "0"), (.spec.cpu.cores // 0), (.spec.cpu.coreFraction // "100%")] + | @tsv + ' +} + +sort_by_mem_desc() { + while IFS=$'\t' read -r ns name mem_qty cores core_frac; do + [[ -n "$ns" ]] || continue + mem_gi=$(mem_to_gi "$mem_qty") + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$mem_gi" "$ns" "$name" "$mem_qty" "$cores" "$core_frac" + done | sort -t$'\t' -k1,1 -rn +} + +vm_cpu_from_cores_and_fraction() { + local cores="$1" core_frac="$2" frac_pct=100 + [[ "$core_frac" =~ ^([0-9]+)%$ ]] && frac_pct="${BASH_REMATCH[1]}" + echo "scale=2; $cores * $frac_pct / 100" | bc +} + +print_power_off_plan() { + local plan_index=0 cumulative_mem=0 cumulative_cpu=0 + local vm_mem_gi ns name mem_qty cores core_frac vm_cpu + + echo "[INFO] Planned power-off order with projected VM-spec resources:" + echo "[INFO] Projection is based on VM spec memory/cpu; actual placement improvement depends on where workloads are running." + + while IFS=$'\t' read -r vm_mem_gi ns name mem_qty cores core_frac; do + [[ -n "$ns" ]] || continue + plan_index=$((plan_index + 1)) + vm_cpu=$(vm_cpu_from_cores_and_fraction "$cores" "$core_frac") + cumulative_mem=$(echo "$cumulative_mem + $vm_mem_gi" | bc) + cumulative_cpu=$(echo "$cumulative_cpu + $vm_cpu" | bc) + echo "[PLAN] ${plan_index}. ${ns}/${name} -> ${vm_mem_gi} Gi RAM, ${vm_cpu} CPU (cumulative: ${cumulative_mem} Gi RAM, ${cumulative_cpu} CPU)" + done < "$1" + + if [[ $plan_index -eq 0 ]]; then + echo "[WARN] No VM candidates available for power off" + fi +} + +count_stopped_requested_vms() { + local requested_vms_file="$1" + local stopped_requested=0 total_requested=0 + local ns name phase + + while IFS=$'\t' read -r ns name; do + [[ -n "$ns" ]] || continue + total_requested=$((total_requested + 1)) + phase=$(kubectl get vm -n "$ns" "$name" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + if [[ "$phase" == "Stopped" ]]; then + stopped_requested=$((stopped_requested + 1)) + fi + done < "$requested_vms_file" + + printf '%s\t%s\n' "$stopped_requested" "$total_requested" +} + +still_need_to_free() { + if ! $placement_sufficient; then return 0; fi + if ! $total_sufficient; then return 0; fi + return 1 +} + +main() { + refresh_resource_state + echo "[INFO] Workers: free ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min} (need at least ${MIN_NODES_FOR_PLACEMENT})" + echo "[INFO] Required: ${REQUIRED_MEM_GI} Gi, ${REQUIRED_CPU} CPU; need >= ${MIN_NODES_FOR_PLACEMENT} nodes with >= ${MIN_MEM_GI_PER_NODE} Gi and >= ${MIN_CPU_PER_NODE} CPU" + echo " " + + if $total_sufficient && $placement_sufficient; then + echo "[INFO] Resources sufficient (total + placement), no VMs to power off" + exit 0 + fi + + if $total_sufficient; then + echo "[INFO] Cluster has enough free memory and cpu." + else + shortage_parts="" + float_gt "$deficit_mem" 0 && shortage_parts="${deficit_mem} Gi RAM" + float_gt "$deficit_cpu" 0 && shortage_parts="${shortage_parts:+$shortage_parts, }${deficit_cpu} CPU" + echo "[INFO] Resources shortage: need to free ${shortage_parts}. Proceed with power off some VMs to free cluster resources." + fi + if $placement_sufficient; then + echo "[INFO] Cluster has enough available nodes." + else + echo "[INFO] Available nodes shortage: only ${nodes_meeting_min} node(s) meet free resources requirement, expect at least ${MIN_NODES_FOR_PLACEMENT} available nodes. Proceed with power off some VMs to free resources." + fi + echo "[Note] Will ignore VMs in 'nightly-e2e-*', 'static-cse' namespaces, and VMs with the 'e2e-cluster/do-not-stop-vm-on-e2e-run' label." + echo "[INFO] Power off candidates sorted by memory (largest first); stop when enough resources are freed." + + vm_candidates_file=$(mktemp) + requested_vms_file=$(mktemp) + trap 'rm -f "$vm_candidates_file" "$requested_vms_file"' EXIT + get_vms_candidates | sort_by_mem_desc > "$vm_candidates_file" + print_power_off_plan "$vm_candidates_file" + + requested_count=0 + while IFS=$'\t' read -r vm_mem_gi ns name mem_qty cores core_frac; do + [[ -n "$ns" ]] || continue + vm_cpu=$(vm_cpu_from_cores_and_fraction "$cores" "$core_frac") + + echo "[INFO] Request power off for vm $ns/$name (${vm_mem_gi} Gi, ${vm_cpu} CPU)" + if ! kubectl patch vm -n "$ns" "$name" --type=merge -p '{"spec":{"runPolicy":"AlwaysOff"}}'; then + echo "[WARN] Failed to power off vm $ns/$name, skip it and continue with next candidate" + continue + fi + printf '%s\t%s\n' "$ns" "$name" >> "$requested_vms_file" + requested_count=$((requested_count + 1)) + done < "$vm_candidates_file" + + if [[ $requested_count -eq 0 ]]; then + echo "[ERROR] No running VM candidates available for power off, but resources are still insufficient." + echo "[ERROR] Human intervention is required." + rm -f "$vm_candidates_file" "$requested_vms_file" + trap - EXIT + exit 1 + fi + + echo "[INFO] Requested power off for ${requested_count} VM(s). Waiting up to ${POWER_OFF_WAIT_TIMEOUT_SEC}s and checking cluster resources every ${POWER_OFF_POLL_INTERVAL_SEC}s." + + wait_elapsed=0 + prev_nodes_meeting_min="$nodes_meeting_min" + while true; do + refresh_resource_state + IFS=$'\t' read -r stopped_requested total_requested < <(count_stopped_requested_vms "$requested_vms_file") + echo "[INFO] Current workers free: ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min}" + echo "[INFO] Requested VMs stopped: ${stopped_requested}/${total_requested}; waited ${wait_elapsed}s/${POWER_OFF_WAIT_TIMEOUT_SEC}s" + if [[ $prev_nodes_meeting_min -lt $MIN_NODES_FOR_PLACEMENT && $nodes_meeting_min -ge $MIN_NODES_FOR_PLACEMENT ]]; then + echo "[INFO] Placement now sufficient: ${nodes_meeting_min} nodes with >= ${MIN_MEM_GI_PER_NODE} Gi and >= ${MIN_CPU_PER_NODE} CPU" + fi + prev_nodes_meeting_min="$nodes_meeting_min" + + if ! still_need_to_free; then + break + fi + + if [[ $total_requested -gt 0 && $stopped_requested -eq $total_requested ]]; then + echo "[INFO] All requested VMs are already stopped; no need to wait further." + break + fi + + if [[ $wait_elapsed -ge $POWER_OFF_WAIT_TIMEOUT_SEC ]]; then + break + fi + + sleep "$POWER_OFF_POLL_INTERVAL_SEC" + wait_elapsed=$((wait_elapsed + POWER_OFF_POLL_INTERVAL_SEC)) + done + + rm -f "$vm_candidates_file" "$requested_vms_file" + trap - EXIT + + echo "[INFO] Final workers free: ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min}" + + if still_need_to_free; then + echo "[ERROR] Stopping VMs did not free enough resources. Human intervention is required." + exit 1 + fi +} + +main "$@" diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 49c0600c9f..018ef12046 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -102,6 +102,8 @@ jobs: needs: cleanup-nested-clusters runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 with: @@ -109,296 +111,7 @@ jobs: context: e2e-cluster-nightly-e2e-virt-sa kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} - name: Power off VMs to free resources for nested cluster setup - run: | - set -euo pipefail - - # Constants (nested cluster: 1 master + 3 workers x2) - REQUIRED_MEM_GI=86 - REQUIRED_CPU=26 - MIN_MEM_GI_PER_NODE=12 - MIN_CPU_PER_NODE=4 - MIN_NODES_FOR_PLACEMENT=3 - POWER_OFF_POLL_INTERVAL_SEC=10 - POWER_OFF_WAIT_TIMEOUT_SEC=180 - - # Helpers: Kubernetes quantity -> numeric (portable, no bash 4+) - mem_to_gi() { - local q="$1" q_lower - q_lower=$(echo "$q" | tr '[:upper:]' '[:lower:]') - if [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)gi?$ ]]; then - echo "${BASH_REMATCH[1]}" - elif [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)mi?$ ]]; then - echo "scale=4; ${BASH_REMATCH[1]} / 1024" | bc - elif [[ "$q_lower" =~ ^([0-9]+\.?[0-9]*)ki?$ ]]; then - echo "scale=6; ${BASH_REMATCH[1]} / 1024 / 1024" | bc - elif [[ "$q" =~ ^[0-9]+\.?[0-9]*$ ]]; then - echo "scale=6; $q / 1024 / 1024 / 1024" | bc - else - echo "0" - fi - } - - cpu_to_cores() { - local q="${1:-0}" q_lower - q_lower=$(echo "$q" | tr '[:upper:]' '[:lower:]') - if [[ "$q_lower" == *m ]]; then - echo "scale=4; ${q%[mM]} / 1000" | bc - else - echo "${q:-0}" - fi - } - - # Compare two numbers (bc outputs 0 or 1; (( )) treats 0 as false) - float_ge() { (( $(echo "$1 >= $2" | bc) )); } - float_gt() { (( $(echo "$1 > $2" | bc) )); } - float_le() { (( $(echo "$1 <= $2" | bc) )); } - float_lt() { (( $(echo "$1 < $2" | bc) )); } - - # Gather free resources like the scheduler: allocatable - sum(pod requests) per node. - worker_nodes=$(kubectl get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[*].metadata.name}') - gather_node_resources() { - local available_mem_gi=0 - local available_cpu=0 - local nodes_meeting_min=0 - local node node_json alloc_mem_gi alloc_cpu pods_json requested_mem_gi requested_cpu - local node_free_mem node_free_cpu node_ok_mem node_ok_cpu - - for node in $worker_nodes; do - [[ -n "$node" ]] || continue - node_json=$(kubectl get node "$node" -o json 2>/dev/null) || true - if [[ -z "$node_json" ]]; then - echo "[WARN] Node $node: could not get node spec, skipping" >&2 - continue - fi - - alloc_mem_gi=$(mem_to_gi "$(echo "$node_json" | jq -r '.status.allocatable.memory // "0"')") - alloc_cpu=$(cpu_to_cores "$(echo "$node_json" | jq -r '.status.allocatable.cpu // "0"')") - - pods_json=$(kubectl get pods -A --field-selector spec.nodeName="$node" -o json 2>/dev/null) || true - requested_mem_gi=0 - requested_cpu=0 - if [[ -n "$pods_json" ]]; then - while read -r qty; do - [[ -z "$qty" ]] && continue - requested_mem_gi=$(echo "$requested_mem_gi + $(mem_to_gi "$qty")" | bc) - done < <(echo "$pods_json" | jq -r ' - .items[] - | select(.status.phase == "Running" or .status.phase == "Pending") - | [(.spec.containers[]? | try .resources.requests.memory catch null), (.spec.initContainers[]? | try .resources.requests.memory catch null)] - | .[] | . // "0" - ') - - while read -r qty; do - [[ -z "$qty" ]] && continue - requested_cpu=$(echo "$requested_cpu + $(cpu_to_cores "$qty")" | bc) - done < <(echo "$pods_json" | jq -r ' - .items[] - | select(.status.phase == "Running" or .status.phase == "Pending") - | [(.spec.containers[]? | try .resources.requests.cpu catch null), (.spec.initContainers[]? | try .resources.requests.cpu catch null)] - | .[] | . // "0" - ') - fi - - node_free_mem=$(echo "x = $alloc_mem_gi - $requested_mem_gi; if (x < 0) 0 else x" | bc 2>/dev/null || echo "0") - node_free_cpu=$(echo "x = $alloc_cpu - $requested_cpu; if (x < 0) 0 else x" | bc 2>/dev/null || echo "0") - - available_mem_gi=$(echo "$available_mem_gi + $node_free_mem" | bc) - available_cpu=$(echo "$available_cpu + $node_free_cpu" | bc) - - node_ok_mem=$(echo "$node_free_mem >= $MIN_MEM_GI_PER_NODE" | bc) - node_ok_cpu=$(echo "$node_free_cpu >= $MIN_CPU_PER_NODE" | bc) - if [[ "$node_ok_mem" -eq 1 && "$node_ok_cpu" -eq 1 ]]; then - nodes_meeting_min=$((nodes_meeting_min + 1)) - else - echo "[INFO] Node $node: does not meet placement min — free ${node_free_mem} Gi RAM, ${node_free_cpu} CPU (required: >= ${MIN_MEM_GI_PER_NODE} Gi, >= ${MIN_CPU_PER_NODE} CPU)" >&2 - fi - done - - printf '%s\t%s\t%s\n' "$available_mem_gi" "$available_cpu" "$nodes_meeting_min" - } - - refresh_resource_state() { - IFS=$'\t' read -r available_mem_gi available_cpu nodes_meeting_min < <(gather_node_resources) - deficit_mem=$(echo "$REQUIRED_MEM_GI - $available_mem_gi" | bc 2>/dev/null || echo "$REQUIRED_MEM_GI") - deficit_cpu=$(echo "$REQUIRED_CPU - $available_cpu" | bc 2>/dev/null || echo "$REQUIRED_CPU") - - total_sufficient=false - if float_le "$deficit_mem" 0 && float_le "$deficit_cpu" 0; then - total_sufficient=true - fi - - placement_sufficient=false - if [[ $nodes_meeting_min -ge $MIN_NODES_FOR_PLACEMENT ]]; then - placement_sufficient=true - fi - } - - refresh_resource_state - echo "[INFO] Workers: free ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min} (need at least ${MIN_NODES_FOR_PLACEMENT})" - echo "[INFO] Required: ${REQUIRED_MEM_GI} Gi, ${REQUIRED_CPU} CPU; need >= ${MIN_NODES_FOR_PLACEMENT} nodes with >= ${MIN_MEM_GI_PER_NODE} Gi and >= ${MIN_CPU_PER_NODE} CPU" - echo " " - - if $total_sufficient && $placement_sufficient; then - echo "[INFO] Resources sufficient (total + placement), no VMs to power off" - exit 0 - fi - - if $total_sufficient; then - echo "[INFO] Cluster has enough free memory and cpu." - else - shortage_parts="" - float_gt "$deficit_mem" 0 && shortage_parts="${deficit_mem} Gi RAM" - float_gt "$deficit_cpu" 0 && shortage_parts="${shortage_parts:+$shortage_parts, }${deficit_cpu} CPU" - echo "[INFO] Resources shortage: need to free ${shortage_parts}. Proceed with power off some VMs to free cluster resources." - fi - if $placement_sufficient; then - echo "[INFO] Cluster has enough available nodes." - else - echo "[INFO] Available nodes shortage: only ${nodes_meeting_min} node(s) meet free resources requirement, expect at least ${MIN_NODES_FOR_PLACEMENT} available nodes. Proceed with power off some VMs to free resources." - fi - echo "[Note] Will ignore VMs in 'nightly-e2e-*', 'static-cse' namespaces, and VMs with the 'e2e-cluster/do-not-stop-vm-on-e2e-run' label." - echo "[INFO] Power off candidates sorted by memory (largest first); stop when enough resources are freed." - - # Power off VMs until we have enough (exclude nightly-e2e-*, static-cse, do-not-stop) - get_vms_candidates() { - kubectl get vm -A -o json | jq -r ' - .items[] - | select(.metadata.namespace | test("^nightly-e2e-|static-cse") | not) - | select(.metadata.labels | tostring | test("e2e-cluster/do-not-stop-vm-on-e2e-run") | not) - | select(.status.phase != "Stopped") - | [.metadata.namespace, .metadata.name, (.spec.memory.size // "0"), (.spec.cpu.cores // 0), (.spec.cpu.coreFraction // "100%")] - | @tsv - ' - } - - # Sort by memory descending (largest first) so we free the most with fewer power-offs - sort_by_mem_desc() { - while IFS=$'\t' read -r ns name mem_qty cores core_frac; do - [[ -n "$ns" ]] || continue - mem_gi=$(mem_to_gi "$mem_qty") - printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$mem_gi" "$ns" "$name" "$mem_qty" "$cores" "$core_frac" - done | sort -t$'\t' -k1,1 -rn - } - - vm_cpu_from_cores_and_fraction() { - local cores="$1" core_frac="$2" frac_pct=100 - [[ "$core_frac" =~ ^([0-9]+)%$ ]] && frac_pct="${BASH_REMATCH[1]}" - echo "scale=2; $cores * $frac_pct / 100" | bc - } - - print_power_off_plan() { - local plan_index=0 cumulative_mem=0 cumulative_cpu=0 - local vm_mem_gi ns name mem_qty cores core_frac vm_cpu - - echo "[INFO] Planned power-off order with projected VM-spec resources:" - echo "[INFO] Projection is based on VM spec memory/cpu; actual placement improvement depends on where workloads are running." - - while IFS=$'\t' read -r vm_mem_gi ns name mem_qty cores core_frac; do - [[ -n "$ns" ]] || continue - plan_index=$((plan_index + 1)) - vm_cpu=$(vm_cpu_from_cores_and_fraction "$cores" "$core_frac") - cumulative_mem=$(echo "$cumulative_mem + $vm_mem_gi" | bc) - cumulative_cpu=$(echo "$cumulative_cpu + $vm_cpu" | bc) - echo "[PLAN] ${plan_index}. ${ns}/${name} -> ${vm_mem_gi} Gi RAM, ${vm_cpu} CPU (cumulative: ${cumulative_mem} Gi RAM, ${cumulative_cpu} CPU)" - done < "$1" - - if [[ $plan_index -eq 0 ]]; then - echo "[WARN] No VM candidates available for power off" - fi - } - - count_stopped_requested_vms() { - local requested_vms_file="$1" - local stopped_requested=0 total_requested=0 - local ns name phase - - while IFS=$'\t' read -r ns name; do - [[ -n "$ns" ]] || continue - total_requested=$((total_requested + 1)) - phase=$(kubectl get vm -n "$ns" "$name" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - if [[ "$phase" == "Stopped" ]]; then - stopped_requested=$((stopped_requested + 1)) - fi - done < "$requested_vms_file" - - printf '%s\t%s\n' "$stopped_requested" "$total_requested" - } - - # Keep powering off while current cluster state still does not satisfy placement or total resources. - still_need_to_free() { - if ! $placement_sufficient; then return 0; fi - if ! $total_sufficient; then return 0; fi - return 1 - } - - vm_candidates_file=$(mktemp) - requested_vms_file=$(mktemp) - trap 'rm -f "$vm_candidates_file" "$requested_vms_file"' EXIT - get_vms_candidates | sort_by_mem_desc > "$vm_candidates_file" - print_power_off_plan "$vm_candidates_file" - - requested_count=0 - while IFS=$'\t' read -r vm_mem_gi ns name mem_qty cores core_frac; do - [[ -n "$ns" ]] || continue - vm_cpu=$(vm_cpu_from_cores_and_fraction "$cores" "$core_frac") - - echo "[INFO] Request power off for vm $ns/$name (${vm_mem_gi} Gi, ${vm_cpu} CPU)" - if ! kubectl patch vm -n "$ns" "$name" --type=merge -p '{"spec":{"runPolicy":"AlwaysOff"}}'; then - echo "[WARN] Failed to power off vm $ns/$name, skip it and continue with next candidate" - continue - fi - printf '%s\t%s\n' "$ns" "$name" >> "$requested_vms_file" - requested_count=$((requested_count + 1)) - done < "$vm_candidates_file" - - if [[ $requested_count -eq 0 ]]; then - echo "[ERROR] No running VM candidates available for power off, but resources are still insufficient." - echo "[ERROR] Human intervention is required." - rm -f "$vm_candidates_file" "$requested_vms_file" - trap - EXIT - exit 1 - fi - - echo "[INFO] Requested power off for ${requested_count} VM(s). Waiting up to ${POWER_OFF_WAIT_TIMEOUT_SEC}s and checking cluster resources every ${POWER_OFF_POLL_INTERVAL_SEC}s." - - wait_elapsed=0 - prev_nodes_meeting_min="$nodes_meeting_min" - while true; do - refresh_resource_state - IFS=$'\t' read -r stopped_requested total_requested < <(count_stopped_requested_vms "$requested_vms_file") - echo "[INFO] Current workers free: ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min}" - echo "[INFO] Requested VMs stopped: ${stopped_requested}/${total_requested}; waited ${wait_elapsed}s/${POWER_OFF_WAIT_TIMEOUT_SEC}s" - if [[ $prev_nodes_meeting_min -lt $MIN_NODES_FOR_PLACEMENT && $nodes_meeting_min -ge $MIN_NODES_FOR_PLACEMENT ]]; then - echo "[INFO] Placement now sufficient: ${nodes_meeting_min} nodes with >= ${MIN_MEM_GI_PER_NODE} Gi and >= ${MIN_CPU_PER_NODE} CPU" - fi - prev_nodes_meeting_min="$nodes_meeting_min" - - if ! still_need_to_free; then - break - fi - - if [[ $total_requested -gt 0 && $stopped_requested -eq $total_requested ]]; then - echo "[INFO] All requested VMs are already stopped; no need to wait further." - break - fi - - if [[ $wait_elapsed -ge $POWER_OFF_WAIT_TIMEOUT_SEC ]]; then - break - fi - - sleep "$POWER_OFF_POLL_INTERVAL_SEC" - wait_elapsed=$((wait_elapsed + POWER_OFF_POLL_INTERVAL_SEC)) - done - - rm -f "$vm_candidates_file" "$requested_vms_file" - trap - EXIT - - echo "[INFO] Final workers free: ${available_mem_gi} Gi RAM, ${available_cpu} CPU; nodes with enough free resources for placement: ${nodes_meeting_min}" - - if still_need_to_free; then - echo "[ERROR] Stopping VMs did not free enough resources. Human intervention is required." - exit 1 - fi + run: .github/scripts/bash/e2e/power-off-nested-vms.sh set-vars: name: Set vars needs: power-off-vms-for-nested From 545498d47553534430a4687288d5317f092dc72f Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 15:37:29 +0300 Subject: [PATCH 05/12] chore(ci): extract e2e wait helpers Signed-off-by: Nikita Korolev --- .github/scripts/bash/e2e/deckhouse.sh | 40 ++ .../scripts/bash/e2e/wait-sds-replicated.sh | 157 ++++++++ .../bash/e2e/wait-virtualization-ready.sh | 158 ++++++++ .../e2e-nightly-reusable-pipeline.yml | 327 +--------------- .../e2e-test-releases-reusable-pipeline.yml | 356 +----------------- 5 files changed, 360 insertions(+), 678 deletions(-) create mode 100644 .github/scripts/bash/e2e/wait-sds-replicated.sh create mode 100644 .github/scripts/bash/e2e/wait-virtualization-ready.sh diff --git a/.github/scripts/bash/e2e/deckhouse.sh b/.github/scripts/bash/e2e/deckhouse.sh index f5534c17c5..bafc0d657c 100644 --- a/.github/scripts/bash/e2e/deckhouse.sh +++ b/.github/scripts/bash/e2e/deckhouse.sh @@ -7,6 +7,46 @@ show_deckhouse_state() { d8 s queue list | head -n25 || true } +d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" +} + +d8_queue() { + local count=90 + local delay=10 + local queue_count + + for i in $(seq 1 "$count"); do + queue_count="$(d8_queue_list)" + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + + if [ "$i" -lt "$count" ]; then + sleep "$delay" + fi + done + + echo "[ERROR] Deckhouse queue is not clear after ${count} attempts" + return 1 +} + wait_for_deckhouse_queue() { local count=60 local delay=10 diff --git a/.github/scripts/bash/e2e/wait-sds-replicated.sh b/.github/scripts/bash/e2e/wait-sds-replicated.sh new file mode 100644 index 0000000000..d5d2bc81cd --- /dev/null +++ b/.github/scripts/bash/e2e/wait-sds-replicated.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" +# shellcheck source=.github/scripts/bash/e2e/deckhouse.sh +source "${SCRIPT_DIR}/deckhouse.sh" + +sds_replicated_ready() { + local count=60 + local sds_replicated_volume_status + + for i in $(seq 1 "$count"); do + sds_replicated_volume_status="$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False")" + + if [[ "$sds_replicated_volume_status" = "Active" ]]; then + echo "[SUCCESS] Namespaces sds-replicated-volume are Active" + kubectl get ns d8-sds-replicated-volume + return 0 + fi + + echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[INFO] Show namespaces sds-replicated-volume" + kubectl get ns | grep sds-replicated-volume || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + sleep 10 + done + + echo "[ERROR] Namespaces sds-replicated-volume are not ready after ${count} attempts" + echo "[DEBUG] Show namespaces sds" + kubectl get ns | grep sds || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "No queues" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + return 1 +} + +sds_pods_ready() { + local count=100 + local linstor_node + local csi_node + local workers + + workers="$(kubectl get nodes -o name | grep -c worker || true)" + workers=$((workers)) + + echo "[INFO] Wait while linstor-node csi-node webhooks pods are ready" + for i in $(seq 1 "$count"); do + linstor_node="$(kubectl -n d8-sds-replicated-volume get pods | grep -c "linstor-node.*Running" || true)" + csi_node="$(kubectl -n d8-sds-replicated-volume get pods | grep -c "csi-node.*Running" || true)" + + echo "[INFO] Check if sds-replicated pods are ready" + if [[ "$linstor_node" -ge "$workers" && "$csi_node" -ge "$workers" ]]; then + echo "[SUCCESS] sds-replicated-volume is ready" + return 0 + fi + + echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}" + echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Get pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "Failed to retrieve list queue" + echo " " + fi + sleep 10 + done + + echo "[ERROR] sds-replicated-volume is not ready after ${count} attempts" + echo "[DEBUG] Get pods" + echo "::group::sds-replicated-volume pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "::endgroup::" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "Failed to retrieve list queue" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + return 1 +} + +blockdevices_ready() { + local count=60 + local workers + local blockdevices + + workers="$(kubectl get nodes -o name | grep -c worker || true)" + workers=$((workers)) + + if [[ "$workers" -eq 0 ]]; then + echo "[ERROR] No worker nodes found" + return 1 + fi + + for i in $(seq 1 "$count"); do + blockdevices="$(kubectl get blockdevice -o name | wc -l | tr -d ' ' || true)" + blockdevices=$((blockdevices)) + if [[ "$blockdevices" -ge "$workers" ]]; then + echo "[SUCCESS] Blockdevices is greater or equal to $workers" + kubectl get blockdevice + return 0 + fi + + echo "[INFO] Wait 10 sec until blockdevices is greater or equal to $workers (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + + sleep 10 + done + + echo "[ERROR] Blockdevices is not 3" + echo "[DEBUG] Show cluster nodes" + kubectl get nodes || echo "[WARNING] Failed to get cluster nodes" + echo "[DEBUG] Show blockdevices" + kubectl get blockdevice || echo "[WARNING] Failed to get blockdevices" + echo "[DEBUG] Show sds namespaces" + kubectl get ns | grep sds || echo "[WARNING] Namespace sds is not found" + echo "[DEBUG] Show pods in sds-replicated-volume" + echo "::group::pods in sds-replicated-volume" + kubectl -n d8-sds-replicated-volume get pods || echo "[WARNING] Failed to get pods in sds-replicated-volume" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 || echo "[WARNING] Failed to get deckhouse logs" + echo "::endgroup::" + return 1 +} diff --git a/.github/scripts/bash/e2e/wait-virtualization-ready.sh b/.github/scripts/bash/e2e/wait-virtualization-ready.sh new file mode 100644 index 0000000000..a505d8e3c1 --- /dev/null +++ b/.github/scripts/bash/e2e/wait-virtualization-ready.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" +# shellcheck source=.github/scripts/bash/e2e/deckhouse.sh +source "${SCRIPT_DIR}/deckhouse.sh" + +debug_output() { + local nodes + + echo "[ERROR] Virtualization module deploy failed" + echo "[DEBUG] Show describe virtualization module" + echo "::group::describe virtualization module" + kubectl describe modules virtualization || true + echo "::endgroup::" + echo "[DEBUG] Show namespace d8-virtualization" + kubectl get ns d8-virtualization || true + echo "[DEBUG] Show pods in namespace d8-virtualization" + kubectl -n d8-virtualization get pods || true + echo "[DEBUG] Show dvcr info" + echo "::group::dvcr pod describe" + kubectl -n d8-virtualization describe pod -l app=dvcr || true + echo "::endgroup::" + echo " " + echo "::group::dvcr pod yaml" + kubectl -n d8-virtualization get pods -l app=dvcr -o yaml || true + echo "::endgroup::" + echo " " + echo "::group::dvcr deployment yaml" + kubectl -n d8-virtualization get deployment -l app=dvcr -o yaml || true + echo "::endgroup::" + echo " " + echo "::group::dvcr deployment describe" + kubectl -n d8-virtualization describe deployment -l app=dvcr || true + echo "::endgroup::" + echo " " + echo "::group::dvcr service yaml" + kubectl -n d8-virtualization get service -l app=dvcr -o yaml || true + echo "::endgroup::" + echo " " + echo "[DEBUG] Show pvc in namespace d8-virtualization" + kubectl get pvc -n d8-virtualization || true + echo "[DEBUG] Show cluster StorageClasses" + kubectl get storageclasses || true + echo "[DEBUG] Show cluster nodes" + kubectl get node || true + + echo "[DEBUG] Show cluster node yaml and describe" + nodes="$(kubectl get no -o jsonpath='{range .items[?(@.metadata.name)]}{.metadata.name}{"\n"}{end}')" + for node in $nodes; do + echo "::group::show cluster node ${node} yaml" + kubectl get node "$node" -o yaml + echo "::endgroup::" + echo "::group::show cluster node ${node} describe" + kubectl describe node "$node" + echo "::endgroup::" + done + + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" +} + +virtualization_ready() { + local count=90 + local virtualization_status + + for i in $(seq 1 "$count"); do + virtualization_status="$(kubectl get modules virtualization -o jsonpath='{.status.phase}')" + if [ "$virtualization_status" = "Ready" ]; then + echo "[SUCCESS] Virtualization module is ready" + kubectl get modules virtualization + kubectl -n d8-virtualization get pods + kubectl get vmclass || echo "[WARNING] no vmclasses found" + return 0 + fi + + echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt ${i}/${count})" + + if (( i % 5 == 0 )); then + echo " " + echo "[DEBUG] Show additional info" + kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" + echo " " + kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" + kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" + echo " " + echo "d8-virtualization module status: ${virtualization_status}" + echo " " + fi + sleep 10 + done + + debug_output + return 1 +} + +virt_handler_ready() { + local count=180 + local virt_handler_ready + local workers + local time_wait=10 + + for i in $(seq 1 "$count"); do + workers="$(kubectl get nodes -o name | grep -c worker || true)" + workers=$((workers)) + if [[ "$workers" -eq 0 ]]; then + echo "[WARNING] No worker nodes found, keep waiting" + echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt ${i}/${count})" + sleep "$time_wait" + continue + fi + + virt_handler_ready="$(kubectl -n d8-virtualization get pods | grep -c "virt-handler.*Running" || true)" + + if [[ "$virt_handler_ready" -ge "$workers" ]]; then + echo "[SUCCESS] virt-handlers pods are ready ${virt_handler_ready}/${workers}" + return 0 + fi + + echo "[INFO] virt-handler pods ${virt_handler_ready}/${workers}" + echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show pods in namespace d8-virtualization" + echo "::group::virtualization pods" + kubectl -n d8-virtualization get pods || echo "[WARNING] No pods in virtualization namespace found" + echo "::endgroup::" + echo "[DEBUG] Show cluster nodes" + echo "::group::cluster nodes" + kubectl get node || echo "[WARNING] Failed to get cluster nodes" + echo "::endgroup::" + fi + sleep "$time_wait" + done + + debug_output + return 1 +} diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index 2adf9dfb0e..2247bb235f 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -634,166 +634,7 @@ jobs: if: ${{ inputs.storage_type == 'replicated' }} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } - - sds_replicated_ready() { - local count=60 - for i in $(seq 1 $count); do - - sds_replicated_volume_status=$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False") - - if [[ "${sds_replicated_volume_status}" = "Active" ]]; then - echo "[SUCCESS] Namespaces sds-replicated-volume are Active" - kubectl get ns d8-sds-replicated-volume - return 0 - fi - - echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[INFO] Show namespaces sds-replicated-volume" - kubectl get ns | grep sds-replicated-volume || echo "Namespaces sds-replicated-volume are not ready" - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n25 || echo "No queues" - fi - sleep 10 - done - - echo "[ERROR] Namespaces sds-replicated-volume are not ready after ${count} attempts" - echo "[DEBUG] Show namespaces sds" - kubectl get ns | grep sds || echo "Namespaces sds-replicated-volume are not ready" - echo "[DEBUG] Show queue" - echo "::group::📦 Show queue" - d8 s queue list || echo "No queues" - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - exit 1 - } - - sds_pods_ready() { - local count=100 - local linstor_node - local csi_node - local webhooks - local workers=$(kubectl get nodes -o name | grep worker | wc -l || true) - workers=$((workers)) - - echo "[INFO] Wait while linstor-node csi-node webhooks pods are ready" - for i in $(seq 1 $count); do - linstor_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "linstor-node.*Running" | wc -l || true) - csi_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "csi-node.*Running" | wc -l || true) - - echo "[INFO] Check if sds-replicated pods are ready" - if [[ ${linstor_node} -ge ${workers} && ${csi_node} -ge ${workers} ]]; then - echo "[SUCCESS] sds-replicated-volume is ready" - return 0 - fi - - echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}" - echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[DEBUG] Get pods" - kubectl -n d8-sds-replicated-volume get pods || true - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n 25 || echo "Failed to retrieve list queue" - echo " " - fi - sleep 10 - done - - echo "[ERROR] sds-replicated-volume is not ready after ${count} attempts" - echo "[DEBUG] Get pods" - echo "::group::📦 sds-replicated-volume pods" - kubectl -n d8-sds-replicated-volume get pods || true - echo "::endgroup::" - echo "[DEBUG] Show queue" - echo "::group::📦 Show queue" - d8 s queue list || echo "Failed to retrieve list queue" - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - exit 1 - } - - blockdevices_ready() { - local count=60 - workers=$(kubectl get nodes -o name | grep worker | wc -l) - workers=$((workers)) - - if [[ $workers -eq 0 ]]; then - echo "[ERROR] No worker nodes found" - exit 1 - fi - - for i in $(seq 1 $count); do - blockdevices=$(kubectl get blockdevice -o name | wc -l || true) - if [ $blockdevices -ge $workers ]; then - echo "[SUCCESS] Blockdevices is greater or equal to $workers" - kubectl get blockdevice - return 0 - fi - - echo "[INFO] Wait 10 sec until blockdevices is greater or equal to $workers (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n25 || echo "No queues" - fi - - sleep 10 - done - - echo "[ERROR] Blockdevices is not 3" - echo "[DEBUG] Show cluster nodes" - kubectl get nodes || echo "[WARNING] Failed to get cluster nodes" - echo "[DEBUG] Show blockdevices" - kubectl get blockdevice || echo "[WARNING] Failed to get blockdevices" - echo "[DEBUG] Show sds namespaces" - kubectl get ns | grep sds || echo "[WARNING] Namespace sds is not found" - echo "[DEBUG] Show pods in sds-replicated-volume" - echo "::group::📦 pods in sds-replicated-volume" - kubectl -n d8-sds-replicated-volume get pods || echo "[WARNING] Failed to get pods in sds-replicated-volume" - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 || echo "[WARNING] Failed to get deckhouse logs" - echo "::endgroup::" - exit 1 - } + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/wait-sds-replicated.sh" d8_queue @@ -921,171 +762,7 @@ jobs: bash .github/scripts/bash/e2e/configure-virtualization.sh - name: Wait for Virtualization to be ready run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "Failed to retrieve list queue" - } - - debug_output() { - local NODES - - echo "[ERROR] Virtualization module deploy failed" - echo "[DEBUG] Show describe virtualization module" - echo "::group::📦 describe virtualization module" - kubectl describe modules virtualization || true - echo "::endgroup::" - echo "[DEBUG] Show namespace d8-virtualization" - kubectl get ns d8-virtualization || true - echo "[DEBUG] Show pods in namespace d8-virtualization" - kubectl -n d8-virtualization get pods || true - echo "[DEBUG] Show dvcr info" - echo "::group::📦 dvcr pod describe" - kubectl -n d8-virtualization describe pod -l app=dvcr || true - echo "::endgroup::" - echo " " - echo "::group::📦 dvcr pod yaml" - kubectl -n d8-virtualization get pods -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "::group::📦 dvcr deployment yaml" - kubectl -n d8-virtualization get deployment -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "::group::📦 dvcr deployment describe" - kubectl -n d8-virtualization describe deployment -l app=dvcr || true - echo "::endgroup::" - echo " " - echo "::group::📦 dvcr service yaml" - kubectl -n d8-virtualization get service -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "[DEBUG] Show pvc in namespace d8-virtualization" - kubectl get pvc -n d8-virtualization || true - echo "[DEBUG] Show cluster StorageClasses" - kubectl get storageclasses || true - echo "[DEBUG] Show cluster nodes" - kubectl get node || true - - echo "[DEBUG] Show cluster node yaml and describe" - NODES=$(kubectl get no -o jsonpath='{range .items[?(@.metadata.name)]}{.metadata.name}{"\n"}{end}') - for node in $NODES; do - echo "::group::📝 show cluster node $node yaml" - kubectl get node $node -o yaml - echo "::endgroup::" - echo "::group::📝 show cluster node $node describe" - kubectl describe node $node - echo "::endgroup::" - done - - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" - echo "[DEBUG] Show deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::📝 deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } - - virtualization_ready() { - local count=90 - local virtualization_status - - for i in $(seq 1 $count) ; do - virtualization_status=$(kubectl get modules virtualization -o jsonpath='{.status.phase}') - if [ "$virtualization_status" == "Ready" ]; then - echo "[SUCCESS] Virtualization module is ready" - kubectl get modules virtualization - kubectl -n d8-virtualization get pods - kubectl get vmclass || echo "[WARNING] no vmclasses found" - return 0 - fi - - echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)" - - if (( i % 5 == 0 )); then - echo " " - echo "[DEBUG] Show additional info" - kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" - echo " " - kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" - kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" - echo " " - fi - sleep 10 - done - - debug_output - exit 1 - } - - virt_handler_ready() { - local count=180 - local virt_handler_ready - local workers - local time_wait=10 - - for i in $(seq 1 $count); do - workers=$(kubectl get nodes -o name | grep worker | wc -l || true) - workers=$((workers)) - if [[ $workers -eq 0 ]]; then - echo "[WARNING] No worker nodes found, keep waiting" - echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" - sleep ${time_wait} - continue - fi - - virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true) - - if [[ $virt_handler_ready -ge $workers ]]; then - echo "[SUCCESS] virt-handlers pods are ready $virt_handler_ready/$workers" - return 0 - fi - - echo "[INFO] virt-handler pods $virt_handler_ready/$workers" - echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" - if (( i % 5 == 0 )); then - echo "[DEBUG] Show pods in namespace d8-virtualization" - echo "::group::📦 virtualization pods" - kubectl -n d8-virtualization get pods || echo "[WARNING] No pods in virtualization namespace found" - echo "::endgroup::" - echo "[DEBUG] Show cluster nodes" - echo "::group::📦 cluster nodes" - kubectl get node || echo "[WARNING] Failed to get cluster nodes" - echo "::endgroup::" - fi - sleep ${time_wait} - done - - debug_output - exit 1 - } + source .github/scripts/bash/e2e/wait-virtualization-ready.sh echo " " echo "[INFO] Waiting for Virtualization module to be ready" diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 619dc64a8c..96b7081cce 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -601,166 +601,7 @@ jobs: if: ${{ inputs.storage_type == 'replicated' || inputs.storage_type == 'mixed' }} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } - - sds_replicated_ready() { - local count=60 - for i in $(seq 1 $count); do - - sds_replicated_volume_status=$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False") - - if [[ "${sds_replicated_volume_status}" = "Active" ]]; then - echo "[SUCCESS] Namespaces sds-replicated-volume are Active" - kubectl get ns d8-sds-replicated-volume - return 0 - fi - - echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[INFO] Show namespaces sds-replicated-volume" - kubectl get ns | grep sds-replicated-volume || echo "Namespaces sds-replicated-volume are not ready" - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n25 || echo "No queues" - fi - sleep 10 - done - - echo "[ERROR] Namespaces sds-replicated-volume are not ready after ${count} attempts" - echo "[DEBUG] Show namespaces sds" - kubectl get ns | grep sds || echo "Namespaces sds-replicated-volume are not ready" - echo "[DEBUG] Show queue" - echo "::group::Show queue" - d8 s queue list || echo "No queues" - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - exit 1 - } - - sds_pods_ready() { - local count=100 - local linstor_node - local csi_node - local webhooks - local workers=$(kubectl get nodes -o name | grep worker | wc -l || true) - workers=$((workers)) - - echo "[INFO] Wait while linstor-node csi-node webhooks pods are ready" - for i in $(seq 1 $count); do - linstor_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "linstor-node.*Running" | wc -l || true) - csi_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "csi-node.*Running" | wc -l || true) - - echo "[INFO] Check if sds-replicated pods are ready" - if [[ ${linstor_node} -ge ${workers} && ${csi_node} -ge ${workers} ]]; then - echo "[SUCCESS] sds-replicated-volume is ready" - return 0 - fi - - echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}" - echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[DEBUG] Get pods" - kubectl -n d8-sds-replicated-volume get pods || true - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n 25 || echo "Failed to retrieve list queue" - echo " " - fi - sleep 10 - done - - echo "[ERROR] sds-replicated-volume is not ready after ${count} attempts" - echo "[DEBUG] Get pods" - echo "::group::sds-replicated-volume pods" - kubectl -n d8-sds-replicated-volume get pods || true - echo "::endgroup::" - echo "[DEBUG] Show queue" - echo "::group::Show queue" - d8 s queue list || echo "Failed to retrieve list queue" - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - exit 1 - } - - blockdevices_ready() { - local count=60 - workers=$(kubectl get nodes -o name | grep worker | wc -l) - workers=$((workers)) - - if [[ $workers -eq 0 ]]; then - echo "[ERROR] No worker nodes found" - exit 1 - fi - - for i in $(seq 1 $count); do - blockdevices=$(kubectl get blockdevice -o name | wc -l || true) - if [ $blockdevices -ge $workers ]; then - echo "[SUCCESS] Blockdevices is greater or equal to $workers" - kubectl get blockdevice - return 0 - fi - - echo "[INFO] Wait 10 sec until blockdevices is greater or equal to $workers (attempt ${i}/${count})" - if (( i % 5 == 0 )); then - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n25 || echo "No queues" - fi - - sleep 10 - done - - echo "[ERROR] Blockdevices is not 3" - echo "[DEBUG] Show cluster nodes" - kubectl get nodes - echo "[DEBUG] Show blockdevices" - kubectl get blockdevice - echo "[DEBUG] Show sds namespaces" - kubectl get ns | grep sds || echo "ns sds is not found" - echo "[DEBUG] Show pods in sds-replicated-volume" - echo "::group::pods in sds-replicated-volume" - kubectl -n d8-sds-replicated-volume get pods || true - echo "::endgroup::" - echo "[DEBUG] Show deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - exit 1 - } + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/wait-sds-replicated.sh" d8_queue @@ -798,38 +639,7 @@ jobs: if: ${{ inputs.storage_type == 'local' || inputs.storage_type == 'mixed' }} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-local-volume run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/deckhouse.sh" sds_local_volume_ready() { local count=90 @@ -1039,167 +849,7 @@ jobs: kubectl get mpo virtualization - name: Wait for Virtualization to be ready run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "Failed to retrieve list queue" - } - - debug_output() { - local NODES - - echo "[ERROR] Virtualization module deploy failed" - echo "[DEBUG] Show describe virtualization module" - echo "::group::describe virtualization module" - kubectl describe modules virtualization || true - echo "::endgroup::" - echo "[DEBUG] Show namespace d8-virtualization" - kubectl get ns d8-virtualization || true - echo "[DEBUG] Show pods in namespace d8-virtualization" - kubectl -n d8-virtualization get pods || true - echo "[DEBUG] Show dvcr info" - echo "::group::dvcr pod describe" - kubectl -n d8-virtualization describe pod -l app=dvcr || true - echo "::endgroup::" - echo " " - echo "::group::dvcr pod yaml" - kubectl -n d8-virtualization get pods -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "::group::dvcr deployment yaml" - kubectl -n d8-virtualization get deployment -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "::group::dvcr deployment describe" - kubectl -n d8-virtualization describe deployment -l app=dvcr || true - echo "::endgroup::" - echo " " - echo "::group::dvcr service yaml" - kubectl -n d8-virtualization get service -l app=dvcr -o yaml || true - echo "::endgroup::" - echo " " - echo "[DEBUG] Show pvc in namespace d8-virtualization" - kubectl get pvc -n d8-virtualization || true - echo "[DEBUG] Show cluster StorageClasses" - kubectl get storageclasses || true - echo "[DEBUG] Show cluster nodes" - kubectl get node - - echo "[DEBUG] Show cluster node yaml and describe" - NODES=$(kubectl get no -o jsonpath='{range .items[?(@.metadata.name)]}{.metadata.name}{"\n"}{end}') - for node in $NODES; do - echo "::group::show cluster node $node yaml" - kubectl get node $node -o yaml - echo "::endgroup::" - echo "::group::show cluster node $node describe" - kubectl describe node $node - echo "::endgroup::" - done - - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" - echo "[DEBUG] Show deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } - - virtualization_ready() { - local count=90 - local virtualization_status - - for i in $(seq 1 $count) ; do - virtualization_status=$(kubectl get modules virtualization -o jsonpath='{.status.phase}') - if [ "$virtualization_status" == "Ready" ]; then - echo "[SUCCESS] Virtualization module is ready" - kubectl get modules virtualization - kubectl -n d8-virtualization get pods - kubectl get vmclass || echo "[WARNING] no vmclasses found" - return 0 - fi - - echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)" - - if (( i % 5 == 0 )); then - echo " " - echo "[DEBUG] Show additional info" - kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" - echo " " - kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" - kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" - echo " " - echo "d8-virtualization module status: $virtualization_status" - echo " " - fi - sleep 10 - done - - debug_output - exit 1 - } - - virt_handler_ready() { - local count=180 - local virt_handler_ready - local workers - local time_wait=10 - - workers=$(kubectl get nodes -o name | grep worker | wc -l || true) - workers=$((workers)) - - for i in $(seq 1 $count); do - virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true) - - if [[ $virt_handler_ready -ge $workers ]]; then - echo "[SUCCESS] virt-handlers pods are ready" - return 0 - fi - - echo "[INFO] virt-handler pods $virt_handler_ready/$workers " - echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" - if (( i % 5 == 0 )); then - echo "[DEBUG] Show pods in namespace d8-virtualization" - echo "::group::virtualization pods" - kubectl -n d8-virtualization get pods || echo "No pods in virtualization namespace found" - echo "::endgroup::" - echo "[DEBUG] Show cluster nodes" - echo "::group::cluster nodes" - kubectl get node - echo "::endgroup::" - fi - sleep ${time_wait} - done - - debug_output - exit 1 - } + source .github/scripts/bash/e2e/wait-virtualization-ready.sh echo " " echo "[INFO] Waiting for Virtualization module to be ready" From 5937983fab134de0c178888e82ef6d308816bb9f Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 17:41:33 +0300 Subject: [PATCH 06/12] chore(ci): drop unrelated script changes Signed-off-by: Nikita Korolev --- api/scripts/update-codegen.sh | 3 +-- images/virtualization-artifact/hack/args.sh | 2 +- images/virtualization-artifact/hack/dlv.sh | 1 - images/virtualization-artifact/hack/pyroscope.sh | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/api/scripts/update-codegen.sh b/api/scripts/update-codegen.sh index ab2f593493..fccf4f01da 100755 --- a/api/scripts/update-codegen.sh +++ b/api/scripts/update-codegen.sh @@ -43,7 +43,6 @@ function source::settings { "NodeUSBDevice" "USBDevice") - # shellcheck disable=SC1091 # kube_codegen.sh is resolved from the Go module cache at runtime. source "${CODEGEN_PKG}/kube_codegen.sh" } @@ -87,7 +86,7 @@ function generate::crds { if ! [[ " ${ALLOWED_RESOURCE_GEN_CRD[*]} " =~ [[:space:]]$(cat "$file" | yq '.spec.names.kind')[[:space:]] ]]; then continue fi - cp "$file" "${ROOT}/crds/$(echo "$file" | awk -Fio_ '{print $2}')" + cp "$file" "${ROOT}/crds/$(echo $file | awk -Fio_ '{print $2}')" done } diff --git a/images/virtualization-artifact/hack/args.sh b/images/virtualization-artifact/hack/args.sh index 98c08cfa03..481043edc2 100644 --- a/images/virtualization-artifact/hack/args.sh +++ b/images/virtualization-artifact/hack/args.sh @@ -36,7 +36,7 @@ function parse_flag() { local DEFAULT="${3:-}" local RESULT="" - for f in "${FLAGS[@]}"; do + for f in ${FLAGS[*]}; do case "${f}" in --${NAME}=*|-${SHORT_NAME}=*) RESULT="${f#*=}" diff --git a/images/virtualization-artifact/hack/dlv.sh b/images/virtualization-artifact/hack/dlv.sh index 665a341911..9881789165 100755 --- a/images/virtualization-artifact/hack/dlv.sh +++ b/images/virtualization-artifact/hack/dlv.sh @@ -109,7 +109,6 @@ DIR="$(dirname "$0")" ROOT="${DIR}/../../../" cd "$ROOT" -# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" diff --git a/images/virtualization-artifact/hack/pyroscope.sh b/images/virtualization-artifact/hack/pyroscope.sh index af5e1b3e5c..80028a3428 100755 --- a/images/virtualization-artifact/hack/pyroscope.sh +++ b/images/virtualization-artifact/hack/pyroscope.sh @@ -88,7 +88,6 @@ function stop-pyroscope() { docker compose -f "${DOCKER_COMPOSE_FILE_PYROSCOPE_ONLY}" down } -# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" From 9a915ec2039a0bbd4db5fff0db02c8fedc0d76eb Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 18:08:41 +0300 Subject: [PATCH 07/12] chore(ci): extract remaining nightly e2e scripts Move large nightly E2E shell blocks into reusable scripts so the workflow stays focused on orchestration and the scripts can be shellchecked directly. --- .../bash/e2e/cleanup-nightly-resources.sh | 85 ++++++ .github/scripts/bash/e2e/common.sh | 2 + .github/scripts/bash/e2e/configure-csi-nfs.sh | 77 ++++++ .../bash/e2e/configure-sds-replicated.sh | 53 ++++ .../scripts/bash/e2e/detect-k8s-version.sh | 52 ++++ .../bash/e2e/render-dvp-static-values.sh | 53 ++++ .../e2e-nightly-reusable-pipeline.yml | 242 ++---------------- .github/workflows/e2e-nightly.yml | 67 +---- test/dvp-static-cluster/values.yaml.tmpl | 50 ++++ test/e2e/Taskfile.yaml | 9 + test/e2e/scripts/e2e-ci.sh | 57 +++++ 11 files changed, 465 insertions(+), 282 deletions(-) create mode 100644 .github/scripts/bash/e2e/cleanup-nightly-resources.sh create mode 100644 .github/scripts/bash/e2e/configure-csi-nfs.sh create mode 100644 .github/scripts/bash/e2e/configure-sds-replicated.sh create mode 100644 .github/scripts/bash/e2e/detect-k8s-version.sh create mode 100644 .github/scripts/bash/e2e/render-dvp-static-values.sh create mode 100644 test/dvp-static-cluster/values.yaml.tmpl create mode 100644 test/e2e/scripts/e2e-ci.sh diff --git a/.github/scripts/bash/e2e/cleanup-nightly-resources.sh b/.github/scripts/bash/e2e/cleanup-nightly-resources.sh new file mode 100644 index 0000000000..60ba28546b --- /dev/null +++ b/.github/scripts/bash/e2e/cleanup-nightly-resources.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +LABEL_SELECTOR="${LABEL_SELECTOR:-test=nightly-e2e}" +KEEP_HOURS="${KEEP_HOURS:-47}" +FRIDAY_KEEP_HOURS="${FRIDAY_KEEP_HOURS:-71}" + +current_date_seconds="$(date -u +%s)" + +collect_items_json() { + local resource="$1" + + kubectl get "${resource}" -l "${LABEL_SELECTOR}" -o json \ + | jq -c '.items[] | {name: .metadata.name, created_at: .metadata.creationTimestamp}' +} + +should_keep() { + local created_at="$1" + local resource_created_at_seconds + local age_seconds + local weekday_of_day + + resource_created_at_seconds="$(date -d "${created_at}" -u +%s)" + age_seconds="$(( current_date_seconds - resource_created_at_seconds ))" + weekday_of_day="$(date -d "${created_at}" -u +%u)" + + if [ "${age_seconds}" -lt "$(( KEEP_HOURS * 3600 ))" ]; then + echo "keep" + return 0 + fi + + if [ "${weekday_of_day}" -eq 5 ] && [ "${age_seconds}" -lt "$(( FRIDAY_KEEP_HOURS * 3600 ))" ]; then + echo "keep" + return 0 + fi + + echo "delete" +} + +cleanup_kind() { + local kind="$1" + local item + local name + local created_at + local decision + + echo "[INFO] Process ${kind} with label ${LABEL_SELECTOR}" + collect_items_json "${kind}" | while read -r item; do + name="$(echo "${item}" | jq -r '.name')" + created_at="$(echo "${item}" | jq -r '.created_at')" + [ -z "${name}" ] && continue + + decision="$(should_keep "${created_at}")" + if [ "${decision}" = "keep" ]; then + printf "%-63s %22s\n" "[INFO] Keep ${kind}/${name}:" "created_at ${created_at}" + continue + fi + + printf "%-63s %22s\n" "[INFO] Delete ${kind}/${name}:" "created_at ${created_at}" + kubectl delete "${kind}" "${name}" --timeout=300s || true + done || true +} + +cleanup_kind "namespaces" +echo " " +cleanup_kind "vmclass" diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh index 24899f23e8..9fbfc0186a 100644 --- a/.github/scripts/bash/e2e/common.sh +++ b/.github/scripts/bash/e2e/common.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -Eeuo pipefail + on_error() { local exit_code=$? echo "[ERROR] Command failed with exit code ${exit_code} at line ${BASH_LINENO[0]}: ${BASH_COMMAND}" >&2 diff --git a/.github/scripts/bash/e2e/configure-csi-nfs.sh b/.github/scripts/bash/e2e/configure-csi-nfs.sh new file mode 100644 index 0000000000..8475973e68 --- /dev/null +++ b/.github/scripts/bash/e2e/configure-csi-nfs.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +nfs_ready() { + local count=90 + local controller + local csi_controller + local csi_node_desired + local csi_node_ready + + for i in $(seq 1 "${count}"); do + echo "[INFO] Check d8-csi-nfs pods (attempt ${i}/${count})" + controller="$(kubectl -n d8-csi-nfs get deploy controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")" + csi_controller="$(kubectl -n d8-csi-nfs get deploy csi-controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")" + csi_node_desired="$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0")" + csi_node_ready="$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0")" + + if [[ "${controller}" -ge 1 && "${csi_controller}" -ge 1 && "${csi_node_desired}" -gt 0 && "${csi_node_ready}" -eq "${csi_node_desired}" ]]; then + echo "[SUCCESS] NFS CSI is ready (controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired})" + return 0 + fi + + echo "[WARNING] NFS CSI not ready: controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired}" + if (( i % 5 == 0 )); then + echo "[DEBUG] Pods in d8-csi-nfs:" + kubectl -n d8-csi-nfs get pods || echo "[WARNING] Failed to retrieve pods" + echo "[DEBUG] Deployments in d8-csi-nfs:" + kubectl -n d8-csi-nfs get deploy || echo "[WARNING] Failed to retrieve deployments" + echo "[DEBUG] DaemonSets in d8-csi-nfs:" + kubectl -n d8-csi-nfs get ds || echo "[WARNING] Failed to retrieve daemonsets" + echo "[DEBUG] csi-nfs module status:" + kubectl get modules csi-nfs -o wide || echo "[WARNING] Failed to retrieve module" + fi + sleep 10 + done + + echo "[ERROR] NFS CSI did not become ready in time" + kubectl -n d8-csi-nfs get pods || true + exit 1 +} + +echo "[INFO] Apply csi-nfs ModuleConfig, ModulePullOverride, snapshot-controller" +kubectl apply -f mc.yaml + +echo "[INFO] Wait for csi-nfs module to be ready" +kubectl wait --for=jsonpath='{.status.phase}'=Ready modules csi-nfs --timeout=300s + +echo "[INFO] Wait for csi-nfs pods to be ready" +nfs_ready + +echo "[INFO] Apply NFSStorageClass" +envsubst < storageclass.yaml | kubectl apply -f - + +echo "[INFO] Configure default storage class" +./default-sc-configure.sh + +echo "[INFO] Show existing storageclasses" +kubectl get storageclass diff --git a/.github/scripts/bash/e2e/configure-sds-replicated.sh b/.github/scripts/bash/e2e/configure-sds-replicated.sh new file mode 100644 index 0000000000..deec4479fc --- /dev/null +++ b/.github/scripts/bash/e2e/configure-sds-replicated.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/wait-sds-replicated.sh +source "${SCRIPT_DIR}/wait-sds-replicated.sh" + +d8_queue + +kubectl apply -f ../sds-node-configurator/mc.yaml +kubectl apply -f mc.yaml + +echo "[INFO] Wait for sds-node-configurator" +kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s + +echo "[INFO] Wait for sds-replicated-volume to be ready" +sds_replicated_ready +kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-replicated-volume --timeout=300s + +echo "[INFO] Wait BlockDevice are ready" +blockdevices_ready + +echo "[INFO] Wait pods and webhooks sds-replicated pods" +sds_pods_ready + +chmod +x ../sds-node-configurator/lvg-gen.sh +../sds-node-configurator/lvg-gen.sh + +chmod +x rsc-gen.sh +./rsc-gen.sh + +echo "[INFO] Show existing storageclasses" +if ! kubectl get storageclass | grep -q nested; then + echo "[WARNING] No nested storageclasses" +else + kubectl get storageclass | grep nested + echo "[SUCCESS] Done" +fi diff --git a/.github/scripts/bash/e2e/detect-k8s-version.sh b/.github/scripts/bash/e2e/detect-k8s-version.sh new file mode 100644 index 0000000000..e711ff5f75 --- /dev/null +++ b/.github/scripts/bash/e2e/detect-k8s-version.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +require_env GITHUB_OUTPUT +: "${GITHUB_OUTPUT:?}" + +version_json="$(kubectl version -o json)" +server_version="$(echo "${version_json}" | jq -r '.serverVersion.gitVersion')" +server_major="$(echo "${version_json}" | jq -r '.serverVersion.major' | tr -cd '0-9')" +server_minor="$(echo "${version_json}" | jq -r '.serverVersion.minor' | tr -cd '0-9')" + +if [[ -z "${server_major}" || -z "${server_minor}" ]]; then + echo "[ERROR] Failed to parse Kubernetes server version: ${server_version}" + exit 1 +fi + +label_filter="" +usb_supported=false + +if (( server_major > 1 || (server_major == 1 && server_minor >= 34) )); then + usb_supported=true + echo "[INFO] Kubernetes server version ${server_version} supports USB E2E tests" +else + label_filter="!usb-precheck" + echo "[INFO] Kubernetes server version ${server_version} does not support USB E2E tests" + echo "[INFO] USB-labeled specs will be excluded with label filter: ${label_filter}" +fi + +{ + echo "server-version=${server_version}" + echo "usb-supported=${usb_supported}" + echo "label-filter=${label_filter}" +} >> "${GITHUB_OUTPUT}" diff --git a/.github/scripts/bash/e2e/render-dvp-static-values.sh b/.github/scripts/bash/e2e/render-dvp-static-values.sh new file mode 100644 index 0000000000..90795601e9 --- /dev/null +++ b/.github/scripts/bash/e2e/render-dvp-static-values.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +require_env NAMESPACE +require_env STORAGE_TYPE +require_env DECKHOUSE_CHANNEL +require_env POD_SUBNET_CIDR +require_env SERVICE_SUBNET_CIDR +require_env K8S_VERSION +require_env PROD_IO_REGISTRY_DOCKER_CFG +require_env VIRTUALIZATION_IMAGE_URL +require_env DEFAULT_USER +require_env APT_MIRROR_ENABLED +require_env APT_MIRROR_NAME +require_env APT_MIRROR_URL +require_env CLUSTER_CONFIG_WORKERS_MEMORY +require_env NESTED_CLUSTER_NETWORK_NAME +require_env DEV_REGISTRY_DOCKER_CFG +: "${DEV_REGISTRY_DOCKER_CFG:?}" + +default_storage_class="$(kubectl get storageclass -o json \ + | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class" == "true") | .metadata.name')" + +export DEFAULT_STORAGE_CLASS="${default_storage_class}" +envsubst < values.yaml.tmpl > values.yaml + +mkdir -p tmp +touch tmp/discovered-values.yaml + +registry="$(base64 -d <<< "${DEV_REGISTRY_DOCKER_CFG}" | jq '.auths | to_entries | .[] | .key' -r)" +auth="$(base64 -d <<< "${DEV_REGISTRY_DOCKER_CFG}" | jq '.auths | to_entries | .[] | .value.auth' -r)" + +REGISTRY="${registry}" yq eval --inplace '.discovered.registry_url = env(REGISTRY)' tmp/discovered-values.yaml +AUTH="${auth}" yq eval --inplace '.discovered.registry_auth = env(AUTH)' tmp/discovered-values.yaml diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index 2247bb235f..16b8a44b91 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -229,71 +229,20 @@ jobs: - name: Generate values.yaml working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} - run: | - defaultStorageClass=$(kubectl get storageclass -o json \ - | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class" == "true") | .metadata.name') - - cat < values.yaml - namespace: ${{ steps.vars.outputs.namespace }} - storageType: ${{ inputs.storage_type }} - storageClass: ${defaultStorageClass} - sa: dkp-sa - deckhouse: - channel: ${{ env.DECKHOUSE_CHANNEL }} - podSubnetCIDR: ${{ inputs.pod_subnet_cidr }} - serviceSubnetCIDR: ${{ inputs.service_subnet_cidr }} - kubernetesVersion: ${{ env.K8S_VERSION }} - registryDockerCfg: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} - bundle: Default - proxyEnabled: false - image: - url: ${{ inputs.virtualization_image_url }} - defaultUser: ${{ env.DEFAULT_USER }} - bootloader: BIOS - ingressHosts: - - api - - grafana - - dex - - prometheus - - console - - virtualization - instances: - aptMirror: - enabled: ${{ inputs.apt_mirror_enabled }} - name: ${{ inputs.apt_mirror_name }} - url: ${{ inputs.apt_mirror_url }} - masterNodes: - count: 1 - cfg: - rootDiskSize: 60Gi - cpu: - cores: 4 - coreFraction: 50% - memory: - size: 12Gi - additionalNodes: - - name: worker - count: 3 - cfg: - cpu: - cores: 6 - coreFraction: 50% - memory: - size: ${{ inputs.cluster_config_workers_memory }} - additionalDisks: - - size: 50Gi - networkConfig: - clusterNetworkName: ${{ inputs.nested_cluster_network_name }} - EOF - - mkdir -p tmp - touch tmp/discovered-values.yaml - - export REGISTRY=$(base64 -d <<< ${{secrets.DEV_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) - export AUTH=$(base64 -d <<< ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r) - - yq eval --inplace '.discovered.registry_url = env(REGISTRY)' tmp/discovered-values.yaml - yq eval --inplace '.discovered.registry_auth = env(AUTH)' tmp/discovered-values.yaml + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + STORAGE_TYPE: ${{ inputs.storage_type }} + POD_SUBNET_CIDR: ${{ inputs.pod_subnet_cidr }} + SERVICE_SUBNET_CIDR: ${{ inputs.service_subnet_cidr }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRTUALIZATION_IMAGE_URL: ${{ inputs.virtualization_image_url }} + APT_MIRROR_ENABLED: ${{ inputs.apt_mirror_enabled }} + APT_MIRROR_NAME: ${{ inputs.apt_mirror_name }} + APT_MIRROR_URL: ${{ inputs.apt_mirror_url }} + CLUSTER_CONFIG_WORKERS_MEMORY: ${{ inputs.cluster_config_workers_memory }} + NESTED_CLUSTER_NETWORK_NAME: ${{ inputs.nested_cluster_network_name }} + run: bash ${{ github.workspace }}/.github/scripts/bash/e2e/render-dvp-static-values.sh - name: Bootstrap cluster [infra-deploy] working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} @@ -506,7 +455,6 @@ jobs: checkout: "false" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check nested kube-api via generated kubeconfig uses: ./.github/actions/use-nested-kubeconfig with: @@ -624,7 +572,6 @@ jobs: checkout: "false" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check nested kube-api via generated kubeconfig uses: ./.github/actions/use-nested-kubeconfig with: @@ -633,39 +580,7 @@ jobs: id: storage-replicated-setup if: ${{ inputs.storage_type == 'replicated' }} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated - run: | - source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/wait-sds-replicated.sh" - - d8_queue - - kubectl apply -f ../sds-node-configurator/mc.yaml - kubectl apply -f mc.yaml - echo "[INFO] Wait for sds-node-configurator" - kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s - - echo "[INFO] Wait for sds-replicated-volume to be ready" - sds_replicated_ready - kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-replicated-volume --timeout=300s - - echo "[INFO] Wait BlockDevice are ready" - blockdevices_ready - - echo "[INFO] Wait pods and webhooks sds-replicated pods" - sds_pods_ready - - chmod +x ../sds-node-configurator/lvg-gen.sh - ../sds-node-configurator/lvg-gen.sh - - chmod +x rsc-gen.sh - ./rsc-gen.sh - - echo "[INFO] Show existing storageclasses" - if ! kubectl get storageclass | grep -q nested; then - echo "[WARNING] No nested storageclasses" - else - kubectl get storageclass | grep nested - echo "[SUCCESS] Done" - fi + run: bash ${{ github.workspace }}/.github/scripts/bash/e2e/configure-sds-replicated.sh - name: Configure NFS storage if: ${{ inputs.storage_type == 'nfs' }} @@ -673,63 +588,7 @@ jobs: working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs env: NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} - run: | - nfs_ready() { - local count=90 - local controller - local csi_controller - local csi_node_desired - local csi_node_ready - - for i in $(seq 1 $count); do - echo "[INFO] Check d8-csi-nfs pods (attempt ${i}/${count})" - controller=$(kubectl -n d8-csi-nfs get deploy controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") - csi_controller=$(kubectl -n d8-csi-nfs get deploy csi-controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") - csi_node_desired=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") - csi_node_ready=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") - - if [[ "$controller" -ge 1 && "$csi_controller" -ge 1 && "$csi_node_desired" -gt 0 && "$csi_node_ready" -eq "$csi_node_desired" ]]; then - echo "[SUCCESS] NFS CSI is ready (controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired})" - return 0 - fi - - echo "[WARNING] NFS CSI not ready: controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired}" - if (( i % 5 == 0 )); then - echo "[DEBUG] Pods in d8-csi-nfs:" - kubectl -n d8-csi-nfs get pods || echo "[WARNING] Failed to retrieve pods" - echo "[DEBUG] Deployments in d8-csi-nfs:" - kubectl -n d8-csi-nfs get deploy || echo "[WARNING] Failed to retrieve deployments" - echo "[DEBUG] DaemonSets in d8-csi-nfs:" - kubectl -n d8-csi-nfs get ds || echo "[WARNING] Failed to retrieve daemonsets" - echo "[DEBUG] csi-nfs module status:" - kubectl get modules csi-nfs -o wide || echo "[WARNING] Failed to retrieve module" - fi - sleep 10 - done - - echo "[ERROR] NFS CSI did not become ready in time" - kubectl -n d8-csi-nfs get pods || true - exit 1 - } - - echo "[INFO] Apply csi-nfs ModuleConfig, ModulePullOverride, snapshot-controller" - kubectl apply -f mc.yaml - - echo "[INFO] Wait for csi-nfs module to be ready" - kubectl wait --for=jsonpath='{.status.phase}'=Ready modules csi-nfs --timeout=300s - - echo "[INFO] Wait for csi-nfs pods to be ready" - nfs_ready - - echo "[INFO] Apply NFSStorageClass" - envsubst < storageclass.yaml | kubectl apply -f - - - echo "[INFO] Configure default storage class" - ./default-sc-configure.sh - - - echo "[INFO] Show existing storageclasses" - kubectl get storageclass + run: bash ${{ github.workspace }}/.github/scripts/bash/e2e/configure-csi-nfs.sh configure-virtualization: name: Configure Virtualization @@ -827,34 +686,7 @@ jobs: - name: Detect Kubernetes version for E2E id: detect-k8s-version - run: | - set -euo pipefail - - VERSION_JSON=$(kubectl version -o json) - SERVER_VERSION=$(echo "$VERSION_JSON" | jq -r '.serverVersion.gitVersion') - SERVER_MAJOR=$(echo "$VERSION_JSON" | jq -r '.serverVersion.major' | tr -cd '0-9') - SERVER_MINOR=$(echo "$VERSION_JSON" | jq -r '.serverVersion.minor' | tr -cd '0-9') - - if [[ -z "$SERVER_MAJOR" || -z "$SERVER_MINOR" ]]; then - echo "[ERROR] Failed to parse Kubernetes server version: $SERVER_VERSION" - exit 1 - fi - - LABEL_FILTER="" - USB_SUPPORTED=false - - if (( SERVER_MAJOR > 1 || (SERVER_MAJOR == 1 && SERVER_MINOR >= 34) )); then - USB_SUPPORTED=true - echo "[INFO] Kubernetes server version $SERVER_VERSION supports USB E2E tests" - else - LABEL_FILTER="!usb-precheck" - echo "[INFO] Kubernetes server version $SERVER_VERSION does not support USB E2E tests" - echo "[INFO] USB-labeled specs will be excluded with label filter: $LABEL_FILTER" - fi - - echo "server-version=$SERVER_VERSION" >> "$GITHUB_OUTPUT" - echo "usb-supported=$USB_SUPPORTED" >> "$GITHUB_OUTPUT" - echo "label-filter=$LABEL_FILTER" >> "$GITHUB_OUTPUT" + run: bash .github/scripts/bash/e2e/detect-k8s-version.sh - name: Create vmclass for e2e tests run: | @@ -874,45 +706,9 @@ jobs: LABELS: ${{ steps.detect-k8s-version.outputs.label-filter }} SERVER_K8S_VERSION: ${{ steps.detect-k8s-version.outputs.server-version }} USB_SUPPORTED: ${{ steps.detect-k8s-version.outputs.usb-supported }} + FOCUS: ${{ inputs.e2e_focus_tests }} working-directory: ./test/e2e/ - run: | - DATE=$(date +"%Y-%m-%d") - e2e_report_file="e2e_report_${CSI}_${DATE}.json" - e2e_output_file="e2e_output_${CSI}_${DATE}.log" - FOCUS="${{ inputs.e2e_focus_tests }}" - - cp -a legacy/testdata /tmp/testdata - - echo "[INFO] Kubernetes server version: ${SERVER_K8S_VERSION}" - echo "[INFO] USB E2E supported: ${USB_SUPPORTED}" - if [ -n "${LABELS:-}" ]; then - echo "[INFO] Applying Ginkgo label filter: ${LABELS}" - fi - - ./scripts/precheck-prepare_ci.sh - - set +e - GINKGO_ARGS=( - -v - --race - --timeout="$TIMEOUT" - --json-report="$e2e_report_file" - ) - - if [ -n "${LABELS:-}" ]; then - GINKGO_ARGS+=(--label-filter="$LABELS") - fi - - if [ -n "$FOCUS" ]; then - GINKGO_ARGS+=(--focus="$FOCUS") - fi - - go tool ginkgo "${GINKGO_ARGS[@]}" . 2>&1 | tee "$e2e_output_file" - GINKGO_EXIT_CODE=${PIPESTATUS[0]} - set -e - - echo "[INFO] Exit code: $GINKGO_EXIT_CODE" - exit $GINKGO_EXIT_CODE + run: task e2e:ci - name: Upload summary test results (json) uses: actions/upload-artifact@v7 diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 018ef12046..54f58fd304 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -30,6 +30,9 @@ jobs: name: Cleanup nested clusters runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 with: @@ -37,65 +40,11 @@ jobs: context: e2e-cluster-nightly-e2e-virt-sa kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} - name: Delete nested clusters - run: | - current_date_seconds="$(date -u +%s)" - FORMAT="%-63s %22s\n" - - # 47h = ~2 days with CI delay; 71h = ~3 days for Friday clusters - KEEP_HOURS=47 - FRIDAY_KEEP_HOURS=71 - - collect_items_json() { - local resource="$1" - kubectl get "${resource}" -l test=nightly-e2e -o json \ - | jq -c '.items[] | {name: .metadata.name, created_at: .metadata.creationTimestamp}' - } - - should_keep() { - local created_at="$1" - local recourse_created_at_seconds age_seconds weekday_of_day - - recourse_created_at_seconds="$(date -d "${created_at}" -u +%s)" - age_seconds="$(( current_date_seconds - recourse_created_at_seconds ))" - weekday_of_day="$(date -d "${created_at}" -u +%u)" - - if [ "${age_seconds}" -lt "$(( KEEP_HOURS * 3600 ))" ]; then - echo "keep" - return 0 - fi - - if [ "${weekday_of_day}" -eq 5 ] && [ "${age_seconds}" -lt "$(( FRIDAY_KEEP_HOURS * 3600 ))" ]; then - echo "keep" - return 0 - fi - - echo "delete" - return 0 - } - - cleanup_kind() { - local kind="$1" - local item name created_at decision parsed - - echo "[INFO] Process ${kind} with label test=nightly-e2e" - collect_items_json "${kind}" | while read -r item; do - name=$(echo $item | jq -r '.name') - created_at=$(echo $item | jq -r '.created_at') - [ -z "${name}" ] && continue - - decision="$(should_keep "${created_at}")" - if [ "${decision}" = "keep" ]; then - printf "$FORMAT" "[INFO] Keep ${kind}/${name}:" "created_at ${created_at}" - continue - fi - printf "$FORMAT" "[INFO] Delete ${kind}/${name}:" "created_at ${created_at}" - kubectl delete "${kind}" "${name}" --timeout=300s || true - done || true - } - - cleanup_kind "namespaces" - echo " " - cleanup_kind "vmclass" + env: + LABEL_SELECTOR: test=nightly-e2e + KEEP_HOURS: "47" + FRIDAY_KEEP_HOURS: "71" + run: bash .github/scripts/bash/e2e/cleanup-nightly-resources.sh power-off-vms-for-nested: name: Power off VMs for nested clusters diff --git a/test/dvp-static-cluster/values.yaml.tmpl b/test/dvp-static-cluster/values.yaml.tmpl new file mode 100644 index 0000000000..17cc5bd0fe --- /dev/null +++ b/test/dvp-static-cluster/values.yaml.tmpl @@ -0,0 +1,50 @@ +namespace: ${NAMESPACE} +storageType: ${STORAGE_TYPE} +storageClass: ${DEFAULT_STORAGE_CLASS} +sa: dkp-sa +deckhouse: + channel: ${DECKHOUSE_CHANNEL} + podSubnetCIDR: ${POD_SUBNET_CIDR} + serviceSubnetCIDR: ${SERVICE_SUBNET_CIDR} + kubernetesVersion: ${K8S_VERSION} + registryDockerCfg: ${PROD_IO_REGISTRY_DOCKER_CFG} + bundle: Default + proxyEnabled: false +image: + url: ${VIRTUALIZATION_IMAGE_URL} + defaultUser: ${DEFAULT_USER} + bootloader: BIOS +ingressHosts: + - api + - grafana + - dex + - prometheus + - console + - virtualization +instances: + aptMirror: + enabled: ${APT_MIRROR_ENABLED} + name: ${APT_MIRROR_NAME} + url: ${APT_MIRROR_URL} + masterNodes: + count: 1 + cfg: + rootDiskSize: 60Gi + cpu: + cores: 4 + coreFraction: 50% + memory: + size: 12Gi + additionalNodes: + - name: worker + count: 3 + cfg: + cpu: + cores: 6 + coreFraction: 50% + memory: + size: ${CLUSTER_CONFIG_WORKERS_MEMORY} + additionalDisks: + - size: 50Gi + networkConfig: + clusterNetworkName: ${NESTED_CLUSTER_NETWORK_NAME} diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index ba503e6e96..1ae425d0c1 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -44,6 +44,15 @@ tasks: cmds: - ./scripts/task_run_ci.sh + e2e:ci: + desc: "Run e2e tests in GitHub Actions" + deps: + - copy + - kubectl + - d8 + cmds: + - bash ./scripts/e2e-ci.sh + precheck:prepare: desc: "Generate JSON report via ginkgo dry-run for precheck preparation" cmds: diff --git a/test/e2e/scripts/e2e-ci.sh b/test/e2e/scripts/e2e-ci.sh new file mode 100644 index 0000000000..6a0856717a --- /dev/null +++ b/test/e2e/scripts/e2e-ci.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +TIMEOUT="${TIMEOUT:-3h}" +FOCUS="${FOCUS:-}" +LABELS="${LABELS:-}" +CSI="${CSI:-unknown}" + +date_tag="$(date +"%Y-%m-%d")" +e2e_report_file="e2e_report_${CSI}_${date_tag}.json" +e2e_output_file="e2e_output_${CSI}_${date_tag}.log" + +echo "[INFO] Kubernetes server version: ${SERVER_K8S_VERSION:-unknown}" +echo "[INFO] USB E2E supported: ${USB_SUPPORTED:-unknown}" +if [ -n "${LABELS}" ]; then + echo "[INFO] Applying Ginkgo label filter: ${LABELS}" +fi + +./scripts/precheck-prepare_ci.sh + +set +e +ginkgo_args=( + -v + --race + --timeout="${TIMEOUT}" + --json-report="${e2e_report_file}" +) + +if [ -n "${LABELS}" ]; then + ginkgo_args+=(--label-filter="${LABELS}") +fi + +if [ -n "${FOCUS}" ]; then + ginkgo_args+=(--focus="${FOCUS}") +fi + +go tool ginkgo "${ginkgo_args[@]}" . 2>&1 | tee "${e2e_output_file}" +ginkgo_exit_code="${PIPESTATUS[0]}" +set -e + +echo "[INFO] Exit code: ${ginkgo_exit_code}" +exit "${ginkgo_exit_code}" From 60af74020f9c32dbaa9b9587802eb0b0f94ec7c5 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 20:22:29 +0300 Subject: [PATCH 08/12] chore(ci): extract e2e ssh helpers Signed-off-by: Nikita Korolev --- .github/scripts/bash/e2e/d8-ssh.sh | 69 +++++++++++++++++++ .../e2e-nightly-reusable-pipeline.yml | 55 ++------------- .../e2e-test-releases-reusable-pipeline.yml | 49 ++----------- 3 files changed, 81 insertions(+), 92 deletions(-) create mode 100644 .github/scripts/bash/e2e/d8-ssh.sh diff --git a/.github/scripts/bash/e2e/d8-ssh.sh b/.github/scripts/bash/e2e/d8-ssh.sh new file mode 100644 index 0000000000..6480b455d7 --- /dev/null +++ b/.github/scripts/bash/e2e/d8-ssh.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +d8vssh() { + require_env DEFAULT_USER + require_env NAMESPACE + + local default_user="${DEFAULT_USER:-}" + local namespace="${NAMESPACE:-}" + local host + local cmd + + case "$#" in + 1) + require_env nested_master + host="${nested_master:-}" + cmd="$1" + ;; + 2) + host="$1" + cmd="$2" + ;; + *) + echo "[ERROR] Usage: d8vssh [host] command" >&2 + return 1 + ;; + esac + + d8 v ssh -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + --local-ssh-opts="-o ServerAliveInterval=15" \ + --local-ssh-opts="-o ServerAliveCountMax=8" \ + --local-ssh-opts="-o ConnectTimeout=10" \ + "${default_user}@${host}.${namespace}" \ + -c "$cmd" +} + +d8vscp() { + local source=$1 + local dest=$2 + + d8 v scp -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + "$source" "$dest" + echo "d8vscp: ${source} -> ${dest} - done" +} diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index 16b8a44b91..f32132694a 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -279,21 +279,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vssh() { - local host=$1 - local cmd=$2 - d8 v ssh -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - --local-ssh-opts="-o ServerAliveInterval=15" \ - --local-ssh-opts="-o ServerAliveCountMax=8" \ - --local-ssh-opts="-o ConnectTimeout=10" \ - ${DEFAULT_USER}@${host}.${NAMESPACE} \ - -c "$cmd" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Pods in namespace $NAMESPACE" kubectl get pods -n "${NAMESPACE}" @@ -319,31 +307,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vscp() { - local source=$1 - local dest=$2 - d8 v scp -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - "$source" "$dest" - echo "d8vscp: $source -> $dest - done" - } - - d8vssh() { - local cmd=$1 - d8 v ssh -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - --local-ssh-opts="-o ServerAliveInterval=15" \ - --local-ssh-opts="-o ServerAliveCountMax=8" \ - --local-ssh-opts="-o ConnectTimeout=10" \ - ${DEFAULT_USER}@${nested_master}.${NAMESPACE} \ - -c "$cmd" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Copy script for generating kubeconfig in nested cluster" echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" @@ -383,18 +349,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vscp() { - local source=$1 - local dest=$2 - d8 v scp -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - "$source" "$dest" - echo "d8vscp: $source -> $dest - done" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 96b7081cce..de3333b5a6 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -299,18 +299,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vssh() { - local host=$1 - local cmd=$2 - d8 v ssh -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - ${DEFAULT_USER}@${host}.${NAMESPACE} \ - -c "$cmd" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Pods in namespace $NAMESPACE" kubectl get pods -n "${NAMESPACE}" @@ -336,28 +327,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") - - d8vscp() { - local source=$1 - local dest=$2 - d8 v scp -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - "$source" "$dest" - echo "d8vscp: $source -> $dest - done" - } + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vssh() { - local cmd=$1 - d8 v ssh -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - ${DEFAULT_USER}@${nested_master}.${NAMESPACE} \ - -c "$cmd" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Copy script for generating kubeconfig in nested cluster" echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" @@ -397,18 +369,9 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} PREFIX: ${{ inputs.storage_type }} run: | - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - d8vscp() { - local source=$1 - local dest=$2 - d8 v scp -i ./tmp/ssh/cloud \ - --local-ssh=true \ - --local-ssh-opts="-o StrictHostKeyChecking=no" \ - --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ - "$source" "$dest" - echo "d8vscp: $source -> $dest - done" - } + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" From a7783341357b5623db8c1c05e373131f5d639b6e Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 28 May 2026 20:57:07 +0300 Subject: [PATCH 09/12] chore(ci): add e2e workflow lint gates Signed-off-by: Nikita Korolev --- .github/workflows/dev_validation.yaml | 64 +++++++++++++++++-- .../e2e-nightly-reusable-pipeline.yml | 42 ++++++------ .../e2e-test-releases-reusable-pipeline.yml | 41 ++++++------ .github/workflows/e2e-test-releases.yml | 4 +- Taskfile.yaml | 1 + api/scripts/update-codegen.sh | 3 +- images/virtualization-artifact/hack/args.sh | 2 +- images/virtualization-artifact/hack/dlv.sh | 1 + .../virtualization-artifact/hack/pyroscope.sh | 1 + 9 files changed, 108 insertions(+), 51 deletions(-) diff --git a/.github/workflows/dev_validation.yaml b/.github/workflows/dev_validation.yaml index 3d600131a4..b111bed50c 100644 --- a/.github/workflows/dev_validation.yaml +++ b/.github/workflows/dev_validation.yaml @@ -103,6 +103,59 @@ jobs: run: | task validation:doc-changes + shellcheck: + if: "!contains(github.event.pull_request.labels.*.name, 'validation/skip/shellcheck')" + runs-on: ubuntu-latest + name: Validation shellcheck + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Run shellcheck + uses: ludeeus/action-shellcheck@2.0.0 + with: + scandir: .github/scripts/bash/e2e + additional_files: >- + api/scripts/update-codegen.sh + images/virtualization-artifact/hack/args.sh + images/virtualization-artifact/hack/dlv.sh + images/virtualization-artifact/hack/pyroscope.sh + check_together: "yes" + severity: warning + ignore_paths: >- + vendor + images/cdi-artifact + images/virt-api/__virt + images/virt-controller/__virt + images/virt-handler/__virt + images/virt-launcher/__virt + + actionlint: + if: "!contains(github.event.pull_request.labels.*.name, 'validation/skip/actionlint')" + runs-on: ubuntu-latest + name: Validation actionlint + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Install actionlint + run: | + curl -sSfL \ + https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash \ + -o /tmp/download-actionlint.bash + bash /tmp/download-actionlint.bash 1.7.7 + + - name: Run actionlint + run: | + shellcheck_path="$(command -v shellcheck)" + ./actionlint -color -shellcheck="${shellcheck_path}" \ + .github/workflows/dev_validation.yaml \ + .github/workflows/e2e*.yml + # Run helm templates validation on changes in related files and without the skip labels. helm_templates: needs: paths_filter @@ -146,9 +199,9 @@ jobs: run: | # Only run vm-route-forge if relevant files were changed if [[ "${{ needs.paths_filter.outputs.vm_route_forge }}" == "true" ]]; then - echo "route_forge_skip=false" >> $GITHUB_ENV + echo "route_forge_skip=false" >> "$GITHUB_ENV" else - echo "route_forge_skip=true" >> $GITHUB_ENV + echo "route_forge_skip=true" >> "$GITHUB_ENV" fi - name: Setup Go ${{ matrix.components.go-version }} @@ -171,7 +224,7 @@ jobs: export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y -qq \ - llvm linux-headers-$(uname -r) clang \ + llvm "linux-headers-$(uname -r)" clang \ libbpf-dev uuid-runtime \ gcc-multilib yq sudo apt-get clean @@ -215,8 +268,9 @@ jobs: ;; api) cd ./api - export GOPATH=$(go env GOPATH) - echo "GOPATH=$GOPATH" >> $GITHUB_ENV + GOPATH="$(go env GOPATH)" + export GOPATH + echo "GOPATH=$GOPATH" >> "$GITHUB_ENV" echo "Installing k8s tools..." go install tool diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index f32132694a..dc3429c86d 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -190,11 +190,11 @@ jobs: namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" - echo "namespace=$namespace" >> $GITHUB_OUTPUT - echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT + echo "namespace=$namespace" >> "$GITHUB_OUTPUT" + echo "sha_short=$GIT_SHORT_HASH" >> "$GITHUB_OUTPUT" - REGISTRY=$(base64 -d <<< ${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) - echo "registry=$REGISTRY" >> $GITHUB_OUTPUT + REGISTRY=$(base64 -d <<< "${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) + echo "registry=$REGISTRY" >> "$GITHUB_OUTPUT" - name: Install htpasswd utility run: | @@ -271,7 +271,7 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} if: always() && steps.dhctl-bootstrap.outcome == 'success' run: | - kubectl -n $NAMESPACE create secret generic ssh-key --from-file=${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud + kubectl -n "$NAMESPACE" create secret generic ssh-key --from-file="${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud" - name: Get info about nested cluster and master VM working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} @@ -281,7 +281,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Pods in namespace $NAMESPACE" kubectl get pods -n "${NAMESPACE}" @@ -309,7 +309,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Copy script for generating kubeconfig in nested cluster" echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" @@ -335,13 +335,13 @@ jobs: echo "[INFO] Set rights for kubeconfig" echo "[INFO] sudo chown 1001:1001 ${kubeConfigPath}" - sudo chown 1001:1001 ${kubeConfigPath} + sudo chown 1001:1001 "${kubeConfigPath}" echo " " echo "[INFO] Kubeconf to github output" - CONFIG=$(cat ${kubeConfigPath} | base64 -w 0) - CONFIG=$(echo $CONFIG | base64 -w 0) - echo "kubeconfig=$CONFIG" >> $GITHUB_OUTPUT + CONFIG=$(base64 -w 0 < "${kubeConfigPath}") + CONFIG=$(echo "${CONFIG}" | base64 -w 0) + echo "kubeconfig=$CONFIG" >> "$GITHUB_OUTPUT" - name: cloud-init logs if: steps.dhctl-bootstrap.outcome == 'failure' @@ -351,7 +351,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" @@ -428,7 +428,7 @@ jobs: success=false wait_time_seconds=5 - for i in $(seq 1 $count); do + for i in $(seq 1 "$count"); do nodes=$(kubectl get nodes -o name | wc -l) actual=$(kubectl get nodenetworkinterfaces -o json | jq -r '.items[] | select(.status.operationalState == "Up") | .metadata.name' | wc -l) || true expected=$((nodes * 2)) @@ -452,10 +452,10 @@ jobs: echo ::endgroup:: echo "[INFO] Retrying in 10 seconds..." - sleep $wait_time_seconds - elif [ $i -lt $count ]; then + sleep "$wait_time_seconds" + elif [ "$i" -lt "$count" ]; then echo "[INFO] Retrying in 10 seconds..." - sleep $wait_time_seconds + sleep "$wait_time_seconds" fi done @@ -474,7 +474,7 @@ jobs: for nic in $extraNic; do echo "[INFO] Label nodenetworkinterface $nic nic-group=extra" - kubectl label nodenetworkinterfaces $nic nic-group=extra + kubectl label nodenetworkinterfaces "$nic" nic-group=extra done kubectl get nodenetworkinterface -l nic-group=extra @@ -746,7 +746,7 @@ jobs: id: set-artifact-name run: | ARTIFACT_NAME="e2e-report-${{ inputs.storage_type }}-${{ github.run_id }}-${{ inputs.date_start }}" - echo "artifact-name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "artifact-name=$ARTIFACT_NAME" >> "$GITHUB_OUTPUT" echo "[INFO] Artifact name: $ARTIFACT_NAME" undeploy-cluster: @@ -779,12 +779,12 @@ jobs: GPG_PASSPHRASE: ${{ secrets.E2E_ARTIFACTS_GPG_PASSPHRASE }} ARTIFACT_NAME: ${{ env.STORAGE_TYPE }}-generated-files-${{ env.E2E_START_TIME }} run: | - artifact_path=${{ runner.temp }}/encrypted-generated-files/${ARTIFACT_NAME}.zip.gpg + artifact_path="${{ runner.temp }}/encrypted-generated-files/${ARTIFACT_NAME}.zip.gpg" gpg --decrypt --batch --yes --pinentry-mode loopback \ --passphrase "$GPG_PASSPHRASE" \ - --output $RUNNER_TEMP/${ARTIFACT_NAME}.zip \ + --output "${RUNNER_TEMP}/${ARTIFACT_NAME}.zip" \ "$artifact_path" - unzip -o $RUNNER_TEMP/${ARTIFACT_NAME}.zip -d ${{ env.SETUP_CLUSTER_TYPE_PATH }} + unzip -o "${RUNNER_TEMP}/${ARTIFACT_NAME}.zip" -d "${{ env.SETUP_CLUSTER_TYPE_PATH }}" - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index de3333b5a6..9db8257ccc 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -153,11 +153,11 @@ jobs: namespace="release-test-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" - echo "namespace=$namespace" >> $GITHUB_OUTPUT - echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT + echo "namespace=$namespace" >> "$GITHUB_OUTPUT" + echo "sha_short=$GIT_SHORT_HASH" >> "$GITHUB_OUTPUT" - REGISTRY=$(base64 -d <<< ${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) - echo "registry=$REGISTRY" >> $GITHUB_OUTPUT + REGISTRY=$(base64 -d <<< "${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) + echo "registry=$REGISTRY" >> "$GITHUB_OUTPUT" - name: Install htpasswd utility run: | @@ -254,8 +254,9 @@ jobs: mkdir -p tmp touch tmp/discovered-values.yaml - export REGISTRY=$(base64 -d <<< ${{secrets.DEV_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) - export AUTH=$(base64 -d <<< ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r) + REGISTRY=$(base64 -d <<< "${{secrets.DEV_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) + AUTH=$(base64 -d <<< "${{ secrets.DEV_REGISTRY_DOCKER_CFG }}" | jq '.auths | to_entries | .[] | .value.auth' -r) + export REGISTRY AUTH yq eval --inplace '.discovered.registry_url = env(REGISTRY)' tmp/discovered-values.yaml yq eval --inplace '.discovered.registry_auth = env(AUTH)' tmp/discovered-values.yaml @@ -279,8 +280,8 @@ jobs: timeout-minutes: 60 - name: Label cluster to prevent deletion run: | - kubectl label namespace ${{ steps.vars.outputs.namespace }} e2e-cluster/do-not-stop-vm-on-e2e-run=true - kubectl label vmclass ${{ steps.vars.outputs.namespace }}-cpu e2e-cluster/do-not-stop-vm-on-e2e-run=true + kubectl label namespace "${{ steps.vars.outputs.namespace }}" e2e-cluster/do-not-stop-vm-on-e2e-run=true + kubectl label vmclass "${{ steps.vars.outputs.namespace }}-cpu" e2e-cluster/do-not-stop-vm-on-e2e-run=true - name: Bootstrap cluster [show-connection-info] working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} run: | @@ -291,7 +292,7 @@ jobs: NAMESPACE: ${{ steps.vars.outputs.namespace }} if: always() && steps.dhctl-bootstrap.outcome == 'success' run: | - kubectl -n $NAMESPACE create secret generic ssh-key --from-file=${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud + kubectl -n "$NAMESPACE" create secret generic ssh-key --from-file="${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud" - name: Get info about nested cluster and master VM working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} @@ -301,7 +302,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Pods in namespace $NAMESPACE" kubectl get pods -n "${NAMESPACE}" @@ -329,7 +330,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") echo "[INFO] Copy script for generating kubeconfig in nested cluster" echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" @@ -355,13 +356,13 @@ jobs: echo "[INFO] Set rights for kubeconfig" echo "[INFO] sudo chown 1001:1001 ${kubeConfigPath}" - sudo chown 1001:1001 ${kubeConfigPath} + sudo chown 1001:1001 "${kubeConfigPath}" echo " " echo "[INFO] Kubeconf to github output" CONFIG=$(cat "${kubeConfigPath}" | base64 -w 0) CONFIG=$(echo "${CONFIG}" | base64 -w 0) - echo "kubeconfig=$CONFIG" >> $GITHUB_OUTPUT + echo "kubeconfig=$CONFIG" >> "$GITHUB_OUTPUT" - name: cloud-init logs if: steps.dhctl-bootstrap.outcome == 'failure' @@ -371,7 +372,7 @@ jobs: run: | source "${GITHUB_WORKSPACE}/.github/scripts/bash/e2e/d8-ssh.sh" - nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + nested_master=$(kubectl -n "${NAMESPACE}" get vm -l "group=${PREFIX}-master" -o jsonpath="{.items[0].metadata.name}") d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" @@ -432,7 +433,6 @@ jobs: checkout: "false" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check nested kube-api via generated kubeconfig uses: ./.github/actions/use-nested-kubeconfig with: @@ -455,7 +455,7 @@ jobs: success=false wait_time_seconds=5 - for i in $(seq 1 $count); do + for i in $(seq 1 "$count"); do nodes=$(kubectl get nodes -o name | wc -l) actual=$(kubectl get nodenetworkinterfaces -o json | jq -r '.items[] | select(.status.operationalState == "Up") | .metadata.name' | wc -l) || true expected=$((nodes * 2)) @@ -479,10 +479,10 @@ jobs: echo "::endgroup::" echo "[INFO] Retrying in 10 seconds..." - sleep $wait_time_seconds - elif [ $i -lt $count ]; then + sleep "$wait_time_seconds" + elif [ "$i" -lt "$count" ]; then echo "[INFO] Retrying in 10 seconds..." - sleep $wait_time_seconds + sleep "$wait_time_seconds" fi done @@ -501,7 +501,7 @@ jobs: for nic in $extraNic; do echo "[INFO] Label nodenetworkinterface $nic nic-group=extra" - kubectl label nodenetworkinterfaces $nic nic-group=extra + kubectl label nodenetworkinterfaces "$nic" nic-group=extra done kubectl get nodenetworkinterface -l nic-group=extra @@ -554,7 +554,6 @@ jobs: checkout: "false" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check nested kube-api via generated kubeconfig uses: ./.github/actions/use-nested-kubeconfig with: diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 1e49a9664d..41e903a948 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -167,7 +167,7 @@ jobs: steps: - name: Setup Docker config run: | - echo "DOCKER_CONFIG=$(mktemp -d)" >> $GITHUB_ENV + echo "DOCKER_CONFIG=$(mktemp -d)" >> "$GITHUB_ENV" - name: Print vars run: | @@ -176,7 +176,7 @@ jobs: echo MODULES_MODULE_NAME=${{ vars.MODULE_NAME }} echo MODULES_MODULE_TAG=${{ matrix.module_tag }} echo CHECKOUT_REF=${{ matrix.checkout_ref }} - echo DOCKER_CONFIG=$DOCKER_CONFIG + echo "DOCKER_CONFIG=$DOCKER_CONFIG" - uses: actions/checkout@v4 with: diff --git a/Taskfile.yaml b/Taskfile.yaml index 3090c30af9..9b1611c071 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -176,6 +176,7 @@ tasks: -v "$PWD:/mnt" \ -w /mnt \ koalaman/shellcheck-alpine:v0.10.0 \ + shellcheck \ .github/scripts/bash/e2e/*.sh \ api/scripts/update-codegen.sh \ images/virtualization-artifact/hack/args.sh \ diff --git a/api/scripts/update-codegen.sh b/api/scripts/update-codegen.sh index fccf4f01da..b07d310fc5 100755 --- a/api/scripts/update-codegen.sh +++ b/api/scripts/update-codegen.sh @@ -43,6 +43,7 @@ function source::settings { "NodeUSBDevice" "USBDevice") + # shellcheck source=/dev/null source "${CODEGEN_PKG}/kube_codegen.sh" } @@ -86,7 +87,7 @@ function generate::crds { if ! [[ " ${ALLOWED_RESOURCE_GEN_CRD[*]} " =~ [[:space:]]$(cat "$file" | yq '.spec.names.kind')[[:space:]] ]]; then continue fi - cp "$file" "${ROOT}/crds/$(echo $file | awk -Fio_ '{print $2}')" + cp "$file" "${ROOT}/crds/$(echo "$file" | awk -Fio_ '{print $2}')" done } diff --git a/images/virtualization-artifact/hack/args.sh b/images/virtualization-artifact/hack/args.sh index 481043edc2..98c08cfa03 100644 --- a/images/virtualization-artifact/hack/args.sh +++ b/images/virtualization-artifact/hack/args.sh @@ -36,7 +36,7 @@ function parse_flag() { local DEFAULT="${3:-}" local RESULT="" - for f in ${FLAGS[*]}; do + for f in "${FLAGS[@]}"; do case "${f}" in --${NAME}=*|-${SHORT_NAME}=*) RESULT="${f#*=}" diff --git a/images/virtualization-artifact/hack/dlv.sh b/images/virtualization-artifact/hack/dlv.sh index 9881789165..665a341911 100755 --- a/images/virtualization-artifact/hack/dlv.sh +++ b/images/virtualization-artifact/hack/dlv.sh @@ -109,6 +109,7 @@ DIR="$(dirname "$0")" ROOT="${DIR}/../../../" cd "$ROOT" +# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" diff --git a/images/virtualization-artifact/hack/pyroscope.sh b/images/virtualization-artifact/hack/pyroscope.sh index 80028a3428..af5e1b3e5c 100755 --- a/images/virtualization-artifact/hack/pyroscope.sh +++ b/images/virtualization-artifact/hack/pyroscope.sh @@ -88,6 +88,7 @@ function stop-pyroscope() { docker compose -f "${DOCKER_COMPOSE_FILE_PYROSCOPE_ONLY}" down } +# shellcheck source=images/virtualization-artifact/hack/args.sh source "${DIR}/args.sh" set_flags_args "$@" From 5ab463a636331f9c9b4226c6dacae25d88cb427a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 29 May 2026 14:55:13 +0300 Subject: [PATCH 10/12] chore(ci): unify e2e nightly action versions Signed-off-by: Nikita Korolev --- .github/workflows/e2e-nightly-reusable-pipeline.yml | 10 +++------- .github/workflows/e2e-nightly.yml | 8 ++++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index dc3429c86d..c1cae79f21 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -178,7 +178,7 @@ jobs: kubeconfig: ${{ steps.generate-kubeconfig.outputs.kubeconfig }} namespace: ${{ steps.vars.outputs.namespace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set outputs env: @@ -201,8 +201,6 @@ jobs: sudo apt-get update sudo apt-get install -y apache2-utils - - uses: actions/checkout@v6 - - name: Setup E2E toolchain uses: ./.github/actions/setup-e2e-toolchain with: @@ -609,15 +607,13 @@ jobs: - configure-storage - configure-virtualization steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: go-version: "${{ env.GO_VERSION }}" - - uses: actions/checkout@v6 - - name: Setup E2E toolchain uses: ./.github/actions/setup-e2e-toolchain with: @@ -708,7 +704,7 @@ jobs: workflow_run_url: ${{ steps.determine-stage.outputs.workflow_run_url }} branch: ${{ steps.determine-stage.outputs.branch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Download E2E test results if available uses: actions/download-artifact@v8 diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 54f58fd304..b41fd87fa5 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 with: @@ -139,7 +139,7 @@ jobs: - e2e-nfs if: ${{ always()}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Download E2E report artifacts uses: actions/download-artifact@v8 @@ -188,7 +188,7 @@ jobs: --top-n 5 - name: Upload top-5 slowest Describe charts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: e2e-report-slowest-by-describe path: tmp/charts/ From 50099a62739ec692637a96755918bc502dea4f3a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 29 May 2026 16:53:52 +0300 Subject: [PATCH 11/12] refactor conf-vir detect-k8s Signed-off-by: Nikita Korolev --- .../scripts/bash/e2e/configure-virtualization.sh | 15 +++++++++------ .github/scripts/bash/e2e/detect-k8s-version.sh | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/scripts/bash/e2e/configure-virtualization.sh b/.github/scripts/bash/e2e/configure-virtualization.sh index db9a9d908d..38941f697b 100644 --- a/.github/scripts/bash/e2e/configure-virtualization.sh +++ b/.github/scripts/bash/e2e/configure-virtualization.sh @@ -8,12 +8,15 @@ source "${SCRIPT_DIR}/common.sh" # shellcheck source=.github/scripts/bash/e2e/deckhouse.sh source "${SCRIPT_DIR}/deckhouse.sh" -require_env DEV_REGISTRY_DOCKER_CFG -require_env NESTED_STORAGE_CLASS_NAME -require_env VIRTUALIZATION_TAG -: "${DEV_REGISTRY_DOCKER_CFG:?}" -: "${NESTED_STORAGE_CLASS_NAME:?}" -: "${VIRTUALIZATION_TAG:?}" +required_env_vars=( + DEV_REGISTRY_DOCKER_CFG + NESTED_STORAGE_CLASS_NAME + VIRTUALIZATION_TAG +) + +for env_var in "${required_env_vars[@]}"; do + require_env "${env_var}" +done kubectl_apply_with_retry() { local count=20 diff --git a/.github/scripts/bash/e2e/detect-k8s-version.sh b/.github/scripts/bash/e2e/detect-k8s-version.sh index e711ff5f75..38a74a49d3 100644 --- a/.github/scripts/bash/e2e/detect-k8s-version.sh +++ b/.github/scripts/bash/e2e/detect-k8s-version.sh @@ -21,7 +21,6 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" require_env GITHUB_OUTPUT -: "${GITHUB_OUTPUT:?}" version_json="$(kubectl version -o json)" server_version="$(echo "${version_json}" | jq -r '.serverVersion.gitVersion')" From 773ba1146e2a668916ab4d6845f8e19bd045b7d9 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 29 May 2026 18:04:10 +0300 Subject: [PATCH 12/12] fix formatting Signed-off-by: Nikita Korolev --- .github/workflows/e2e-nightly-reusable-pipeline.yml | 2 ++ .github/workflows/e2e-nightly.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index c1cae79f21..bec1d534e5 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -246,6 +246,7 @@ jobs: working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} run: | task infra-deploy + - name: Bootstrap cluster [dhctl-bootstrap] id: dhctl-bootstrap working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} @@ -259,6 +260,7 @@ jobs: task dhctl-bootstrap echo "[SUCCESS] Done" timeout-minutes: 60 + - name: Bootstrap cluster [show-connection-info] working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} run: | diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index b41fd87fa5..4ca01cc4fc 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -39,6 +39,7 @@ jobs: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + - name: Delete nested clusters env: LABEL_SELECTOR: test=nightly-e2e @@ -53,14 +54,17 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v6 + - name: Configure kubectl via azure/k8s-set-context@v4 uses: azure/k8s-set-context@v4 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + - name: Power off VMs to free resources for nested cluster setup run: .github/scripts/bash/e2e/power-off-nested-vms.sh + set-vars: name: Set vars needs: power-off-vms-for-nested