From 3c5fd74060750a19aff46b838a45b24b7da3070a Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Sat, 6 Jun 2026 18:07:27 +0000 Subject: [PATCH 1/4] Add nvidia-container-toolkit install and CDI generation In a L4 GPU passtrhough setup (vllm-cuda-rhel9:3.3.3, vLLM v0.13.0, supports CUDA 12.8 / GPU driver 570+) gpu validation could not pass unless CDI was generated (even though there is no MIG mode to apply). The role vars accept nvidia_container_toolkit_install but never acts on it unless that is a MIG-enabled setup. Make CUDA toolkit installation, node rebooting, and config/generation a dedicated task gated on the new default variable. Also increase wait for reboot to a 30 min (multiple GPUs with large VRAM take lot of time to initialize for VMs). This fixes the "unresolvable CDI devices nvidia.com/gpu=all" error when running vLLM containers on CentOS Stream 9 VMs with passthrough GPUs. Signed-off-by: Bohdan Dobrelia --- gpu-validation/defaults/main.yaml | 6 +++ gpu-validation/tasks/main.yaml | 4 ++ .../tasks/nvidia-container-toolkit.yaml | 45 +++++++++++++++++++ gpu-validation/tasks/nvidia-cuda-repos.yaml | 37 --------------- 4 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 gpu-validation/tasks/nvidia-container-toolkit.yaml diff --git a/gpu-validation/defaults/main.yaml b/gpu-validation/defaults/main.yaml index 529ad5e..ccc8dac 100644 --- a/gpu-validation/defaults/main.yaml +++ b/gpu-validation/defaults/main.yaml @@ -76,6 +76,12 @@ gpu_validation_dns_server: 192.168.122.1 gpu_validation_pci_devices: 10de:27b8: 1 +# [bool] Install nvidia-container-toolkit and generate CDI specs for container GPU access. +# Required when the guest VM does not have the toolkit pre-installed. +gpu_validation_nvidia_container_toolkit_install: false +# [string] URL of the nvidia-container-toolkit .repo file to add. +# When empty, the repo is not added (toolkit must already be available from an existing repo). +gpu_validation_nvidia_container_toolkit_repo_url: "https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo" # [string] PyTorch pip index URL (e.g. https://download.pytorch.org/whl/cu128 for Blackwell) gpu_validation_pytorch_url: https://download.pytorch.org/whl/cu126 diff --git a/gpu-validation/tasks/main.yaml b/gpu-validation/tasks/main.yaml index 2ade4d5..bc65403 100644 --- a/gpu-validation/tasks/main.yaml +++ b/gpu-validation/tasks/main.yaml @@ -16,6 +16,10 @@ - ansible_distribution == "RedHat" - gpu_validation_mode == "mig" +- name: Install NVIDIA container toolkit and configure CDI + ansible.builtin.import_tasks: nvidia-container-toolkit.yaml + when: gpu_validation_nvidia_container_toolkit_install | bool + - name: Check GPUs ansible.builtin.import_tasks: gpus.yaml - name: GPUs Assertions # noqa: ignore-errors diff --git a/gpu-validation/tasks/nvidia-container-toolkit.yaml b/gpu-validation/tasks/nvidia-container-toolkit.yaml new file mode 100644 index 0000000..8269d08 --- /dev/null +++ b/gpu-validation/tasks/nvidia-container-toolkit.yaml @@ -0,0 +1,45 @@ +--- +- name: Add NVIDIA container toolkit repository + when: gpu_validation_nvidia_container_toolkit_repo_url | default('') | length > 0 + become: true + ansible.builtin.get_url: + url: "{{ gpu_validation_nvidia_container_toolkit_repo_url }}" + dest: /etc/yum.repos.d/nvidia-container-toolkit.repo + mode: "0644" + +- name: Install nvidia-container-toolkit + become: true + ansible.builtin.dnf: + use_backend: dnf4 + name: nvidia-container-toolkit + state: present + +- name: Reboot the VM to find the installed drivers + become: true + ansible.builtin.reboot: + reboot_timeout: 1800 + +- name: Check if CDI configfile exists + become: true + ansible.builtin.stat: + path: /etc/cdi/nvidia.yaml + register: nvidia_driver_cdi_config_file + +- name: Configure NVIDIA CDI for container GPU access + when: not nvidia_driver_cdi_config_file.stat.exists + become: true + block: + - name: Ensure CDI directory exists + ansible.builtin.file: + path: /etc/cdi + state: directory + mode: "0755" + owner: root + + - name: Configure NVIDIA container runtime + ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd + changed_when: true + + - name: Generate NVIDIA CDI configuration + ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + changed_when: true diff --git a/gpu-validation/tasks/nvidia-cuda-repos.yaml b/gpu-validation/tasks/nvidia-cuda-repos.yaml index a235f0e..949116f 100644 --- a/gpu-validation/tasks/nvidia-cuda-repos.yaml +++ b/gpu-validation/tasks/nvidia-cuda-repos.yaml @@ -8,43 +8,6 @@ gpgcheck: true gpgkey: "{{ edpm_accel_drivers_nvidia_repo_gpgkey }}" -- name: Install nvidia-container-toolkit - become: true - ansible.builtin.dnf: - use_backend: dnf4 - name: nvidia-container-toolkit - state: present - -- name: Reboot the VM to find the installed drivers - become: true - ansible.builtin.reboot: - reboot_timeout: 600 - -- name: Check if CDI configfile exists - become: true - ansible.builtin.stat: - path: /etc/cdi/nvidia.yaml - register: nvidia_driver_cdi_config_file - -- name: Configure NVIDIA container runtime - when: not nvidia_driver_cdi_config_file.stat.exists - become: true - block: - - name: Ensure CDI directory exists - ansible.builtin.file: - path: /etc/cdi - state: directory - mode: "0755" - owner: root - - - name: Configure NVIDIA container runtime - ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd - changed_when: true - - - name: Generate NVIDIA CDI configuration - ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml - changed_when: true - - name: Install NVIDIA Management Library become: true ansible.builtin.dnf: From ceaf4251930d46af31dcb1e552407b2969fd7710 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Fri, 10 Jul 2026 11:45:52 +0200 Subject: [PATCH 2/4] Drop unsupported dnf use_backend Ansible core 2.14.18 which comes from RHEL 9.6 repos does not support this param. Signed-off-by: Bohdan Dobrelia --- gpu-validation/tasks/nvidia-container-toolkit.yaml | 1 - gpu-validation/tasks/nvidia-cuda-repos.yaml | 1 - gpu-validation/tasks/setup.yaml | 4 ---- 3 files changed, 6 deletions(-) diff --git a/gpu-validation/tasks/nvidia-container-toolkit.yaml b/gpu-validation/tasks/nvidia-container-toolkit.yaml index 8269d08..0206bcc 100644 --- a/gpu-validation/tasks/nvidia-container-toolkit.yaml +++ b/gpu-validation/tasks/nvidia-container-toolkit.yaml @@ -10,7 +10,6 @@ - name: Install nvidia-container-toolkit become: true ansible.builtin.dnf: - use_backend: dnf4 name: nvidia-container-toolkit state: present diff --git a/gpu-validation/tasks/nvidia-cuda-repos.yaml b/gpu-validation/tasks/nvidia-cuda-repos.yaml index 949116f..3b92fe4 100644 --- a/gpu-validation/tasks/nvidia-cuda-repos.yaml +++ b/gpu-validation/tasks/nvidia-cuda-repos.yaml @@ -11,7 +11,6 @@ - name: Install NVIDIA Management Library become: true ansible.builtin.dnf: - use_backend: dnf4 name: "{{ gpu_validation_libnvidia_ml_version }}" state: present diff --git a/gpu-validation/tasks/setup.yaml b/gpu-validation/tasks/setup.yaml index c9b0edc..8257446 100644 --- a/gpu-validation/tasks/setup.yaml +++ b/gpu-validation/tasks/setup.yaml @@ -31,7 +31,6 @@ - name: Install DKMS (UNSUPPORTED) ansible.builtin.dnf: - use_backend: dnf4 name: - dkms - kernel-devel @@ -40,19 +39,16 @@ - name: Install pciutils package ansible.builtin.dnf: - use_backend: dnf4 name: pciutils state: present - name: Install pip package ansible.builtin.dnf: - use_backend: dnf4 name: python-pip state: present - name: Install podman package ansible.builtin.dnf: - use_backend: dnf4 name: podman state: present From 8021ce3830741557656c29a13c0042f574693812 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Fri, 10 Jul 2026 12:30:43 +0200 Subject: [PATCH 3/4] Allow running for pre-deployed VM with drivers Add rescue block for reboot calls to handle the error "Running ansible.builtin.reboot with local connection would reboot the control node". In local runner mode unconditional reboot action would always fail the playbook (VM as a control node cannot self-reboot). We want it to only print a RED error msg and continue. The user is expected to notice it in the log after the validation fails later because reboot wasn't done. Then user should reboot the node, re-run the suit, and ignore the red error msg after that. Signed-off-by: Bohdan Dobrelia --- .../tasks/model_download_and_serve.yaml | 2 +- .../tasks/nvidia-container-toolkit.yaml | 16 +++++++++++++--- gpu-validation/tasks/reboot_if_needed.yaml | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/gpu-validation/tasks/model_download_and_serve.yaml b/gpu-validation/tasks/model_download_and_serve.yaml index e7f5a06..c1e3f00 100644 --- a/gpu-validation/tasks/model_download_and_serve.yaml +++ b/gpu-validation/tasks/model_download_and_serve.yaml @@ -62,7 +62,7 @@ validate_certs: false register: api_result until: ("status" in api_result) and (api_result.status == 200) - retries: 180 # 180 * 10 = 30 mins - this includes the time to download the model + retries: "{{ [1, (((gpu_validation_server_timeout | int) + 9) // 10)] | max }}" delay: 10 changed_when: false diff --git a/gpu-validation/tasks/nvidia-container-toolkit.yaml b/gpu-validation/tasks/nvidia-container-toolkit.yaml index 0206bcc..f8f7ebd 100644 --- a/gpu-validation/tasks/nvidia-container-toolkit.yaml +++ b/gpu-validation/tasks/nvidia-container-toolkit.yaml @@ -14,9 +14,19 @@ state: present - name: Reboot the VM to find the installed drivers - become: true - ansible.builtin.reboot: - reboot_timeout: 1800 + block: + - name: Reboot + become: true + ansible.builtin.reboot: + reboot_timeout: "{{ gpu_validation_reboot_timeout }}" + post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" + rescue: + - name: Debug message (local runner mode) # noqa: ignore-errors + ansible.builtin.fail: + msg: >- + In local runner mode, the VM is the control node and cannot be rebooted! + Make sure to reboot it manually (just once) and restart the validation suite after. + ignore_errors: true - name: Check if CDI configfile exists become: true diff --git a/gpu-validation/tasks/reboot_if_needed.yaml b/gpu-validation/tasks/reboot_if_needed.yaml index 6a45dbf..3211e2c 100644 --- a/gpu-validation/tasks/reboot_if_needed.yaml +++ b/gpu-validation/tasks/reboot_if_needed.yaml @@ -17,9 +17,18 @@ var: _gpu_validation_needs_restarting.stdout - name: Reboot to apply system updates - become: true - ansible.builtin.reboot: - msg: "Rebooting to apply kernel and driver updates" - reboot_timeout: "{{ gpu_validation_reboot_timeout }}" - post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" when: gpu_validation_reboot_force | bool or _gpu_validation_needs_restarting.rc == 1 + become: true + block: + - name: Reboot to apply system updates + ansible.builtin.reboot: + msg: "Rebooting to apply kernel and driver updates" + reboot_timeout: "{{ gpu_validation_reboot_timeout }}" + post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" + rescue: + - name: Debug message (local runner mode) # noqa: ignore-errors + ansible.builtin.fail: + msg: >- + In local runner mode, the VM is the control node and cannot be rebooted! + Make sure to reboot it manually (just once) and restart the validation suite after. + ignore_errors: true From 451963b080188d2926e6062bd793776be93c0362 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Fri, 10 Jul 2026 18:30:01 +0200 Subject: [PATCH 4/4] Fix order and idempotency for CDI generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix CDI generation was skipped if /etc/cdi/nvidia.yaml already exists — so a stale spec (generated before nvidia-driver-cuda-libs was installed) is never refreshed. Make CDI always be regenerated after a toolkit install/reboot — the command is fast and idempotent in output; the cost of skipping is a broken container runtime. Also make sure the delayed CDI generation is imported at the intended point and the earlier task file no longer generates the spec prematurely. Signed-off-by: Bohdan Dobrelia --- gpu-validation/tasks/main.yaml | 6 ++++- gpu-validation/tasks/nvidia-cdi.yaml | 15 +++++++++++ .../tasks/nvidia-container-toolkit.yaml | 26 +++---------------- 3 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 gpu-validation/tasks/nvidia-cdi.yaml diff --git a/gpu-validation/tasks/main.yaml b/gpu-validation/tasks/main.yaml index bc65403..dfbc144 100644 --- a/gpu-validation/tasks/main.yaml +++ b/gpu-validation/tasks/main.yaml @@ -16,7 +16,7 @@ - ansible_distribution == "RedHat" - gpu_validation_mode == "mig" -- name: Install NVIDIA container toolkit and configure CDI +- name: Install NVIDIA container toolkit and configure runtime ansible.builtin.import_tasks: nvidia-container-toolkit.yaml when: gpu_validation_nvidia_container_toolkit_install | bool @@ -38,6 +38,10 @@ ansible.builtin.import_tasks: cuda_assertions.yaml ignore_errors: true +- name: Generate NVIDIA CDI configuration + ansible.builtin.import_tasks: nvidia-cdi.yaml + when: gpu_validation_nvidia_container_toolkit_install | bool + - name: Check vllm ansible.builtin.import_tasks: vllm.yaml when: gpu_validation_model_tests_enabled | bool diff --git a/gpu-validation/tasks/nvidia-cdi.yaml b/gpu-validation/tasks/nvidia-cdi.yaml new file mode 100644 index 0000000..6e1f509 --- /dev/null +++ b/gpu-validation/tasks/nvidia-cdi.yaml @@ -0,0 +1,15 @@ +--- +- name: Ensure CDI directory exists + become: true + ansible.builtin.file: + path: /etc/cdi + state: directory + mode: "0755" + owner: root + +# Generate CDI after the driver/CUDA installation flow has completed. +# If it is generated too early, libcuda.so.1 can be missing from the spec. +- name: Generate NVIDIA CDI configuration + become: true + ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + changed_when: true diff --git a/gpu-validation/tasks/nvidia-container-toolkit.yaml b/gpu-validation/tasks/nvidia-container-toolkit.yaml index f8f7ebd..6ee5129 100644 --- a/gpu-validation/tasks/nvidia-container-toolkit.yaml +++ b/gpu-validation/tasks/nvidia-container-toolkit.yaml @@ -28,27 +28,7 @@ Make sure to reboot it manually (just once) and restart the validation suite after. ignore_errors: true -- name: Check if CDI configfile exists +- name: Configure NVIDIA container runtime for CDI become: true - ansible.builtin.stat: - path: /etc/cdi/nvidia.yaml - register: nvidia_driver_cdi_config_file - -- name: Configure NVIDIA CDI for container GPU access - when: not nvidia_driver_cdi_config_file.stat.exists - become: true - block: - - name: Ensure CDI directory exists - ansible.builtin.file: - path: /etc/cdi - state: directory - mode: "0755" - owner: root - - - name: Configure NVIDIA container runtime - ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd - changed_when: true - - - name: Generate NVIDIA CDI configuration - ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml - changed_when: true + ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd + changed_when: true