Add nvidia-container-toolkit install and CDI generation

gpu-validation/defaults/main.yaml

@@ -76,6 +76,12 @@ gpu_validation_dns_server: 192.168.122.1
 gpu_validation_pci_devices:
   10de:27b8: 1
 
+# [bool] Install nvidia-container-toolkit and generate CDI specs for container GPU access.
+# Required when the guest VM does not have the toolkit pre-installed.
+gpu_validation_nvidia_container_toolkit_install: false
+# [string] URL of the nvidia-container-toolkit .repo file to add.
+# When empty, the repo is not added (toolkit must already be available from an existing repo).
+gpu_validation_nvidia_container_toolkit_repo_url: "https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo"
 # [string] PyTorch pip index URL (e.g. https://download.pytorch.org/whl/cu128 for Blackwell)
 gpu_validation_pytorch_url: https://download.pytorch.org/whl/cu126
 

gpu-validation/tasks/main.yaml

@@ -16,6 +16,10 @@
     - ansible_distribution == "RedHat"
     - gpu_validation_mode == "mig"
 
+- name: Install NVIDIA container toolkit and configure runtime
+  ansible.builtin.import_tasks: nvidia-container-toolkit.yaml
+  when: gpu_validation_nvidia_container_toolkit_install | bool
+
 - name: Check GPUs
   ansible.builtin.import_tasks: gpus.yaml
 - name: GPUs Assertions  # noqa: ignore-errors
@@ -34,6 +38,10 @@
   ansible.builtin.import_tasks: cuda_assertions.yaml
   ignore_errors: true
 
+- name: Generate NVIDIA CDI configuration
+  ansible.builtin.import_tasks: nvidia-cdi.yaml
+  when: gpu_validation_nvidia_container_toolkit_install | bool
+
 - name: Check vllm
   ansible.builtin.import_tasks: vllm.yaml
   when: gpu_validation_model_tests_enabled | bool

gpu-validation/tasks/model_download_and_serve.yaml

@@ -62,7 +62,7 @@
     validate_certs: false
   register: api_result
   until: ("status" in api_result) and (api_result.status == 200)
-  retries: 180  # 180 * 10 = 30 mins - this includes the time to download the model
+  retries: "{{ [1, (((gpu_validation_server_timeout | int) + 9) // 10)] | max }}"
   delay: 10
   changed_when: false
 

gpu-validation/tasks/nvidia-cdi.yaml

@@ -0,0 +1,15 @@
+---
+- name: Ensure CDI directory exists
+  become: true
+  ansible.builtin.file:
+    path: /etc/cdi
+    state: directory
+    mode: "0755"
+    owner: root
+
+# Generate CDI after the driver/CUDA installation flow has completed.
+# If it is generated too early, libcuda.so.1 can be missing from the spec.
+- name: Generate NVIDIA CDI configuration
+  become: true
+  ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+  changed_when: true

gpu-validation/tasks/nvidia-container-toolkit.yaml

@@ -0,0 +1,34 @@
+---
+- name: Add NVIDIA container toolkit repository
+  when: gpu_validation_nvidia_container_toolkit_repo_url | default('') | length > 0
+  become: true
+  ansible.builtin.get_url:
+    url: "{{ gpu_validation_nvidia_container_toolkit_repo_url }}"
+    dest: /etc/yum.repos.d/nvidia-container-toolkit.repo
+    mode: "0644"
+
+- name: Install nvidia-container-toolkit
+  become: true
+  ansible.builtin.dnf:
+    name: nvidia-container-toolkit
+    state: present
+
+- name: Reboot the VM to find the installed drivers
+  block:
+    - name: Reboot
+      become: true
+      ansible.builtin.reboot:
+        reboot_timeout: "{{ gpu_validation_reboot_timeout }}"
+        post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}"
+  rescue:
+    - name: Debug message (local runner mode)  # noqa: ignore-errors
+      ansible.builtin.fail:
+        msg: >-
+          In local runner mode, the VM is the control node and cannot be rebooted!
+          Make sure to reboot it manually (just once) and restart the validation suite after.
+      ignore_errors: true
+
+- name: Configure NVIDIA container runtime for CDI
+  become: true
+  ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd
+  changed_when: true

gpu-validation/tasks/nvidia-cuda-repos.yaml

@@ -8,47 +8,9 @@
     gpgcheck: true
     gpgkey: "{{ edpm_accel_drivers_nvidia_repo_gpgkey }}"
 
-- name: Install nvidia-container-toolkit
-  become: true
-  ansible.builtin.dnf:
-    use_backend: dnf4
-    name: nvidia-container-toolkit
-    state: present
-
-- name: Reboot the VM to find the installed drivers
-  become: true
-  ansible.builtin.reboot:
-    reboot_timeout: 600
-
-- name: Check if CDI configfile exists
-  become: true
-  ansible.builtin.stat:
-    path: /etc/cdi/nvidia.yaml
-  register: nvidia_driver_cdi_config_file
-
-- name: Configure NVIDIA container runtime
-  when: not nvidia_driver_cdi_config_file.stat.exists
-  become: true
-  block:
-    - name: Ensure CDI directory exists
-      ansible.builtin.file:
-        path: /etc/cdi
-        state: directory
-        mode: "0755"
-        owner: root
-
-    - name: Configure NVIDIA container runtime
-      ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd
-      changed_when: true
-
-    - name: Generate NVIDIA CDI configuration
-      ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
-      changed_when: true
-
 - name: Install NVIDIA Management Library
   become: true
   ansible.builtin.dnf:
-    use_backend: dnf4
     name: "{{ gpu_validation_libnvidia_ml_version }}"
     state: present
 

gpu-validation/tasks/reboot_if_needed.yaml

@@ -17,9 +17,18 @@
     var: _gpu_validation_needs_restarting.stdout
 
 - name: Reboot to apply system updates
-  become: true
-  ansible.builtin.reboot:
-    msg: "Rebooting to apply kernel and driver updates"
-    reboot_timeout: "{{ gpu_validation_reboot_timeout }}"
-    post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}"
   when: gpu_validation_reboot_force | bool or _gpu_validation_needs_restarting.rc == 1
+  become: true
+  block:
+    - name: Reboot to apply system updates
+      ansible.builtin.reboot:
+        msg: "Rebooting to apply kernel and driver updates"
+        reboot_timeout: "{{ gpu_validation_reboot_timeout }}"
+        post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}"
+  rescue:
+    - name: Debug message (local runner mode)  # noqa: ignore-errors
+      ansible.builtin.fail:
+        msg: >-
+          In local runner mode, the VM is the control node and cannot be rebooted!
+          Make sure to reboot it manually (just once) and restart the validation suite after.
+      ignore_errors: true

gpu-validation/tasks/setup.yaml

@@ -31,7 +31,6 @@
 
     - name: Install DKMS (UNSUPPORTED)
       ansible.builtin.dnf:
-        use_backend: dnf4
         name:
           - dkms
           - kernel-devel
@@ -40,19 +39,16 @@
 
 - name: Install pciutils package
   ansible.builtin.dnf:
-    use_backend: dnf4
     name: pciutils
     state: present
 
 - name: Install pip package
   ansible.builtin.dnf:
-    use_backend: dnf4
     name: python-pip
     state: present
 
 - name: Install podman package
   ansible.builtin.dnf:
-    use_backend: dnf4
     name: podman
     state: present