diff --git a/gpu-validation/defaults/main.yaml b/gpu-validation/defaults/main.yaml index 529ad5e..ccc8dac 100644 --- a/gpu-validation/defaults/main.yaml +++ b/gpu-validation/defaults/main.yaml @@ -76,6 +76,12 @@ gpu_validation_dns_server: 192.168.122.1 gpu_validation_pci_devices: 10de:27b8: 1 +# [bool] Install nvidia-container-toolkit and generate CDI specs for container GPU access. +# Required when the guest VM does not have the toolkit pre-installed. +gpu_validation_nvidia_container_toolkit_install: false +# [string] URL of the nvidia-container-toolkit .repo file to add. +# When empty, the repo is not added (toolkit must already be available from an existing repo). +gpu_validation_nvidia_container_toolkit_repo_url: "https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo" # [string] PyTorch pip index URL (e.g. https://download.pytorch.org/whl/cu128 for Blackwell) gpu_validation_pytorch_url: https://download.pytorch.org/whl/cu126 diff --git a/gpu-validation/tasks/main.yaml b/gpu-validation/tasks/main.yaml index 2ade4d5..dfbc144 100644 --- a/gpu-validation/tasks/main.yaml +++ b/gpu-validation/tasks/main.yaml @@ -16,6 +16,10 @@ - ansible_distribution == "RedHat" - gpu_validation_mode == "mig" +- name: Install NVIDIA container toolkit and configure runtime + ansible.builtin.import_tasks: nvidia-container-toolkit.yaml + when: gpu_validation_nvidia_container_toolkit_install | bool + - name: Check GPUs ansible.builtin.import_tasks: gpus.yaml - name: GPUs Assertions # noqa: ignore-errors @@ -34,6 +38,10 @@ ansible.builtin.import_tasks: cuda_assertions.yaml ignore_errors: true +- name: Generate NVIDIA CDI configuration + ansible.builtin.import_tasks: nvidia-cdi.yaml + when: gpu_validation_nvidia_container_toolkit_install | bool + - name: Check vllm ansible.builtin.import_tasks: vllm.yaml when: gpu_validation_model_tests_enabled | bool diff --git a/gpu-validation/tasks/model_download_and_serve.yaml b/gpu-validation/tasks/model_download_and_serve.yaml index e7f5a06..c1e3f00 100644 --- a/gpu-validation/tasks/model_download_and_serve.yaml +++ b/gpu-validation/tasks/model_download_and_serve.yaml @@ -62,7 +62,7 @@ validate_certs: false register: api_result until: ("status" in api_result) and (api_result.status == 200) - retries: 180 # 180 * 10 = 30 mins - this includes the time to download the model + retries: "{{ [1, (((gpu_validation_server_timeout | int) + 9) // 10)] | max }}" delay: 10 changed_when: false diff --git a/gpu-validation/tasks/nvidia-cdi.yaml b/gpu-validation/tasks/nvidia-cdi.yaml new file mode 100644 index 0000000..6e1f509 --- /dev/null +++ b/gpu-validation/tasks/nvidia-cdi.yaml @@ -0,0 +1,15 @@ +--- +- name: Ensure CDI directory exists + become: true + ansible.builtin.file: + path: /etc/cdi + state: directory + mode: "0755" + owner: root + +# Generate CDI after the driver/CUDA installation flow has completed. +# If it is generated too early, libcuda.so.1 can be missing from the spec. +- name: Generate NVIDIA CDI configuration + become: true + ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + changed_when: true diff --git a/gpu-validation/tasks/nvidia-container-toolkit.yaml b/gpu-validation/tasks/nvidia-container-toolkit.yaml new file mode 100644 index 0000000..6ee5129 --- /dev/null +++ b/gpu-validation/tasks/nvidia-container-toolkit.yaml @@ -0,0 +1,34 @@ +--- +- name: Add NVIDIA container toolkit repository + when: gpu_validation_nvidia_container_toolkit_repo_url | default('') | length > 0 + become: true + ansible.builtin.get_url: + url: "{{ gpu_validation_nvidia_container_toolkit_repo_url }}" + dest: /etc/yum.repos.d/nvidia-container-toolkit.repo + mode: "0644" + +- name: Install nvidia-container-toolkit + become: true + ansible.builtin.dnf: + name: nvidia-container-toolkit + state: present + +- name: Reboot the VM to find the installed drivers + block: + - name: Reboot + become: true + ansible.builtin.reboot: + reboot_timeout: "{{ gpu_validation_reboot_timeout }}" + post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" + rescue: + - name: Debug message (local runner mode) # noqa: ignore-errors + ansible.builtin.fail: + msg: >- + In local runner mode, the VM is the control node and cannot be rebooted! + Make sure to reboot it manually (just once) and restart the validation suite after. + ignore_errors: true + +- name: Configure NVIDIA container runtime for CDI + become: true + ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd + changed_when: true diff --git a/gpu-validation/tasks/nvidia-cuda-repos.yaml b/gpu-validation/tasks/nvidia-cuda-repos.yaml index a235f0e..3b92fe4 100644 --- a/gpu-validation/tasks/nvidia-cuda-repos.yaml +++ b/gpu-validation/tasks/nvidia-cuda-repos.yaml @@ -8,47 +8,9 @@ gpgcheck: true gpgkey: "{{ edpm_accel_drivers_nvidia_repo_gpgkey }}" -- name: Install nvidia-container-toolkit - become: true - ansible.builtin.dnf: - use_backend: dnf4 - name: nvidia-container-toolkit - state: present - -- name: Reboot the VM to find the installed drivers - become: true - ansible.builtin.reboot: - reboot_timeout: 600 - -- name: Check if CDI configfile exists - become: true - ansible.builtin.stat: - path: /etc/cdi/nvidia.yaml - register: nvidia_driver_cdi_config_file - -- name: Configure NVIDIA container runtime - when: not nvidia_driver_cdi_config_file.stat.exists - become: true - block: - - name: Ensure CDI directory exists - ansible.builtin.file: - path: /etc/cdi - state: directory - mode: "0755" - owner: root - - - name: Configure NVIDIA container runtime - ansible.builtin.command: nvidia-ctk runtime configure --runtime=containerd - changed_when: true - - - name: Generate NVIDIA CDI configuration - ansible.builtin.command: nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml - changed_when: true - - name: Install NVIDIA Management Library become: true ansible.builtin.dnf: - use_backend: dnf4 name: "{{ gpu_validation_libnvidia_ml_version }}" state: present diff --git a/gpu-validation/tasks/reboot_if_needed.yaml b/gpu-validation/tasks/reboot_if_needed.yaml index 6a45dbf..3211e2c 100644 --- a/gpu-validation/tasks/reboot_if_needed.yaml +++ b/gpu-validation/tasks/reboot_if_needed.yaml @@ -17,9 +17,18 @@ var: _gpu_validation_needs_restarting.stdout - name: Reboot to apply system updates - become: true - ansible.builtin.reboot: - msg: "Rebooting to apply kernel and driver updates" - reboot_timeout: "{{ gpu_validation_reboot_timeout }}" - post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" when: gpu_validation_reboot_force | bool or _gpu_validation_needs_restarting.rc == 1 + become: true + block: + - name: Reboot to apply system updates + ansible.builtin.reboot: + msg: "Rebooting to apply kernel and driver updates" + reboot_timeout: "{{ gpu_validation_reboot_timeout }}" + post_reboot_delay: "{{ gpu_validation_reboot_post_delay }}" + rescue: + - name: Debug message (local runner mode) # noqa: ignore-errors + ansible.builtin.fail: + msg: >- + In local runner mode, the VM is the control node and cannot be rebooted! + Make sure to reboot it manually (just once) and restart the validation suite after. + ignore_errors: true diff --git a/gpu-validation/tasks/setup.yaml b/gpu-validation/tasks/setup.yaml index c9b0edc..8257446 100644 --- a/gpu-validation/tasks/setup.yaml +++ b/gpu-validation/tasks/setup.yaml @@ -31,7 +31,6 @@ - name: Install DKMS (UNSUPPORTED) ansible.builtin.dnf: - use_backend: dnf4 name: - dkms - kernel-devel @@ -40,19 +39,16 @@ - name: Install pciutils package ansible.builtin.dnf: - use_backend: dnf4 name: pciutils state: present - name: Install pip package ansible.builtin.dnf: - use_backend: dnf4 name: python-pip state: present - name: Install podman package ansible.builtin.dnf: - use_backend: dnf4 name: podman state: present