diff --git a/.ansible-lint b/.ansible-lint index 0b740ce..fb2372e 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -11,6 +11,7 @@ skip_list: # during local development (known issue: ansible-lint#2487). mock_roles: - cozystack.installer.cozystack + - cozystack.installer.nvidia_vgpu_host # examples/*/site.yml imports external playbooks (k3s.orchestration.site) # that may not be installed during linting. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10eb507..12ac76a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -211,6 +211,155 @@ jobs: - name: Test host LVM global_filter rendering and effectiveness run: ansible-playbook tests/test-lvm-global-filter.yml + nvidia-vgpu-host: + name: NVIDIA vGPU host restore unit + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7.0.0 + with: + python-version: "3.14" + + - name: Install Ansible + run: pip install ansible-core + + - name: Build and install collection + run: | + ansible-galaxy collection build + ansible-galaxy collection install cozystack-installer-*.tar.gz --force + + # The runner has no NVIDIA GPU, which is the point: this asserts + # the artifacts render correctly, the unit is skipped by its + # condition rather than failing, and every path the script can + # take on hardware it is not meant for exits the way it should. + # shellcheck and systemd-analyze both run inside the playbook. + - name: Test the vGPU host restore unit and its no-op paths + run: >- + sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook + tests/test-nvidia-vgpu-host.yml + + - name: Test re-application (second run) + run: >- + sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook + tests/test-nvidia-vgpu-host.yml + + # The refusal paths above are everything the script does without a + # GPU. This runs the stages themselves against a fake nvidia-smi, + # a fake sriov-manage and a fake PCI tree, so the code that writes + # to hardware is executed rather than only rendered. + - name: Test the restore stages against a fake GPU + run: >- + sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook + tests/test-nvidia-vgpu-host-stages.yml + + # Two spellings of one card, or of one virtual function, collapse to + # a single key at render time and one of the two profiles is + # silently discarded. The role refuses rather than pick a winner. + - name: Test that duplicate device addresses are rejected + run: | + set +e + output="$(sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook \ + tests/test-nvidia-vgpu-host-idempotency.yml \ + --extra-vars '{"cozystack_nvidia_vgpu_devices":[{"address":"0000:41:00.0","sriov":true},{"address":"41:00.0","sriov":true}]}' 2>&1)" + status=$? + set -e + + if [ "$status" -eq 0 ]; then + echo "ERROR: two spellings of one card were accepted" + exit 1 + fi + + if ! grep -q "names the same GPU more than" <<< "$output"; then + echo "ERROR: rejected, but not by the duplicate-address check" + echo "$output" | tail -30 + exit 1 + fi + + echo "OK: duplicate device addresses correctly rejected" + + # A card and a virtual function are different levels and had + # different rules; the VF check deleted the PCI domain, so two + # functions on different segments looked like one. + - name: Test that duplicate virtual-function addresses are rejected + run: | + set +e + output="$(sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook \ + tests/test-nvidia-vgpu-host-idempotency.yml \ + --extra-vars '{"cozystack_nvidia_vgpu_devices":[{"address":"0000:41:00.0","vgpu_profiles":{"0000:41:00.5":1155,"41:00.5":1160}}]}' 2>&1)" + status=$? + set -e + + if [ "$status" -eq 0 ]; then + echo "ERROR: two spellings of one virtual function were accepted" + exit 1 + fi + + if ! grep -q "names the same virtual function more than" <<< "$output"; then + echo "ERROR: rejected, but not by the duplicate-VF check" + echo "$output" | tail -30 + exit 1 + fi + + echo "OK: duplicate virtual-function addresses correctly rejected" + + # The other direction, which is the one that regressed: cards and + # functions on different PCI segments are distinct and must be + # accepted, not refused as duplicates. + - name: Test that a multi-segment configuration is accepted + run: | + set -euo pipefail + devices='{"cozystack_nvidia_vgpu_devices":[ + {"address":"0000:41:00.0","vgpu_profiles":{"0000:41:00.5":1155}}, + {"address":"0001:41:00.0","vgpu_profiles":{"0001:41:00.5":1160}}]}' + sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook \ + tests/test-nvidia-vgpu-host-idempotency.yml --extra-vars "$devices" + echo "OK: cards on different PCI segments accepted" + + # Writing 0 to a function's current_vgpu_type does not set a + # profile, so a profile id of 0 is a configuration error rather + # than a value to pass through. + - name: Test that a zero profile id is rejected + run: | + set +e + for v in '{"cozystack_nvidia_vgpu_devices":[{"address":"0000:41:00.0","vgpu_profile":0}]}' \ + '{"cozystack_nvidia_vgpu_devices":[{"address":"0000:41:00.0","vgpu_profiles":{"0000:41:00.5":0}}]}' \ + '{"cozystack_nvidia_vgpu_devices":[{"address":"0000:41:00.0","mig":false}]}'; do + output="$(sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook \ + tests/test-nvidia-vgpu-host-idempotency.yml --extra-vars "$v" 2>&1)" + status=$? + if [ "$status" -eq 0 ]; then + echo "ERROR: accepted $v" + exit 1 + fi + if ! grep -qE "Invalid entry in|Invalid per-VF" <<< "$output"; then + echo "ERROR: rejected for the wrong reason: $v" + echo "$output" | tail -20 + exit 1 + fi + done + set -e + echo "OK: zero profile ids and entries that ask for nothing are rejected" + + - name: Apply the role on its own + run: >- + sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook + tests/test-nvidia-vgpu-host-idempotency.yml + + - name: Test idempotency (second run reports no change) + run: | + set -euo pipefail + output="$(sudo env "PATH=$PATH" "HOME=$HOME" ansible-playbook \ + tests/test-nvidia-vgpu-host-idempotency.yml)" + echo "$output" + if ! grep -q "changed=0" <<< "$output"; then + echo "ERROR: re-applying the role reported changes" + exit 1 + fi + echo "OK: role is idempotent" + e2e: name: E2E runs-on: ubuntu-latest diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 71e99d9..c0d1600 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,19 @@ Unreleased IP addresses for ingress-nginx Service ``externalIPs``. Required on ``isp-full-generic`` platform variant when nodes lack a native load balancer (cloud VMs, bare metal). +- New role ``cozystack.installer.nvidia_vgpu_host``, opt-in and disabled + by default via ``cozystack_enable_nvidia_vgpu_host``. It installs a + systemd unit that restores vGPU state at boot on hosts where the NVIDIA + vGPU host driver is installed directly on the node. A reboot on such a + host disables SR-IOV virtual functions, resets each function's + ``current_vgpu_type``, and on Hopper and later loses MIG mode, and + nothing on the node puts any of it back. Where gpu-operator manages the + vGPU Manager, its container entrypoint already enables the virtual + functions and the unit skips that host. The unit acts only on GPUs + named in ``cozystack_nvidia_vgpu_devices``, empty by default, addressed + by PCI address or GPU UUID rather than by index. It never resets a GPU. + Setting ``cozystack_enable_nvidia_vgpu_host`` back to ``false`` and + re-running disables the unit and removes it. Bugfixes -------- diff --git a/README.md b/README.md index c576b7d..2d9c2a1 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,58 @@ k3s also exposes a native `--nonroot-devices` flag (valid on both server and age The restart handler only fires when the drop-in is first created or its content changes; idempotent re-runs leave k3s untouched. When it does fire, `systemctl restart k3s` (or `k3s-agent`) briefly disrupts the control plane and the node's workloads on that host, so apply such a change in a maintenance window rather than casually mid-day. +#### Opt-in: NVIDIA vGPU host state across reboots + +`cozystack_enable_nvidia_vgpu_host: false` (default) installs nothing. Set it to `true` on hosts where you installed the NVIDIA vGPU host driver directly on the node, and the prepare playbook drops a systemd unit that restores vGPU state at boot. + +A reboot on such a host loses more than this role restores. These are the three it restores: + +- SR-IOV virtual functions. NVIDIA states it outright: "the virtual functions for the physical GPU in the sysfs file system are disabled after the hypervisor host is rebooted or if the driver is reloaded or upgraded". The same section says to use `sriov-manage` and nothing else. Loading the kernel module does not bring them back. +- Per-VF vGPU profiles. Each function's `current_vgpu_type` resets on PCI re-enumeration, and a function with no profile advertises nothing, so no VM can request it. +- MIG mode, on Hopper and later. The status bit that kept the setting across reboots on Ampere is gone from the InfoROM on newer parts, so the mode has to be set again after every boot. Enabling it there needs no GPU reset. On Ampere, where a reset would be needed, the mode already persisted and there is nothing to restore. So the unit never resets a GPU. + +Where gpu-operator manages the vGPU Manager, that container's entrypoint enables the virtual functions itself and its driver root lives under `/run/nvidia/driver`, so the host has no `sriov-manage`. The unit carries `ConditionPathExists` on the host's `sriov-manage` and systemd reports it skipped. The same covers a host with no NVIDIA GPU and a host running a plain compute driver. If a host-installed driver and an operator-managed driver root are both present, ownership is undecidable, and the unit declines every stage and logs why. + +Those are skips, and they exit zero. The unit fails instead when work the operator named did not happen, and the journal names which one. The exception is the per-PF `vgpu_profile` shorthand, which the role writes to every function the card exposes: functions past the card's instance limit reject it, and that is logged and skipped rather than failed, as described below. Common causes of a failure are a declared address that is not present on this host, virtual functions that could not be created, a function named in `vgpu_profiles` that would not take its profile, and MIG mode that could not be set. Treat that as examples rather than the whole set: every failing path logs its own line, so read the journal instead of matching a failure against this paragraph. The declared-address case is the one to watch when copying the example below, since an address left unedited names a card that does not exist. Declaring nothing at all is neither a skip nor a failure: with an empty device list the unit exits zero before looking at the hardware. + +This covers the host-installed, ansible-managed path only. For clusters where the operator manages the driver, a DaemonSet reconciling per-VF profiles from a ConfigMap is the right mechanism and this role is not a substitute for one. + +The unit only touches GPUs named in `cozystack_nvidia_vgpu_devices`, which is empty by default, so enabling the role is not enough to make it act. Nothing is inferred from the hardware: a card that is not named is never touched, whatever its PCI capabilities advertise. Addresses are PCI addresses or GPU UUIDs, never indices, because indices shift on PCI, BIOS and kernel re-enumeration. + +A card sliced uniformly, which is the common case: + +```yaml +cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" # PF PCI address, or GPU- + sriov: true # create virtual functions on this PF + vgpu_profile: 1155 # the same profile on every VF +``` + +A card whose functions do not all carry the same profile: + +```yaml +cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + sriov: true + vgpu_profiles: + "0000:41:00.4": 1155 + "0000:41:00.5": 1160 +``` + +Every entry must ask for something: `sriov` or `mig` set to `true`, a `vgpu_profile`, or a non-empty `vgpu_profiles`. An entry carrying only an address is rejected rather than ignored, so trimming an example down to `mig: false` on a card you do not want sliced is a playbook failure. Leave that card out instead. + +`vgpu_profile` is written to every virtual function the card exposes, which is the maximum the driver created rather than the number you intend to use. A card holds only as many instances of a type as its frame buffer divides into, so the functions past that limit reject the write; the unit logs each one and does not treat it as a failure. `vgpu_profiles` names one function each and wins over `vgpu_profile` for any function it names, and functions named by neither are left alone. Setting both on one card only works where the driver and the GPU support heterogeneous types on a single device, so check that first. Profile ids are the numeric vGPU types listed in a function's `creatable_vgpu_types`; they are per-SKU and per-driver, so read them off the host rather than copying them from here. + +The `sriov` and `mig` flags are separate on purpose. Creating virtual functions on a card whose driver reports SR-IOV mode is additive, while enabling MIG mode changes how the card is partitioned. They also have different preconditions, and on a mixed host the answer is per card. + +MIG instances are out of scope. The unit restores MIG mode and does not create GPU instances: that geometry is declared state owned by another component, and a second copy of it in a host script will drift from the first. MIG instances survive a reboot on no architecture, and the MIG user guide points at the MIG Partition Editor (`nvidia-mig-parted`) for it, "including creating a systemd service that could recreate the MIG geometry at system startup". That is what to pair with this role if you slice cards. + +Setting `cozystack_enable_nvidia_vgpu_host: false` and re-running the prepare playbook disables the unit and removes it along with its script, so the host stops restoring GPU state at the next boot. It does not undo state already applied to the hardware; virtual functions and MIG mode stay as they are until the host reboots or you change them. + +Applying the role enables the unit but does not start it, because creating virtual functions or enabling MIG mode changes hardware state that running VMs depend on. To apply it sooner, start `cozystack-nvidia-vgpu-restore.service` inside a maintenance window. Check what it did with `systemctl status cozystack-nvidia-vgpu-restore` and `journalctl --unit cozystack-nvidia-vgpu-restore`. Every declined stage logs its reason, and where a driver-reported value drove the decision, the value it saw. + +The unit is ordered after the driver's own vGPU daemons and before nothing else, so a node finishes booting and rejoins the cluster while its GPUs are still being restored. GPU VMs may sit `Pending` for a short window after a reboot until the device plugin rescans and advertises the functions again. That clears on its own. + #### Known limitations ZFS support depends on the OS ecosystem and kernel flavor. The prepare playbooks skip ZFS automation gracefully in these cases and emit an informational notice: @@ -382,6 +434,23 @@ These variables are consumed only by the example prepare playbooks in `examples/ | `cozystack_drbd_ppa` | `ppa:linbit/linbit-drbd9-stack` | `examples/ubuntu/` only: override to point at a Launchpad PPA mirror of the LINBIT archive. `ansible.builtin.apt_repository` resolves the signing key for `ppa:` URIs by querying Launchpad's REST API directly (no extra packages required). Non-Launchpad URIs (`deb http://internal-mirror/...`) work but you must manage the apt signing key separately — drop a keyring under `/etc/apt/keyrings/` and add `signed-by=` to the repo line. | | `cozystack_drbd_supported_releases` | `[jammy, noble]` | `examples/ubuntu/` only: list of Ubuntu release codenames LINBIT's PPA publishes drbd-dkms for. Extend from inventory when LINBIT adds a new series (e.g. `[jammy, noble, resolute]`) without waiting for a collection release. The playbook skips the install and emits a notice on Ubuntu hosts whose `ansible_distribution_release` is not in this list. | +## Role: cozystack.installer.nvidia_vgpu_host + +Installs a systemd unit that restores vGPU-relevant GPU state at boot on hosts where the NVIDIA vGPU host driver is installed directly on the node. Opt-in and disabled by default; a no-op on every other host. See [Opt-in: NVIDIA vGPU host state across reboots](#opt-in-nvidia-vgpu-host-state-across-reboots) for what a reboot loses, when the unit declines to act, and how per-VF profiles are addressed. + +Runs on every node in the `cluster` group. The `examples/*/prepare-*.yml` playbooks include it unconditionally and the role gates itself on `cozystack_enable_nvidia_vgpu_host`, so turning the toggle off reaches the path that removes the unit. + +### Optional variables + +| Variable | Default | Description | +| --- | --- | --- | +| `cozystack_enable_nvidia_vgpu_host` | `false` | Install and enable the boot unit. Off by default. Enabling it alone changes nothing: the unit still acts only on GPUs named in `cozystack_nvidia_vgpu_devices`, and declines when its preconditions do not hold. | +| `cozystack_nvidia_vgpu_devices` | `[]` | GPUs the unit may touch and what it may do to each. Empty means the unit is installed but inert. Entry keys: `address` (PCI address or `GPU-`, required), `sriov` (create virtual functions on this PF), `mig` (enable MIG mode on this GPU), `vgpu_profile` (profile for every VF of this PF), `vgpu_profiles` (per-VF map; overrides `vgpu_profile`). A bare GPU index is rejected, because indices shift on re-enumeration. Every entry must ask for something: `sriov` or `mig` set to `true`, a `vgpu_profile`, or a non-empty `vgpu_profiles`. An entry that asks for nothing is rejected rather than ignored. Profile ids must be positive, since 0 is not a profile id. | +| `cozystack_nvidia_vgpu_wait_seconds` | `120` | How long the unit waits for `nvidia-smi` to enumerate a GPU before giving up. It also sets the unit's `TimeoutStartSec`, as this value plus two minutes plus a minute for each declared card. The profile stage spends one waiting budget per card rather than one per function, so the margin does not need raising as a card exposes more of them. The driver's own units may still be starting, and `sriov-manage` is documented to fail while the Virtual GPU Manager initialises. Exceeding it is the one precondition that fails the unit rather than skipping quietly, because reaching it means a host driver is installed and GPUs were declared. | +| `cozystack_nvidia_vgpu_sriov_manage` | `/usr/lib/nvidia/sriov-manage` | Where the vGPU host driver installs `sriov-manage`. Doubles as the unit's `ConditionPathExists`: absent means either no host-installed driver or a gpu-operator-managed vGPU Manager, and systemd skips the unit. Override only for a non-standard driver install. | +| `cozystack_nvidia_vgpu_pci_root` | `/sys/bus/pci/devices` | Where the unit looks up PCI devices. No reason to change this on a real host; it exists so the boot script's stages can be exercised against a fake device tree in tests rather than only on GPU hardware. | +| `cozystack_nvidia_vgpu_operator_driver_root` | `/run/nvidia/driver` | Driver root that gpu-operator's driver container mounts on the host. Finding a driver there as well as on the host leaves GPU ownership ambiguous, and the unit declines every stage rather than guess. | + ## Using with k3s This collection is designed to work alongside [k3s.orchestration](https://github.com/k3s-io/k3s-ansible). The inventory structure (groups: `cluster`, `server`, `agent`) is fully compatible. diff --git a/examples/rhel/prepare-rhel.yml b/examples/rhel/prepare-rhel.yml index ea49f2d..600598e 100644 --- a/examples/rhel/prepare-rhel.yml +++ b/examples/rhel/prepare-rhel.yml @@ -518,3 +518,18 @@ path: /etc/modules-load.d/cozystack-kubevirt.conf state: absent when: not (cozystack_enable_kubevirt | default(true) | bool) + + # NVIDIA vGPU host state does not survive a reboot: SR-IOV virtual + # functions are dropped, per-VF vGPU profiles reset, and on Hopper + # and later MIG mode is lost with them. Where the vGPU host driver + # is installed directly on the node, nothing restores any of it. + # gpu-operator's vGPU Manager container does, but only on hosts it + # manages. Opt in per host group and name the GPUs; see the role's + # README section. + # Included unconditionally: the role gates both its paths on + # cozystack_enable_nvidia_vgpu_host itself, and gating the include + # too would make turning the toggle back off unreachable, leaving a + # host that was enabled once still restoring GPU state at every boot. + - name: Restore NVIDIA vGPU host state across reboots + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host diff --git a/examples/suse/prepare-suse.yml b/examples/suse/prepare-suse.yml index 3f0ce69..87cabec 100644 --- a/examples/suse/prepare-suse.yml +++ b/examples/suse/prepare-suse.yml @@ -492,3 +492,18 @@ path: /etc/modules-load.d/cozystack-kubevirt.conf state: absent when: not (cozystack_enable_kubevirt | default(true) | bool) + + # NVIDIA vGPU host state does not survive a reboot: SR-IOV virtual + # functions are dropped, per-VF vGPU profiles reset, and on Hopper + # and later MIG mode is lost with them. Where the vGPU host driver + # is installed directly on the node, nothing restores any of it. + # gpu-operator's vGPU Manager container does, but only on hosts it + # manages. Opt in per host group and name the GPUs; see the role's + # README section. + # Included unconditionally: the role gates both its paths on + # cozystack_enable_nvidia_vgpu_host itself, and gating the include + # too would make turning the toggle back off unreachable, leaving a + # host that was enabled once still restoring GPU state at every boot. + - name: Restore NVIDIA vGPU host state across reboots + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host diff --git a/examples/ubuntu/prepare-ubuntu.yml b/examples/ubuntu/prepare-ubuntu.yml index fcccbd4..eec6825 100644 --- a/examples/ubuntu/prepare-ubuntu.yml +++ b/examples/ubuntu/prepare-ubuntu.yml @@ -726,3 +726,18 @@ path: /etc/modules-load.d/cozystack-kubevirt.conf state: absent when: not (cozystack_enable_kubevirt | default(true) | bool) + + # NVIDIA vGPU host state does not survive a reboot: SR-IOV virtual + # functions are dropped, per-VF vGPU profiles reset, and on Hopper + # and later MIG mode is lost with them. Where the vGPU host driver + # is installed directly on the node, nothing restores any of it. + # gpu-operator's vGPU Manager container does, but only on hosts it + # manages. Opt in per host group and name the GPUs; see the role's + # README section. + # Included unconditionally: the role gates both its paths on + # cozystack_enable_nvidia_vgpu_host itself, and gating the include + # too would make turning the toggle back off unreachable, leaving a + # host that was enabled once still restoring GPU state at every boot. + - name: Restore NVIDIA vGPU host state across reboots + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host diff --git a/roles/nvidia_vgpu_host/defaults/main.yml b/roles/nvidia_vgpu_host/defaults/main.yml new file mode 100644 index 0000000..3200ca7 --- /dev/null +++ b/roles/nvidia_vgpu_host/defaults/main.yml @@ -0,0 +1,54 @@ +--- +# Install a boot unit that restores vGPU-relevant GPU state on hosts +# where the NVIDIA vGPU host driver is installed directly. Off by +# default. Enabling it is not sufficient on its own: the unit still +# declines to act unless its preconditions hold, and it touches only +# GPUs named in cozystack_nvidia_vgpu_devices. +cozystack_enable_nvidia_vgpu_host: false + +# Which GPUs the boot unit may touch, and what it may do to each. +# Empty means the unit is installed but inert. Nothing is derived from +# the hardware: a card that is not named here is never touched, +# whatever its PCI capabilities advertise. +# +# A card sliced uniformly: +# +# cozystack_nvidia_vgpu_devices: +# - address: "0000:41:00.0" # PF PCI address, or GPU- +# sriov: true # create virtual functions on this PF +# vgpu_profile: 1155 # the same profile on every VF +# +# A card whose functions differ, one entry per function: +# +# cozystack_nvidia_vgpu_devices: +# - address: "0000:41:00.0" +# sriov: true +# vgpu_profiles: +# "0000:41:00.4": 1155 +# "0000:41:00.5": 1160 +# +# Setting both forms on one card needs driver and GPU support for +# heterogeneous types on a single device. Every entry must ask for +# something: sriov or mig set to true, a vgpu_profile, or a non-empty +# vgpu_profiles. An address on its own is rejected rather than ignored. +cozystack_nvidia_vgpu_devices: [] + +# How long the unit waits for nvidia-smi to enumerate GPUs before it +# gives up. The driver's own units may still be starting at that point, +# and sriov-manage is known to race driver binding. +cozystack_nvidia_vgpu_wait_seconds: 120 + +# Where the vGPU host driver installs sriov-manage. Also the unit's +# ConditionPathExists: absent means either no host-installed driver or a +# gpu-operator-managed vGPU Manager, and the unit skips. +cozystack_nvidia_vgpu_sriov_manage: /usr/lib/nvidia/sriov-manage + +# Driver root that gpu-operator's driver container mounts on the host. +# Present at the same time as a host-installed driver leaves ownership +# of the GPUs ambiguous, and the unit declines rather than guess. +cozystack_nvidia_vgpu_operator_driver_root: /run/nvidia/driver + +# Where the unit looks up PCI devices. There is no reason to change this +# on a real host; it exists so the boot script's stages can be exercised +# against a fake device tree in tests instead of only on GPU hardware. +cozystack_nvidia_vgpu_pci_root: /sys/bus/pci/devices diff --git a/roles/nvidia_vgpu_host/handlers/main.yml b/roles/nvidia_vgpu_host/handlers/main.yml new file mode 100644 index 0000000..4182c51 --- /dev/null +++ b/roles/nvidia_vgpu_host/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for the vGPU host restore unit + ansible.builtin.systemd: + daemon_reload: true diff --git a/roles/nvidia_vgpu_host/tasks/main.yml b/roles/nvidia_vgpu_host/tasks/main.yml new file mode 100644 index 0000000..023dacf --- /dev/null +++ b/roles/nvidia_vgpu_host/tasks/main.yml @@ -0,0 +1,170 @@ +--- +# Both paths sit inside one gated block each. A per-task `when` can be +# forgotten on the next task added; a single gate cannot. +- name: Restore NVIDIA vGPU host state across reboots + when: cozystack_enable_nvidia_vgpu_host | default(false) | bool + block: + # GPU indices shift on PCI, BIOS and kernel re-enumeration, so an + # index that named the right card yesterday can name a different one + # after a firmware update. Only a PCI address or a GPU UUID is + # accepted, and anything else is rejected here rather than handed to + # nvidia-smi on the host. The vgpu_profiles type check has to pass + # before the next task's dict2items runs, or an operator who wrote a + # list gets a template traceback instead of this message. + - name: Validate declared vGPU device addresses and flags + ansible.builtin.assert: + that: + - item.address is defined + - item.address is string + - >- + item.address is match('^(([0-9a-fA-F]{4}|[0-9a-fA-F]{8}):)?[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]$') + or item.address is match('^GPU-[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$') + - (item.sriov is not defined) or (item.sriov is boolean) + - (item.mig is not defined) or (item.mig is boolean) + - (item.vgpu_profile is not defined) or (item.vgpu_profile is integer and item.vgpu_profile > 0) + - (item.vgpu_profiles is not defined) or (item.vgpu_profiles is mapping) + - >- + (item.sriov | default(false)) or (item.mig | default(false)) + or (item.vgpu_profile is defined) + or ((item.vgpu_profiles | default({})) | length > 0) + fail_msg: >- + Invalid entry in cozystack_nvidia_vgpu_devices: + {{ item | to_json }}. + Each entry needs an `address` that is either a PCI address + (0000:41:00.0, with an optional 4- or 8-digit domain) or a GPU + UUID (GPU-); a bare GPU index is rejected because + indices are not stable across re-enumeration. `sriov` and + `mig` must be booleans and `vgpu_profiles` a mapping of VF + address to profile id, whose values are positive: 0 is not a + profile id. An entry must also ask for something, so set + `sriov` or `mig` to true, give a `vgpu_profile`, or give a + non-empty `vgpu_profiles`, or leave the card out. + loop: "{{ cozystack_nvidia_vgpu_devices }}" + loop_control: + label: "{{ item.address | default('') }}" + + # 0000:41:00.5 and 41:00.5 name the same function, so two spellings + # of one VF collapse to a single key when the script re-keys its + # override map and one of the two profiles is silently discarded. + # Rejected rather than resolved: there is no reading of the + # operator's intent that makes either the obvious winner. + - name: Reject duplicate virtual-function addresses across profile maps + ansible.builtin.assert: + that: + - _cozystack_vgpu_vfs | length == _cozystack_vgpu_vfs | unique | length + fail_msg: >- + cozystack_nvidia_vgpu_devices names the same virtual function + more than once across its vgpu_profiles maps, counting + equivalent PCI address spellings as the same function: + {{ cozystack_nvidia_vgpu_devices | selectattr('vgpu_profiles', 'defined') + | map(attribute='vgpu_profiles') | map('list') | flatten | list }}. + Give each function one entry. + vars: + _cozystack_vgpu_vfs: >- + {{ cozystack_nvidia_vgpu_devices + | selectattr('vgpu_profiles', 'defined') + | map(attribute='vgpu_profiles') | map('list') | flatten + | map('lower') + | map('regex_replace', _cozystack_vgpu_bdf_domain_re, + _cozystack_vgpu_bdf_domain_sub) + | map('regex_replace', _cozystack_vgpu_bdf_bare_re, + _cozystack_vgpu_bdf_bare_sub) + | list }} + + # The same collapse one level up. Two entries naming one card each + # emit their own stage calls, so the later call overwrites the + # earlier one's profiles, and the per-PF override counts collide in + # the same way. Equivalent PCI-address spellings count as one card; + # different PCI domains do not. + - name: Reject duplicate device addresses + ansible.builtin.assert: + that: + - _cozystack_vgpu_addresses | length == _cozystack_vgpu_addresses | unique | length + fail_msg: >- + cozystack_nvidia_vgpu_devices names the same GPU more than + once, counting equivalent PCI address spellings as the same + card: {{ cozystack_nvidia_vgpu_devices | map(attribute='address') | list }}. + Give each card one entry. + vars: + _cozystack_vgpu_addresses: >- + {{ cozystack_nvidia_vgpu_devices + | map(attribute='address') | map('lower') + | map('regex_replace', _cozystack_vgpu_bdf_domain_re, + _cozystack_vgpu_bdf_domain_sub) + | map('regex_replace', _cozystack_vgpu_bdf_bare_re, + _cozystack_vgpu_bdf_bare_sub) + | list }} + + - name: Validate declared per-VF vGPU profile assignments + ansible.builtin.assert: + that: + - >- + item.key is match('^(([0-9a-fA-F]{4}|[0-9a-fA-F]{8}):)?[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]$') + - item.value is integer and item.value > 0 + fail_msg: >- + Invalid per-VF vGPU profile assignment {{ item.key | to_json }}: + {{ item.value | to_json }}. Keys are virtual-function PCI + addresses and values are numeric vGPU type ids as listed in + the VF's creatable_vgpu_types. + loop: >- + {{ cozystack_nvidia_vgpu_devices + | selectattr('vgpu_profiles', 'defined') + | map(attribute='vgpu_profiles') + | map('dict2items') | flatten }} + loop_control: + label: "{{ item.key }}" + + - name: Install the NVIDIA vGPU host restore script + ansible.builtin.template: + src: cozystack-nvidia-vgpu-restore.sh.j2 + dest: /usr/local/sbin/cozystack-nvidia-vgpu-restore + owner: root + group: root + mode: "0755" + + - name: Install the NVIDIA vGPU host restore unit + ansible.builtin.template: + src: cozystack-nvidia-vgpu-restore.service.j2 + dest: /etc/systemd/system/cozystack-nvidia-vgpu-restore.service + owner: root + group: root + mode: "0644" + notify: Reload systemd for the vGPU host restore unit + + # Enabled, not started. Creating virtual functions or enabling MIG + # mode changes hardware state that running VMs depend on, so the + # change lands at the next boot; an operator who wants it now starts + # the unit inside a maintenance window. `systemctl enable` reads the + # unit file from disk, so it does not need the handler's reload to + # have run first. + - name: Enable the NVIDIA vGPU host restore unit for the next boot + ansible.builtin.systemd: + name: cozystack-nvidia-vgpu-restore.service + enabled: true + +# Turning the toggle off has to undo a previous run. Without this, a host +# that was enabled once keeps its unit enabled and keeps changing GPU +# state at every boot from whatever device list was current then, so the +# host no longer matches the toggle. +- name: Remove the NVIDIA vGPU host restore unit when disabled + when: not (cozystack_enable_nvidia_vgpu_host | default(false) | bool) + block: + - name: Check whether the vGPU host restore unit is installed + ansible.builtin.stat: + path: /etc/systemd/system/cozystack-nvidia-vgpu-restore.service + register: _cozystack_vgpu_unit + + - name: Disable the NVIDIA vGPU host restore unit + ansible.builtin.systemd: + name: cozystack-nvidia-vgpu-restore.service + enabled: false + when: _cozystack_vgpu_unit.stat.exists + + - name: Remove the NVIDIA vGPU host restore unit and script + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/cozystack-nvidia-vgpu-restore.service + - /usr/local/sbin/cozystack-nvidia-vgpu-restore + notify: Reload systemd for the vGPU host restore unit diff --git a/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.service.j2 b/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.service.j2 new file mode 100644 index 0000000..a46c95d --- /dev/null +++ b/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.service.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=Restore NVIDIA vGPU host state (SR-IOV virtual functions, MIG mode, vGPU profiles) +Documentation=https://github.com/cozystack/ansible-cozystack + +# Absent on a host with no NVIDIA driver installed, and on a host where +# gpu-operator manages the vGPU Manager: there the driver root is +# {{ cozystack_nvidia_vgpu_operator_driver_root }} and the container's +# own entrypoint enables virtual functions. Systemd reports the unit as +# skipped on a failed condition, not as failed. +ConditionPathExists={{ cozystack_nvidia_vgpu_sriov_manage }} + +# sriov-manage is documented to fail while the Virtual GPU Manager is +# still initialising, logging "NVRM: Timeout occurred in event processing +# by vgpu_mgr daemon" on NVLink systems, so this runs after the driver's +# own daemons rather than before them. The script also retries, which is +# what actually closes the race; After= naming a unit that is not +# installed is a no-op. +After=nvidia-vgpud.service nvidia-vgpu-mgr.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/cozystack-nvidia-vgpu-restore +# The driver wait, plus a per-card allowance. Both retrying parts are +# bounded per card: virtual-function creation, and the wait for a +# function's nvidia sysfs group, which is one budget per card rather than +# one per function. A fixed margin would run out on a host with several +# cards and systemd would kill the script before it could log why. +TimeoutStartSec={{ (cozystack_nvidia_vgpu_wait_seconds | int) + 120 + (cozystack_nvidia_vgpu_devices | length) * 60 }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.sh.j2 b/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.sh.j2 new file mode 100644 index 0000000..f5c7312 --- /dev/null +++ b/roles/nvidia_vgpu_host/templates/cozystack-nvidia-vgpu-restore.sh.j2 @@ -0,0 +1,492 @@ +#!/usr/bin/env bash +# Rendered by the nvidia_vgpu_host role. Edit the role, not this file. +# +# Restores the vGPU-relevant GPU state that a reboot drops on a host +# where the NVIDIA vGPU host driver is installed directly: SR-IOV +# virtual functions, MIG mode, and per-VF vGPU profiles. Acts only on +# the GPUs the operator named, and declines rather than guess whenever a +# precondition is not decidable. +# +# Virtual functions are documented as not surviving a reboot: "The +# virtual functions for the physical GPU in the sysfs file system are +# disabled after the hypervisor host is rebooted or if the driver is +# reloaded or upgraded", Virtual GPU Software User Guide, "Preparing the +# Virtual Function for an NVIDIA vGPU that Supports SR-IOV on a Linux +# with KVM Hypervisor". The same section requires the administrator to +# create them with sriov-manage and no other means. +# +# shellcheck disable=SC2317,SC2329 +# Helpers and stage functions are emitted unconditionally while the calls +# to them are emitted per declared flag, so on any configuration that +# does not use every stage some functions are genuinely never invoked and +# some code is genuinely unreachable. Both notices are artifacts of that +# rendering, not defects, and suppressing them here keeps the lint gate +# meaningful for every device shape rather than only the fullest one. +{% set mig_devices = cozystack_nvidia_vgpu_devices | selectattr('mig', 'defined') | selectattr('mig') | list %} +{% set sriov_devices = cozystack_nvidia_vgpu_devices | selectattr('sriov', 'defined') | selectattr('sriov') | list %} +{% set profile_devices = cozystack_nvidia_vgpu_devices | selectattr('vgpu_profile', 'defined') | list %} +{% set override_devices = cozystack_nvidia_vgpu_devices | selectattr('vgpu_profiles', 'defined') | selectattr('vgpu_profiles') | rejectattr('vgpu_profile', 'defined') | list %} +{% set profile_all = profile_devices + override_devices %} + +set -euo pipefail + +readonly SRIOV_MANAGE={{ cozystack_nvidia_vgpu_sriov_manage | quote }} +readonly OPERATOR_DRIVER_ROOT={{ cozystack_nvidia_vgpu_operator_driver_root | quote }} +readonly WAIT_SECONDS={{ cozystack_nvidia_vgpu_wait_seconds | int }} +readonly PCI_ROOT={{ cozystack_nvidia_vgpu_pci_root | quote }} + +# NVML defines exactly two host vGPU modes, NVML_HOST_VGPU_MODE_SRIOV and +# NVML_HOST_VGPU_MODE_NON_SRIOV, which nvidia-smi prints as "SR-IOV" and +# "Non SR-IOV". Compared for equality, never by substring: "Non SR-IOV" +# contains "SR-IOV". +readonly SRIOV_MODE="SR-IOV" + +# sriov-manage is documented to fail while the Virtual GPU Manager is +# still initialising ("NVRM: Timeout occurred in event processing by +# vgpu_mgr daemon" on NVLink systems), so the call is retried. NVIDIA's +# own driver container retries the same call for the same reason. +readonly SRIOV_RETRIES=5 +readonly RETRY_SLEEP=3 + +# Set when work the operator explicitly declared could not be completed. +# That includes a declared GPU that is not present on this host: the +# operator named a card, the card is not here, and the state it asked +# for was not restored. Copying the README's example address without +# editing it lands exactly there, and a green unit would be the wrong +# answer. Preconditions do not set it: not acting is a valid outcome, +# failing to do declared work is not. Every stage function returns 0 regardless, so +# a failure travels in this variable and never in the exit status of a +# bare top-level call, where it would end the run under errexit before +# the remaining GPUs were touched. +declare -i FAILED=0 + +# Deadline shared by every wait in the profile stage, set on first use. +declare -i PROFILE_WAIT_DEADLINE=0 + +# Keyed "|" so an override declared under one physical function +# can never reach another's virtual functions. Rendered with the +# addresses exactly as the operator wrote them and re-keyed below +# through normalise_bdf, so a lookup matches whichever equivalent +# PCI-address spelling was used. +declare -A VF_PROFILE_OVERRIDE_DECLARED=( +{% for device in cozystack_nvidia_vgpu_devices %} +{% for vf, profile in (device.vgpu_profiles | default({})).items() %} + [{{ ((device.address | lower) ~ '|' ~ (vf | lower)) | quote }}]={{ profile | int | string | quote }} +{% endfor %} +{% endfor %} +) +declare -A VF_PROFILE_OVERRIDE=() + +# How many per-VF overrides the operator declared for each physical +# function. Compared against how many actually matched a function, so an +# override naming a VF that does not belong to this PF is reported +# instead of silently doing nothing. +declare -A VF_OVERRIDE_COUNT_DECLARED=( +{% for device in cozystack_nvidia_vgpu_devices if (device.vgpu_profiles | default({})) | length > 0 %} + [{{ (device.address | lower) | quote }}]={{ device.vgpu_profiles | length }} +{% endfor %} +) +declare -A VF_OVERRIDE_COUNT=() + +# GPUs the virtual-function stage declined because the driver does not +# report SR-IOV mode. The profile stage stays quiet about those rather +# than failing the unit for a card it was already told to leave alone. +declare -A VF_STAGE_DECLINED=() + +# Every printf tail below carries `|| true`. A failed write to the +# journal would otherwise return non-zero from the function, and under +# errexit a bare `log ...` call would end the run at the exact moment it +# was trying to explain itself. +log() { + printf 'cozystack-nvidia-vgpu-restore: %s\n' "$*" || true +} + +trim() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + printf '%s' "${value%"${value##*[![:space:]]}"}" || true +} + +# Rewrites any spelling of a PCI address to the four-digit-domain form +# sysfs uses, so operator-written addresses, nvidia-smi's eight-digit +# output and sysfs names compare equal. The domain is canonicalised, not +# discarded: on a host with more than one PCI segment, 0000:41:00.0 and +# 0001:41:00.0 are different cards, and collapsing them would let the +# script act on hardware nobody declared. A GPU UUID has no domain and +# is only lowercased. +normalise_bdf() { + local value="${1,,}" + if [[ "$value" =~ ^([0-9a-f]+):([0-9a-f]{2}:[0-9a-f]{2}\.[0-7])$ ]]; then + printf '%04x:%s' "$((16#${BASH_REMATCH[1]}))" "${BASH_REMATCH[2]}" || true + return 0 + fi + if [[ "$value" =~ ^[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$ ]]; then + printf '0000:%s' "$value" || true + return 0 + fi + printf '%s' "$value" || true +} + +# Re-keyed through normalise_bdf so the rendered keys and the runtime +# lookups go through one normaliser rather than two that can drift +# apart. Keys are "|", and both halves are normalised here. +# Iterating the keys of an empty associative array is safe under set -u +# from bash 4.4, which every distro this collection supports ships. +# Do not reduce this to a length test: the template renderer reads a +# literal ${ followed by # as the start of a Jinja comment. +for declared_key in "${!VF_PROFILE_OVERRIDE_DECLARED[@]}"; do + VF_PROFILE_OVERRIDE["$(normalise_bdf "${declared_key%%|*}")|$(normalise_bdf "${declared_key##*|}")"]="${VF_PROFILE_OVERRIDE_DECLARED[$declared_key]}" +done + +for declared_pf in "${!VF_OVERRIDE_COUNT_DECLARED[@]}"; do + VF_OVERRIDE_COUNT["$(normalise_bdf "$declared_pf")"]="${VF_OVERRIDE_COUNT_DECLARED[$declared_pf]}" +done + +# Maps a declared PCI address or GPU UUID onto the index nvidia-smi -i +# wants, plus the bus id as nvidia-smi spells it. Indices are never +# accepted from configuration because they shift on re-enumeration, but +# nvidia-smi still needs one, so it is resolved here at every boot. +resolve_gpu() { + local want="${1,,}" want_bdf index bus uuid + want_bdf="$(normalise_bdf "$want")" + while IFS=, read -r index bus uuid; do + index="$(trim "$index")" + bus="$(trim "$bus")" + uuid="$(trim "$uuid")" + if [ "${uuid,,}" = "$want" ] || [ "$(normalise_bdf "$bus")" = "$want_bdf" ]; then + # Trailing newline is load-bearing: callers consume this with + # `read`, which reports failure on EOF without one even after it + # has set the variables. + printf '%s %s\n' "$index" "$bus" + return 0 + fi + done < <(nvidia-smi --query-gpu=index,pci.bus_id,uuid --format=csv,noheader 2>/dev/null) + return 1 +} + +sysfs_for_bdf() { + local want device + want="$(normalise_bdf "$1")" + for device in "${PCI_ROOT}"/*; do + [ -e "$device" ] || continue + if [ "$(normalise_bdf "$(basename "$device")")" = "$want" ]; then + printf '%s' "$device" + return 0 + fi + done + return 1 +} + +# Bounded, and deliberately short: this closes the population race, it +# does not wait for hardware that is never coming. Returns non-zero when +# the node never appears, so every caller must guard the call; a bare +# call would end the whole run under errexit. +wait_for_vgpu_node() { + local node="$1" attempt + [ -e "$node" ] && return 0 + # One budget per card, not one per function. A card exposes as many + # functions as the driver created, so a per-function wait multiplies by + # a number this script does not choose and cannot bound. The card count + # is the operator's, and the unit's TimeoutStartSec already scales with + # it. The caller resets this for each card. + if [ "$PROFILE_WAIT_DEADLINE" -eq 0 ]; then + PROFILE_WAIT_DEADLINE=$((SECONDS + SRIOV_RETRIES * RETRY_SLEEP)) + fi + for attempt in $(seq 1 "$SRIOV_RETRIES"); do + [ -e "$node" ] && return 0 + [ "$SECONDS" -ge "$PROFILE_WAIT_DEADLINE" ] && return 1 + sleep "$RETRY_SLEEP" + done + [ -e "$node" ] +} + +host_vgpu_mode() { + nvidia-smi -q -i "$1" 2>/dev/null \ + | sed -n 's/^[[:space:]]*Host VGPU Mode[[:space:]]*:[[:space:]]*//p' \ + | head -n 1 \ + | tr -d '\r' +} + +{% if not (mig_devices or sriov_devices or profile_all) %} +# Nothing was declared, so there is no work and no reason to look at the +# hardware at all. Rendered before the preconditions on purpose: the +# driver wait below is a hard failure, and an inert configuration must +# never reach it. Validation requires every entry to ask for something, +# so this branch is reached through an empty device list. +log "no GPUs declared in cozystack_nvidia_vgpu_devices; nothing to restore" +exit 0 +{% endif %} + +# ---- preconditions ---- + +# No host-installed vGPU driver. Also the state on a host with no NVIDIA +# GPU at all, and on one where gpu-operator manages the vGPU Manager: the +# container's driver root holds the driver and its entrypoint enables the +# virtual functions itself. The unit carries the same check as +# ConditionPathExists, so normally systemd skips it before this runs. +if [ ! -x "$SRIOV_MANAGE" ]; then + log "no host-installed vGPU driver at ${SRIOV_MANAGE}; nothing to restore" + exit 0 +fi + +# A host-installed driver and an operator-managed driver root at the same +# time leaves it undecidable which one owns the GPUs. Declining covers +# the MIG stage too, not only the VF stage: enabling MIG mode does not +# race the container's VF setup, but it would change hardware state +# underneath whatever the operand had already built. +if [ -x "${OPERATOR_DRIVER_ROOT}/usr/lib/nvidia/sriov-manage" ]; then + log "refusing to act: a host-installed driver at ${SRIOV_MANAGE} and an operator-managed driver root at ${OPERATOR_DRIVER_ROOT} are both present, so GPU ownership is ambiguous" + exit 0 +fi + +# The only precondition that exits non-zero; every other one above +# treats not acting as a valid outcome. Reaching here already proved a +# host vGPU driver is installed and the operator named GPUs on this host, +# so a driver that never becomes usable is a fault the operator has to +# see rather than a no-op to log quietly. +deadline=$((SECONDS + WAIT_SECONDS)) +until [ -n "$(nvidia-smi -L 2>/dev/null)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "nvidia-smi did not enumerate a GPU within ${WAIT_SECONDS}s; the driver is installed but not usable" + exit 1 + fi + sleep 2 +done + +# ---- STAGE 1: MIG mode ---- +# +# Runs before any virtual function exists. MIG mode is a whole-GPU +# property and settles before the GPU is subdivided. +# +# This stage never resets a GPU. MIG mode is persistent across reboots on +# Ampere, where setting it needs a reset, and needs no reset on Hopper +# and later, where it is not persistent, so boot-time restoration never +# requires one. A reset destroys every consumer of the card, which on a +# vGPU host means the tenants' running VMs. +ensure_mig_mode() { + local declared="$1" index bus mode output + if ! read -r index bus < <(resolve_gpu "$declared"); then + log "MIG: ${declared} is not present on this host, so its MIG mode was not restored" + FAILED=1 + return 0 + fi + mode="$(trim "$(nvidia-smi --query-gpu=mig.mode.current --format=csv,noheader -i "$index" 2>/dev/null || true)")" + case "$mode" in + Enabled) + log "MIG: ${declared} already reports MIG mode Enabled" + return 0 + ;; + Disabled) ;; + *) + log "MIG: ${declared} declined, unrecognised MIG mode; observed mode: '${mode}'. The GPU is not MIG-capable, or the driver reported no mode" + return 0 + ;; + esac + if ! output="$(nvidia-smi -i "$index" -mig 1 2>&1)"; then + log "MIG: enabling MIG mode on ${declared} failed; nvidia-smi said: ${output}" + FAILED=1 + return 0 + fi + mode="$(trim "$(nvidia-smi --query-gpu=mig.mode.current --format=csv,noheader -i "$index" 2>/dev/null || true)")" + if [ "$mode" != "Enabled" ]; then + log "MIG: ${declared} still reports '${mode}' after enabling, so a reset or reboot is pending. This script never resets a GPU; do it out of band in a maintenance window. nvidia-smi said: ${output}" + FAILED=1 + return 0 + fi + log "MIG: enabled MIG mode on ${declared}" + return 0 +} + +# ---- STAGE 2: SR-IOV virtual functions ---- + +ensure_vfs() { + local declared="$1" index bus sysfs canonical mode numvfs output attempt + if ! read -r index bus < <(resolve_gpu "$declared"); then + log "VF: ${declared} is not present on this host, so its virtual functions were not restored" + FAILED=1 + return 0 + fi + # Ask the driver to state its own operating mode rather than derive + # SR-IOV capability from the PCI capability bits. Hardware exists that + # advertises virtual functions in sysfs yet drives vGPU through the + # legacy mdev path and never creates them, and enabling functions on + # such a card can unbind a physical function carrying live vGPUs. + mode="$(trim "$(host_vgpu_mode "$index")")" + if [ "$mode" != "$SRIOV_MODE" ]; then + log "VF: ${declared} declined, host vGPU mode is not SR-IOV or was unrecognised; observed mode: '${mode}'. NVML defines only 'SR-IOV' and 'Non SR-IOV'; an empty value means the driver reported no such field" + VF_STAGE_DECLINED["$(normalise_bdf "$declared")"]=1 + return 0 + fi + if ! sysfs="$(sysfs_for_bdf "$bus")"; then + log "VF: no sysfs entry for ${declared} (bus ${bus}), so its virtual functions were not restored" + FAILED=1 + return 0 + fi + # Suppresses a redundant write, and nothing else. It is not capability + # inference: a non-zero count means the work is already done, while a + # zero count never on its own concludes that functions are missing and + # must be created. Only the operator's explicit sriov flag decides + # that, and only after the driver confirmed SR-IOV mode above. + # sriov-manage has no query form, so sysfs is the only place to read + # the current count from. + numvfs="$(trim "$(cat "${sysfs}/sriov_numvfs" 2>/dev/null || printf '0')")" + if [ "${numvfs:-0}" != "0" ]; then + log "VF: ${declared} already has ${numvfs} virtual functions; nothing to do" + return 0 + fi + # sriov-manage is documented taking ::., + # the four-digit-domain form sysfs uses. nvidia-smi pads the domain to + # eight digits, so pass the sysfs name rather than what nvidia-smi + # printed. + canonical="$(basename "$sysfs")" + for attempt in $(seq 1 "$SRIOV_RETRIES"); do + if output="$("$SRIOV_MANAGE" -e "$canonical" 2>&1)"; then + # A zero exit is not proof the functions exist: some driver + # branches return success even when creation failed, which is why + # NVIDIA's own driver container carries a workaround for the same + # bug. Confirm against sysfs before reporting success. + numvfs="$(trim "$(cat "${sysfs}/sriov_numvfs" 2>/dev/null || printf '0')")" + if [ "${numvfs:-0}" != "0" ]; then + log "VF: enabled ${numvfs} virtual functions on ${declared}" + return 0 + fi + output="sriov-manage reported success but sriov_numvfs is still 0" + fi + if [ "$attempt" -lt "$SRIOV_RETRIES" ]; then + sleep "$RETRY_SLEEP" + fi + done + log "VF: enabling virtual functions on ${declared} failed after ${SRIOV_RETRIES} attempts; sriov-manage said: ${output}" + FAILED=1 + return 0 +} + +# ---- STAGE 3: per-VF vGPU profiles ---- +# +# Runs last, because a virtual function has to exist before its profile +# can be written. current_vgpu_type resets on PCI re-enumeration, so +# without this a restored function advertises no profile and no VM can +# request it. +ensure_vf_profiles() { + local declared="$1" fallback="$2" + local bus sysfs prefix virtfn vf_path vf_bdf profile current node + local -i explicit=0 + local -i seen=0 applied_overrides=0 declared_overrides=0 + # Each card waits on its own budget. Sharing one across cards let the + # first card that had to wait spend all of it, after which a function + # named by hand on a later card failed for a node that was seconds from + # appearing. + PROFILE_WAIT_DEADLINE=0 + if ! read -r _ bus < <(resolve_gpu "$declared"); then + log "profile: ${declared} is not present on this host, so no vGPU profile was written" + FAILED=1 + return 0 + fi + if ! sysfs="$(sysfs_for_bdf "$bus")"; then + log "profile: no sysfs entry for ${declared} (bus ${bus}), so no vGPU profile was written" + FAILED=1 + return 0 + fi + prefix="$(normalise_bdf "$declared")" + declared_overrides="${VF_OVERRIDE_COUNT[$prefix]:-0}" + shopt -s nullglob + for virtfn in "${sysfs}"/virtfn*; do + seen=$((seen + 1)) + # Unguarded this would abort the whole run under errexit, skipping + # every GPU after this one, when a function disappears + # mid-enumeration or its symlink cannot be resolved. + if ! vf_path="$(readlink -f "$virtfn")"; then + log "profile: could not resolve ${virtfn}; skipping" + FAILED=1 + continue + fi + vf_bdf="$(normalise_bdf "$(basename "$vf_path")")" + profile="${VF_PROFILE_OVERRIDE[${prefix}|${vf_bdf}]:-}" + if [ -n "$profile" ]; then + explicit=1 + applied_overrides=$((applied_overrides + 1)) + else + explicit=0 + profile="$fallback" + fi + [ -n "$profile" ] || continue + node="${vf_path}/nvidia/current_vgpu_type" + # The driver populates a function's nvidia sysfs group asynchronously + # after the function itself appears, and this stage runs immediately + # behind the one that created them, so a missing node here is usually + # that race rather than a permanent state. + # Guarded: the helper returns non-zero when the node never appears, + # which is precisely the case the branch below exists to report. An + # unguarded call ends the run there instead, with nothing logged and + # every later function and GPU skipped. + wait_for_vgpu_node "$node" || true + if [ ! -w "$node" ]; then + if [ "$explicit" -eq 1 ]; then + # The operator named this function, so failing to write it is + # unapplied declared work. + log "profile: ${vf_bdf} was named explicitly but has no writable current_vgpu_type, so vGPU type ${profile} was not applied" + FAILED=1 + else + # Swept by the per-PF shorthand rather than named, so a function + # that is not vGPU-capable is a legitimate skip. + log "profile: ${vf_bdf} has no writable current_vgpu_type after waiting; either the function is not vGPU-capable or the driver never populated its nvidia sysfs group; skipping" + fi + continue + fi + current="$(trim "$(cat "$node" 2>/dev/null || printf '0')")" + if [ "$current" = "$profile" ]; then + continue + fi + # The subshell catches bash's own redirection error too; a plain + # `printf ... 2>/dev/null > file` only silences printf's stderr and + # still leaks "No such file or directory" into the journal. + if ( printf '%s\n' "$profile" > "$node" ) 2>/dev/null; then + log "profile: set vGPU type ${profile} on ${vf_bdf}" + elif [ "$explicit" -eq 1 ]; then + log "profile: ${vf_bdf} was named explicitly and rejected vGPU type ${profile}" + FAILED=1 + else + # The per-PF shorthand is written to every function the card + # exposes, and sriov-manage creates the card's maximum count while + # a type's instance count is set by frame-buffer division. So the + # card runs out of that type partway down the list and rejects the + # rest. That is the shorthand working as documented, not a fault, + # and failing here would leave the unit red at every boot on the + # configuration the README recommends. + log "profile: ${vf_bdf} rejected vGPU type ${profile} from the per-PF shorthand, which usually means the card is at that type's instance limit; skipping" + fi + done + shopt -u nullglob + if [ "$seen" -eq 0 ]; then + if [ -n "${VF_STAGE_DECLINED[$prefix]:-}" ]; then + # Already reported once by the virtual-function stage. Failing here + # too would give an operator on a legacy mdev card a red unit at + # every boot for a card the script was correct to leave alone. + log "profile: ${declared} has no virtual functions because its virtual-function stage declined it; nothing to write" + return 0 + fi + log "profile: ${declared} has no virtual functions, so no vGPU profile could be written" + FAILED=1 + return 0 + fi + if [ "$applied_overrides" -lt "$declared_overrides" ]; then + log "profile: ${declared} declares ${declared_overrides} per-VF overrides but ${applied_overrides} matched a function on this GPU; check those addresses belong to it" + FAILED=1 + fi + return 0 +} + +{% for device in mig_devices %} +ensure_mig_mode {{ device.address | quote }} +{% endfor %} +{% for device in sriov_devices %} +ensure_vfs {{ device.address | quote }} +{% endfor %} +{% for device in profile_devices %} +ensure_vf_profiles {{ device.address | quote }} {{ device.vgpu_profile | int | string | quote }} +{% endfor %} +{% for device in override_devices %} +ensure_vf_profiles {{ device.address | quote }} '' +{% endfor %} + +exit "$FAILED" diff --git a/roles/nvidia_vgpu_host/vars/main.yml b/roles/nvidia_vgpu_host/vars/main.yml new file mode 100644 index 0000000..e5d1152 --- /dev/null +++ b/roles/nvidia_vgpu_host/vars/main.yml @@ -0,0 +1,15 @@ +--- +# Canonicalisation for PCI addresses, shared by every check that compares +# them. One definition on purpose: the two duplicate checks previously +# carried their own regexes, they disagreed about the domain, and the +# disagreement was invisible because each one looked correct alone. +# +# The rule is canonicalise, never delete. Stripping the domain aliases +# 0000:41:00.0 and 0001:41:00.0 onto one key, and those are different +# cards on a host with more than one PCI segment. Leading zeros go, a +# non-zero domain stays, and a bare address is treated as domain zero, +# which is the same rule normalise_bdf applies in the boot script. +_cozystack_vgpu_bdf_domain_re: '^0*([0-9a-f]*):([0-9a-f]{2}:[0-9a-f]{2}\.[0-7])$' +_cozystack_vgpu_bdf_domain_sub: '\1:\2' +_cozystack_vgpu_bdf_bare_re: '^([0-9a-f]{2}:[0-9a-f]{2}\.[0-7])$' +_cozystack_vgpu_bdf_bare_sub: ':\1' diff --git a/tests/test-nvidia-vgpu-host-idempotency.yml b/tests/test-nvidia-vgpu-host-idempotency.yml new file mode 100644 index 0000000..e9d3008 --- /dev/null +++ b/tests/test-nvidia-vgpu-host-idempotency.yml @@ -0,0 +1,28 @@ +--- +# Applies the role and nothing else, so a second run of this playbook +# reports changed=0 if the role settles. Kept separate from +# test-nvidia-vgpu-host.yml, which deliberately mutates host state to +# exercise the script's refusal paths and therefore reports changes on +# every run. + +- name: Apply the NVIDIA vGPU host role and nothing else + hosts: localhost + connection: local + become: true + gather_facts: true + vars: + cozystack_enable_nvidia_vgpu_host: true + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + sriov: true + mig: true + vgpu_profile: 1155 + vgpu_profiles: + "0000:41:00.5": 1160 + - address: "GPU-12345678-1234-1234-1234-123456789abc" + sriov: true + + tasks: + - name: Apply the role + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host diff --git a/tests/test-nvidia-vgpu-host-stages.yml b/tests/test-nvidia-vgpu-host-stages.yml new file mode 100644 index 0000000..3f5c23a --- /dev/null +++ b/tests/test-nvidia-vgpu-host-stages.yml @@ -0,0 +1,714 @@ +--- +# Runs the restore script's three stages for real against a fake GPU. +# +# The other test playbook covers the refusal paths, which is everything +# the script does on a host with no GPU. That leaves the stages +# themselves untested, and they are where the consequences live: a wrong +# argument to sriov-manage, or a profile that silently never gets +# written, produces a VM that cannot start. Faking nvidia-smi and +# sriov-manage on PATH and pointing the script at a fake PCI tree +# exercises them on any runner. +# +# Everything lives under a temporary directory, including the driver +# paths and the PCI root, so this is safe on a host that has the real +# vGPU driver installed. + +- name: Exercise the restore script's stages against a fake GPU + hosts: localhost + connection: local + become: true + gather_facts: true + vars: + _tmp: /tmp/cozystack-vgpu-stage-test + _bin: /tmp/cozystack-vgpu-stage-test/bin + _pci: /tmp/cozystack-vgpu-stage-test/pci + _pf: /tmp/cozystack-vgpu-stage-test/pci/0000:41:00.0 + _script: /usr/local/sbin/cozystack-nvidia-vgpu-restore + _uuid: GPU-12345678-1234-1234-1234-123456789abc + cozystack_enable_nvidia_vgpu_host: true + cozystack_nvidia_vgpu_sriov_manage: /tmp/cozystack-vgpu-stage-test/bin/sriov-manage + cozystack_nvidia_vgpu_operator_driver_root: /tmp/cozystack-vgpu-stage-test/absent + cozystack_nvidia_vgpu_pci_root: /tmp/cozystack-vgpu-stage-test/pci + cozystack_nvidia_vgpu_wait_seconds: 5 + + tasks: + - name: Start from a clean temporary tree + ansible.builtin.file: + path: "{{ _tmp }}" + state: absent + + - name: Create the fake PCI tree + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ _bin }}" + - "{{ _pf }}" + - "{{ _pci }}/0000:41:00.4/nvidia" + - "{{ _pci }}/0000:41:00.5/nvidia" + # Third function, deliberately without an nvidia/ group: the + # driver populates that group asynchronously, and the profile + # stage has to survive one that never appears. + - "{{ _pci }}/0000:41:00.6" + - "{{ _pci }}/0000:41:00.7/nvidia" + # A second PCI domain holding a card with the same bus address. + # Different hardware, and the script must never confuse the two. + - "{{ _pci }}/0001:41:00.0" + - "{{ _pci }}/0001:41:00.4/nvidia" + + - name: Start the physical function with no virtual functions + ansible.builtin.copy: + dest: "{{ _pf }}/sriov_numvfs" + content: "0\n" + mode: "0644" + + - name: Give the second domain's card a virtual-function count + ansible.builtin.copy: + dest: "{{ _pci }}/0001:41:00.0/sriov_numvfs" + content: "1\n" + mode: "0644" + + - name: Link the second domain's virtual function + ansible.builtin.file: + src: "{{ _pci }}/0001:41:00.4" + dest: "{{ _pci }}/0001:41:00.0/virtfn0" + state: link + + - name: Give the second domain's function a profile node + ansible.builtin.copy: + dest: "{{ _pci }}/0001:41:00.4/nvidia/current_vgpu_type" + content: "0\n" + mode: "0644" + + - name: Link the virtual functions under the physical function + ansible.builtin.file: + src: "{{ _pci }}/0000:41:00.{{ item.0 }}" + dest: "{{ _pf }}/virtfn{{ item.1 }}" + state: link + loop: + - [4, 0] + - [5, 1] + - [6, 2] + - [7, 3] + + - name: Give each virtual function a resettable profile node + ansible.builtin.copy: + dest: "{{ _pci }}/0000:41:00.{{ item }}/nvidia/current_vgpu_type" + content: "0\n" + mode: "0644" + loop: [4, 5] + + # A node that exists and is writable by the test for permissions but + # rejects the write itself. Standing in for the ordinary case on real + # hardware: the per-PF shorthand is written to every function the card + # exposes, and the card runs out of instances of that type partway + # down the list. The unit must not go red for that. + - name: Give one function a node that accepts no value + ansible.builtin.file: + path: "{{ _pci }}/0000:41:00.7/nvidia/current_vgpu_type" + state: directory + mode: "0755" + + # Answers only the queries the script actually makes. Reports the + # eight-digit domain that nvidia-smi really prints, so the script's + # handling of that spelling is what gets exercised. Records every + # call it receives; nothing truncates that file, so it accumulates + # across every case in this play. + - name: Install a fake nvidia-smi + ansible.builtin.copy: + dest: "{{ _bin }}/nvidia-smi" + mode: "0755" + content: | + #!/bin/bash + echo "$*" >> "{{ _tmp }}/nvidia-smi.argv" + case "$*" in + *-L*) + echo "GPU 0: NVIDIA Graphics Device (UUID: {{ _uuid }})" + echo "GPU 1: NVIDIA Graphics Device (UUID: GPU-99999999-9999-9999-9999-999999999999)" ;; + *--query-gpu=index,pci.bus_id,uuid*) + echo "0, 00000000:41:00.0, {{ _uuid }}" + echo "1, 00000001:41:00.0, GPU-99999999-9999-9999-9999-999999999999" ;; + *--query-gpu=mig.mode.current*) + cat "{{ _tmp }}/mig_mode" ;; + *-mig\ 1*) + echo "Enabled" > "{{ _tmp }}/mig_mode" ;; + *-q*) + echo " GPU Virtualization Mode" + echo " Virtualization Mode : Host VGPU" + echo " Host VGPU Mode : $(cat "{{ _tmp }}/host_vgpu_mode")" ;; + esac + + # Records its argv so the exact spelling handed to it can be + # asserted, and creates the functions the way the real one would. + - name: Install a fake sriov-manage + ansible.builtin.copy: + dest: "{{ _bin }}/sriov-manage" + mode: "0755" + content: | + #!/bin/bash + echo "$*" >> "{{ _tmp }}/sriov-manage.argv" + # Derive the target from the argument. Writing into one fixed + # card's node made every case look like the first card's, so a + # run against another card ended in retries and still passed. + echo "2" > "{{ _pci }}/$2/sriov_numvfs" + + - name: Set the fake GPU's reported state + ansible.builtin.copy: + dest: "{{ _tmp }}/{{ item.name }}" + content: "{{ item.value }}\n" + mode: "0644" + loop: + - {name: mig_mode, value: Disabled} + - {name: host_vgpu_mode, value: SR-IOV} + + - name: Render the script for the fake GPU + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + sriov: true + mig: true + vgpu_profile: 1155 + vgpu_profiles: + "0000:41:00.5": 1160 + + - name: Run every stage against the fake GPU + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _stages + changed_when: false + + - name: Read back what the stages did + ansible.builtin.slurp: + src: "{{ item }}" + register: _results + loop: + - "{{ _tmp }}/sriov-manage.argv" + - "{{ _tmp }}/mig_mode" + - "{{ _pci }}/0000:41:00.4/nvidia/current_vgpu_type" + - "{{ _pci }}/0000:41:00.5/nvidia/current_vgpu_type" + + - name: Assert a function that rejects the shorthand does not fail the unit + ansible.builtin.assert: + that: + - "'0000:41:00.7' in _stages.stdout" + - "\"at that type's instance limit\" in _stages.stdout" + fail_msg: >- + A function that rejects a profile written by the per-PF + shorthand must be logged and skipped, not counted as failed + work. Otherwise the configuration the README recommends leaves + the unit red at every boot. + stdout={{ _stages.stdout }} + + - name: Assert the function with no nvidia group is reported, not fatal + ansible.builtin.assert: + that: + - "'0000:41:00.6' in _stages.stdout" + - "'no writable current_vgpu_type after waiting' in _stages.stdout" + fail_msg: >- + A function whose nvidia sysfs group never appears must be + logged and skipped, and the run must continue to the functions + after it. An unguarded wait ends the run there instead, with + nothing logged. stdout={{ _stages.stdout }} + + - name: Assert every stage did its work + ansible.builtin.assert: + that: + - _stages.rc == 0 + # sriov-manage takes an lspci filter, and the four-digit sysfs + # spelling is the documented one and what every other known + # caller passes. nvidia-smi reports eight digits, so this + # asserts the script normalised rather than forwarding. + - (_results.results[0].content | b64decode).strip() == '-e 0000:41:00.0' + # The card in the other PCI domain was never declared, so its + # address must not appear at all. Collapsing the domain would + # send this to 0000:41:00.0 or 0001:41:00.0 interchangeably. + - "'0001:41:00.0' not in (_results.results[0].content | b64decode)" + - (_results.results[1].content | b64decode).strip() == 'Enabled' + # The function the operator did not name individually takes the + # per-PF shorthand. + - (_results.results[2].content | b64decode).strip() == '1155' + # The one named in vgpu_profiles takes its override. + - (_results.results[3].content | b64decode).strip() == '1160' + fail_msg: >- + Stage output was wrong. + rc={{ _stages.rc }}, + sriov-manage argv={{ _results.results[0].content | b64decode }}, + mig={{ _results.results[1].content | b64decode }}, + vf4={{ _results.results[2].content | b64decode }}, + vf5={{ _results.results[3].content | b64decode }}, + stdout={{ _stages.stdout }} + + # A second run must change nothing: MIG mode already Enabled, the + # functions already present, the profiles already right. + - name: Run the stages again + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _stages_again + changed_when: false + + - name: Assert the second run is a no-op that still succeeds + ansible.builtin.assert: + that: + - _stages_again.rc == 0 + - "'already reports MIG mode Enabled' in _stages_again.stdout" + - "'already has 2 virtual functions' in _stages_again.stdout" + fail_msg: >- + Re-running the script on a restored GPU must recognise the work + as already done. rc={{ _stages_again.rc }}, + stdout={{ _stages_again.stdout }} + + # An override naming a function that is not on this GPU would + # otherwise be applied to nothing and never mentioned. + - name: Render the script with an override for a function elsewhere + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + vgpu_profiles: + "0000:99:00.7": 1177 + + - name: Run the stages with a misplaced override + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _misplaced + changed_when: false + failed_when: false + + - name: Assert a misplaced override is reported rather than ignored + ansible.builtin.assert: + that: + - _misplaced.rc == 1 + - "'per-VF overrides but 0 matched' in _misplaced.stdout" + fail_msg: >- + An override naming a function that does not belong to the + declared GPU must be reported, not silently skipped. + rc={{ _misplaced.rc }}, stdout={{ _misplaced.stdout }} + + # A card whose driver does not report SR-IOV must be left alone even + # though the operator asked for virtual functions on it. + - name: Report the fake GPU as not running in SR-IOV mode + ansible.builtin.copy: + dest: "{{ _tmp }}/host_vgpu_mode" + content: "Non SR-IOV\n" + mode: "0644" + + - name: Reset the fake physical function + ansible.builtin.copy: + dest: "{{ _pf }}/sriov_numvfs" + content: "0\n" + mode: "0644" + + - name: Render the script for the non-SR-IOV case + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + sriov: true + vgpu_profile: 1155 + + - name: Truncate the recorded sriov-manage calls + ansible.builtin.copy: + dest: "{{ _tmp }}/sriov-manage.argv" + content: "" + mode: "0644" + + - name: Run the VF stage against a non-SR-IOV card + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _non_sriov + changed_when: false + + - name: Read back whether sriov-manage was called + ansible.builtin.slurp: + src: "{{ _tmp }}/sriov-manage.argv" + register: _argv_after + + - name: Assert a non-SR-IOV card is declined and left untouched + ansible.builtin.assert: + that: + - _non_sriov.rc == 0 + - "'observed mode: ' + \"'Non SR-IOV'\" in _non_sriov.stdout" + - (_argv_after.content | b64decode) | length == 0 + fail_msg: >- + A card whose driver reports a mode other than SR-IOV must be + declined with the observed value logged, and sriov-manage must + not be called for it. rc={{ _non_sriov.rc }}, + stdout={{ _non_sriov.stdout }}, + argv={{ _argv_after.content | b64decode }} + + # The reproduction of the domain-collapse bug rather than a + # restatement of correct behaviour. Declaring only the second-domain + # card is the shape that fails when normalise_bdf drops the domain: + # the call then goes to the first card instead. Asserting the first + # card is called when the first card is declared passes either way. + - name: Truncate the recorded calls before the second-domain run + ansible.builtin.copy: + dest: "{{ _tmp }}/sriov-manage.argv" + content: "" + mode: "0644" + + # This play accumulates state, and the cases below run after the one + # that reports the card as Non SR-IOV. Reset both the mode and the + # function count so the virtual-function stage actually runs here + # rather than declining, or reporting the work already done. + - name: Reset the fixture state before the second-domain run + ansible.builtin.copy: + dest: "{{ item.path }}" + content: "{{ item.content }}\n" + mode: "0644" + loop: + - {path: "{{ _pci }}/0001:41:00.0/sriov_numvfs", content: "0"} + - {path: "{{ _tmp }}/host_vgpu_mode", content: "SR-IOV"} + loop_control: + label: "{{ item.path | basename }}" + + - name: Render for the second-domain card only + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0001:41:00.0" + sriov: true + + - name: Run against the second-domain card only + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + changed_when: false + failed_when: false + + - name: Read back which card was called + ansible.builtin.slurp: + src: "{{ _tmp }}/sriov-manage.argv" + register: _second_domain_argv + + - name: Assert the declared card was the one acted on + ansible.builtin.assert: + that: + - "'0001:41:00.0' in (_second_domain_argv.content | b64decode)" + - "'-e 0000:41:00.0' not in (_second_domain_argv.content | b64decode)" + fail_msg: >- + Declaring only the card in the second PCI segment must act on + that card. Collapsing the domain sends the call to the card in + the first segment, which is hardware nobody declared. + argv={{ _second_domain_argv.content | b64decode }} + + # An override belongs to the physical function it was declared under. + # With a global key it would reach the other card's function instead. + - name: Render with an override declared under the wrong card + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + vgpu_profiles: + "0001:41:00.4": 1177 + - address: "0001:41:00.0" + vgpu_profile: 1200 + + - name: Run with an override declared under the wrong card + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + changed_when: false + failed_when: false + + - name: Read back the second-domain function's profile + ansible.builtin.slurp: + src: "{{ _pci }}/0001:41:00.4/nvidia/current_vgpu_type" + register: _scoped_profile + + - name: Assert an override cannot cross to another card's function + ansible.builtin.assert: + that: + - (_scoped_profile.content | b64decode).strip() == '1200' + fail_msg: >- + The function took 1177, an override declared under a different + physical function, instead of 1200 from its own card's + shorthand. Override keys must be scoped to the card they were + declared under. + got={{ _scoped_profile.content | b64decode }} + + # The other half of the explicit-versus-shorthand distinction. The + # shorthand half is asserted above; this is the half that must fail. + - name: Render with an explicitly named function that rejects its type + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + vgpu_profiles: + "0000:41:00.7": 1160 + + - name: Run with an explicitly named function that rejects its type + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _explicit_reject + changed_when: false + failed_when: false + + - name: Assert an explicitly named function that fails fails the unit + ansible.builtin.assert: + that: + - _explicit_reject.rc == 1 + - "'was named explicitly and rejected' in _explicit_reject.stdout" + fail_msg: >- + A function the operator named individually must fail the unit + when it will not take its profile. Only the per-PF shorthand is + allowed to skip. rc={{ _explicit_reject.rc }}, + stdout={{ _explicit_reject.stdout }} + + - name: Render with a card that is not on this host + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:99:00.0" + sriov: true + + - name: Run with a card that is not on this host + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _absent_card + changed_when: false + failed_when: false + + - name: Assert a declared card that is absent fails the unit + ansible.builtin.assert: + that: + - _absent_card.rc == 1 + - "'is not present on this host' in _absent_card.stdout" + fail_msg: >- + A card the operator declared that is not on the host is + declared work that did not happen, so it must fail rather than + report success. This is where an unedited example address + lands. rc={{ _absent_card.rc }}, stdout={{ _absent_card.stdout }} + + # The branch that keeps the profile stage quiet about a card the + # virtual-function stage already declined. It only runs when the card + # has no functions, which is the real state of a declined card, so the + # fixture has to strip the ones it was given. + - name: Strip the second domain's function + ansible.builtin.file: + path: "{{ _pci }}/0001:41:00.0/virtfn0" + state: absent + + # This play accumulates fixture state, so set every input this case + # depends on rather than inheriting it. An earlier case left the card + # reporting functions, which makes the stage report the work already + # done, and another left the mode as SR-IOV, which makes it succeed. + # Either one stops the branch under test from being entered. + - name: Set the inputs this case depends on + ansible.builtin.copy: + dest: "{{ item.path }}" + content: "{{ item.content }}\n" + mode: "0644" + loop: + - {path: "{{ _pci }}/0001:41:00.0/sriov_numvfs", content: "0"} + - {path: "{{ _tmp }}/host_vgpu_mode", content: "Non SR-IOV"} + loop_control: + label: "{{ item.path | basename }}" + + - name: Render a declined card that also declares a profile + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0001:41:00.0" + sriov: true + vgpu_profile: 1155 + + - name: Run against a declined card that also declares a profile + ansible.builtin.command: + cmd: "{{ _script }}" + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _declined_profile + changed_when: false + failed_when: false + + - name: Assert a declined card does not fail the unit for its profile + ansible.builtin.assert: + that: + - _declined_profile.rc == 0 + - "'virtual-function stage declined it' in _declined_profile.stdout" + fail_msg: >- + A card the virtual-function stage declined has no functions to + write a profile to, and saying so twice must not fail the unit. + Otherwise a legacy mdev card is red at every boot for a card the + script was right to leave alone. rc={{ _declined_profile.rc }}, + stdout={{ _declined_profile.stdout }} + + # Two declared cards where the second one's profile node is still + # being populated when its turn comes. The profile stage's wait is one + # budget for the whole stage, so a first card that spends it leaves + # nothing for the second, and a function the operator named by hand + # then fails for a node that was about to appear. Multi-GPU is the + # host this role exists for, so the budget has to be per card. + # + # The late node is installed on a signal rather than on a clock. A + # fixed delay races the first card's give-up: if that card overruns, + # the node is already there when the second card looks, the existence + # check passes before the deadline test is reached, and the case goes + # green against the broken code. Waiting for the give-up line puts the + # node after the first card's budget by construction and inside the + # second card's own budget regardless of how slow the runner is. + - name: Set the inputs the late-node case depends on + ansible.builtin.copy: + dest: "{{ item.path }}" + content: "{{ item.content }}\n" + mode: "0644" + loop: + - {path: "{{ _tmp }}/host_vgpu_mode", content: "SR-IOV"} + - {path: "{{ _pci }}/0000:41:00.0/sriov_numvfs", content: "4"} + loop_control: + label: "{{ item.path | basename }}" + + # The first card has to spend the budget, which means every function + # it exposes must be missing its node. The fixture gave two of them a + # writable node and one a directory, and any of those would let the + # card finish without waiting. + - name: Take the first card's profile nodes away + ansible.builtin.file: + path: "{{ _pci }}/0000:41:00.{{ item }}/nvidia/current_vgpu_type" + state: absent + loop: [4, 5, 6, 7] + + - name: Take the second card's profile node away + ansible.builtin.file: + path: "{{ _pci }}/0001:41:00.4/nvidia/current_vgpu_type" + state: absent + + - name: Give the second card back the function the previous case stripped + ansible.builtin.file: + src: "{{ _pci }}/0001:41:00.4" + dest: "{{ _pci }}/0001:41:00.0/virtfn0" + state: link + + - name: Start the run's log empty so the watcher cannot match an older line + ansible.builtin.copy: + dest: "{{ _tmp }}/late-run.log" + content: "" + mode: "0644" + + # Fires once the first card reports giving up on a node, which only + # happens after the shared budget is spent. The four seconds after + # that are measured from an observed event, not from the start of the + # run, so the node cannot arrive before the second card's turn. With + # the budget per card the second card is three checks into a fifteen + # second window when it lands; with one shared budget the second card + # has already failed and the node arrives too late to rescue it. + - name: Arm a watcher that installs the second card's node once the first gives up + ansible.builtin.shell: | + for _ in $(seq 1 60); do + if grep -q 'has no writable current_vgpu_type after waiting' \ + "{{ _tmp }}/late-run.log" 2>/dev/null; then + sleep 4 + printf '0\n' > "{{ _pci }}/0001:41:00.4/nvidia/current_vgpu_type" + exit 0 + fi + sleep 1 + done + async: 120 + poll: 0 + changed_when: false + + - name: Render two cards with profiles, the second one named by hand + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + vgpu_profile: 1155 + - address: "0001:41:00.0" + vgpu_profiles: + "0001:41:00.4": 1160 + + # Redirected to a file rather than captured, because the watcher has + # to read the log while the script is still running. + - name: Run the profile stage across both cards + ansible.builtin.shell: '"{{ _script }}" > "{{ _tmp }}/late-run.log" 2>&1' + environment: + PATH: "{{ _bin }}:{{ ansible_env.PATH }}" + register: _late + changed_when: false + failed_when: false + + - name: Read back the late run's log + ansible.builtin.slurp: + src: "{{ _tmp }}/late-run.log" + register: _late_log + + # Read with cat rather than slurp because the node is absent on the + # broken path, and a missing-file module error would replace this + # case's own diagnosis with a stack trace. + - name: Read back the second card's profile node + ansible.builtin.command: + cmd: "cat {{ _pci }}/0001:41:00.4/nvidia/current_vgpu_type" + register: _late_node + changed_when: false + failed_when: false + + - name: Assert the second card got its own waiting budget + ansible.builtin.assert: + that: + - _late.rc == 0 + - "'1160' in _late_node.stdout" + fail_msg: >- + The first card spent the profile stage's wait, and the second + card was given a function to write by hand. If the budget is + shared the second card gets no wait at all, its node is still + arriving, and the unit goes red at boot for work that was about + to be possible. rc={{ _late.rc }}, + node={{ _late_node.stdout }}, + log={{ _late_log.content | b64decode }} + + # A GPU reset takes the card away from every VM running on it, so the + # script must never issue one. A unit test scans the script's text, + # which covers paths this play never runs; this covers any spelling + # on the paths it does run, including ones the scan does not know. + - name: Read back every call the fake nvidia-smi received + ansible.builtin.slurp: + src: "{{ _tmp }}/nvidia-smi.argv" + register: _smi_argv + + - name: Assert no nvidia-smi call carried a reset flag + ansible.builtin.assert: + that: + - (_smi_argv.content | b64decode) | length > 0 + - not ((_smi_argv.content | b64decode) is search('(^|\\s)(-r|--gpu-reset)(\\s|$)')) + fail_msg: >- + Some path in this play called nvidia-smi with a reset flag. A + reset takes the GPU away from every vGPU running on it, and + nothing the unit does at boot needs one. + calls={{ _smi_argv.content | b64decode }} + + - name: Remove the temporary tree + ansible.builtin.file: + path: "{{ _tmp }}" + state: absent + + - name: Remove the artifacts this play installed + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_enable_nvidia_vgpu_host: false diff --git a/tests/test-nvidia-vgpu-host.yml b/tests/test-nvidia-vgpu-host.yml new file mode 100644 index 0000000..f54b8a5 --- /dev/null +++ b/tests/test-nvidia-vgpu-host.yml @@ -0,0 +1,342 @@ +--- +# Applies the nvidia_vgpu_host role on a host with no NVIDIA GPU, which +# is the state of a CI runner, and checks the artifacts it installs plus +# every refusal path the restore script can take there. The no-op +# behaviour is the point: this hardware is exactly what the role must not +# act on. +# +# Everything this play fakes lives under a temporary directory. The role +# exposes the driver paths as variables, so the play never writes to +# /usr/lib/nvidia or /run/nvidia and is safe to run on a host that has +# the real vGPU driver installed. Only the role's own artifacts, the unit +# and its script, land in their real locations. +# +# Run with become; the role installs into /usr/local/sbin and +# /etc/systemd/system. + +- name: Verify the NVIDIA vGPU host restore unit and its no-op paths + hosts: localhost + connection: local + become: true + gather_facts: true + vars: + _tmp: /tmp/cozystack-vgpu-host-test + _fake_host_driver: /tmp/cozystack-vgpu-host-test/usr/lib/nvidia/sriov-manage + _fake_operator_root: /tmp/cozystack-vgpu-host-test/run/nvidia/driver + _fake_operator_driver: /tmp/cozystack-vgpu-host-test/run/nvidia/driver/usr/lib/nvidia/sriov-manage + _script: /usr/local/sbin/cozystack-nvidia-vgpu-restore + _unit: /etc/systemd/system/cozystack-nvidia-vgpu-restore.service + cozystack_enable_nvidia_vgpu_host: true + cozystack_nvidia_vgpu_sriov_manage: /tmp/cozystack-vgpu-host-test/usr/lib/nvidia/sriov-manage + cozystack_nvidia_vgpu_operator_driver_root: /tmp/cozystack-vgpu-host-test/run/nvidia/driver + cozystack_nvidia_vgpu_devices: + - address: "0000:41:00.0" + sriov: true + mig: true + vgpu_profile: 1155 + vgpu_profiles: + "0000:41:00.5": 1160 + - address: "GPU-12345678-1234-1234-1234-123456789abc" + sriov: true + + tasks: + - name: Start from a clean temporary tree + ansible.builtin.file: + path: "{{ _tmp }}" + state: absent + + - name: Apply the role + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + + - name: Read back the rendered script + ansible.builtin.slurp: + src: "{{ _script }}" + register: _rendered_script + + - name: Read back the rendered unit + ansible.builtin.slurp: + src: "{{ _unit }}" + register: _rendered_unit + + - name: Set rendered content facts + ansible.builtin.set_fact: + _script_text: "{{ _rendered_script.content | b64decode }}" + _unit_text: "{{ _rendered_unit.content | b64decode }}" + + - name: Assert the script never resets a GPU + ansible.builtin.assert: + that: + - "'--gpu-reset' not in _script_text" + fail_msg: >- + The rendered script contains a GPU reset. Boot-time restoration + never needs one, and a reset takes down every consumer of the + card. + + - name: Assert declared GPUs are addressed individually + ansible.builtin.assert: + that: + - "'-e ALL' not in _script_text" + - "'ensure_vfs 0000:41:00.0' in _script_text" + - "'ensure_mig_mode 0000:41:00.0' in _script_text" + - "'ensure_vf_profiles 0000:41:00.0 1155' in _script_text" + - "\"['0000:41:00.0|0000:41:00.5']=1160\" in _script_text" + fail_msg: >- + The script must act on the GPUs named in + cozystack_nvidia_vgpu_devices and on no others, and per-VF + overrides must be keyed by the physical function they were + declared under. + + - name: Assert a GPU without the mig flag never reaches the MIG stage + ansible.builtin.assert: + that: + - "'ensure_mig_mode GPU-12345678' not in _script_text" + - "'ensure_vfs GPU-12345678' in _script_text" + fail_msg: >- + The second device sets sriov but not mig, so it must produce an + ensure_vfs call and no ensure_mig_mode call. Each flag gates + its own stage independently. + + # The calls, not the comments that label the stages: the comments + # could stay in place while the calls moved. + - name: Assert the generated calls run MIG mode before virtual functions + ansible.builtin.assert: + that: + - >- + _script_text.find('ensure_mig_mode 0000:41:00.0') + < _script_text.find('ensure_vfs 0000:41:00.0') + - >- + _script_text.find('ensure_vfs 0000:41:00.0') + < _script_text.find('ensure_vf_profiles 0000:41:00.0') + fail_msg: >- + Generated call order is wrong. MIG mode is a whole-GPU property + that settles before the GPU is subdivided, and a profile cannot + be written to a function that does not exist yet. + + - name: Assert the unit skips where no host driver is installed + ansible.builtin.assert: + that: + - "'ConditionPathExists=' + cozystack_nvidia_vgpu_sriov_manage in _unit_text" + - "'After=nvidia-vgpud.service nvidia-vgpu-mgr.service' in _unit_text" + - "'Requires=' not in _unit_text" + fail_msg: >- + The unit must carry the host-driver condition, order after the + driver's own daemons, and require nothing. + + - name: Check the rendered script parses + ansible.builtin.command: + cmd: "bash -n {{ _script }}" + changed_when: false + + # Presence is proven by the run itself. `command -v shellcheck` looks + # portable but is not: `command` is a shell builtin, this module + # execs directly without a shell, and Ubuntu ships no /usr/bin/command. + - name: Require shellcheck + ansible.builtin.command: + cmd: shellcheck --version + changed_when: false + + - name: Lint the rendered script + ansible.builtin.command: + cmd: "shellcheck --shell=bash {{ _script }}" + changed_when: false + + - name: Verify the rendered unit + ansible.builtin.command: + cmd: "systemd-analyze verify {{ _unit }}" + changed_when: false + + # With no host driver present the unit's condition fails, so systemd + # reports the start as successful and leaves the unit inactive. That + # is the difference between skipped and failed, and it is what keeps + # a GPU-less node's boot clean. + # + # A condition-skipped oneshot stays inactive, which the systemd + # module reports as a failure to start. Tolerated because the skip is + # the expected outcome and the next task is what judges it; nothing + # downstream reads this task's registered result. + - name: Start the unit on hardware it is not meant for + ansible.builtin.systemd: + name: cozystack-nvidia-vgpu-restore.service + state: started + failed_when: false + + - name: Read the unit's condition result + ansible.builtin.systemd: + name: cozystack-nvidia-vgpu-restore.service + register: _unit_state + changed_when: false + + - name: Assert the unit was skipped by its condition, not failed + ansible.builtin.assert: + that: + - _unit_state.status.ConditionResult == 'no' + - _unit_state.status.ActiveState != 'failed' + fail_msg: >- + Expected the unit to be skipped by ConditionPathExists on a + host with no NVIDIA driver. Got + ConditionResult={{ _unit_state.status.ConditionResult + | default('') }}, + ActiveState={{ _unit_state.status.ActiveState + | default('') }} + + - name: Run the script directly with no host driver present + ansible.builtin.command: + cmd: "{{ _script }}" + register: _no_driver + changed_when: false + + - name: Assert the script declines and exits zero with no host driver + ansible.builtin.assert: + that: + - _no_driver.rc == 0 + - "'no host-installed vGPU driver' in _no_driver.stdout" + fail_msg: >- + With no host driver the script must log why it is doing nothing + and exit zero. Got rc={{ _no_driver.rc }}, + stdout={{ _no_driver.stdout }} + + # Both a host-installed driver and an operator-managed driver root + # present at once leaves GPU ownership undecidable. The script must + # decline every stage rather than pick one. + - name: Create the directories the fake driver roots live in + ansible.builtin.file: + path: "{{ item | dirname }}" + state: directory + mode: "0755" + loop: + - "{{ _fake_host_driver }}" + - "{{ _fake_operator_driver }}" + + - name: Plant a fake host driver and a fake operator driver root + ansible.builtin.copy: + dest: "{{ item }}" + content: | + #!/bin/sh + exit 0 + mode: "0755" + loop: + - "{{ _fake_host_driver }}" + - "{{ _fake_operator_driver }}" + + - name: Run the script with ambiguous GPU ownership + ansible.builtin.command: + cmd: "{{ _script }}" + register: _ambiguous + changed_when: false + + - name: Assert the script declines on ambiguous ownership + ansible.builtin.assert: + that: + - _ambiguous.rc == 0 + - "'ownership is ambiguous' in _ambiguous.stdout" + fail_msg: >- + With both driver roots present the script must decline every + stage and exit zero. Got rc={{ _ambiguous.rc }}, + stdout={{ _ambiguous.stdout }} + + - name: Remove the fake operator driver root + ansible.builtin.file: + path: "{{ _fake_operator_root }}" + state: absent + + # A host driver present but no usable driver is the one precondition + # that fails the unit rather than skipping quietly: the operator + # declared GPUs on this host and has to learn the driver is broken. + - name: Re-render the script with a short driver wait + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_wait_seconds: 3 + + - name: Run the script with a host driver but no usable GPU + ansible.builtin.command: + cmd: "{{ _script }}" + register: _no_gpu + changed_when: false + failed_when: false + + - name: Assert the driver wait is a hard failure + ansible.builtin.assert: + that: + - _no_gpu.rc == 1 + - "'did not enumerate a GPU' in _no_gpu.stdout" + fail_msg: >- + A host driver that never becomes usable must fail the unit, not + skip. Got rc={{ _no_gpu.rc }}, stdout={{ _no_gpu.stdout }} + + # An enabled role with nothing declared must not look at the hardware + # at all. The fake host driver is still in place, so the + # driver-present precondition would pass; the script has to exit + # before reaching the wait that just failed above. + - name: Re-render the script with no declared work + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_nvidia_vgpu_wait_seconds: 3 + cozystack_nvidia_vgpu_devices: [] + + - name: Run the script with nothing declared + ansible.builtin.command: + cmd: "{{ _script }}" + register: _no_work + changed_when: false + + - name: Assert nothing declared means nothing is touched + ansible.builtin.assert: + that: + - _no_work.rc == 0 + - "'no GPUs declared' in _no_work.stdout" + - "'did not enumerate a GPU' not in _no_work.stdout" + fail_msg: >- + With no work declared the script must exit zero before any + precondition, including the driver wait. Got + rc={{ _no_work.rc }}, stdout={{ _no_work.stdout }} + + - name: Re-apply the role with the original variables + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + + - name: Read back the re-rendered script + ansible.builtin.slurp: + src: "{{ _script }}" + register: _rerendered_script + + # Rendering the per-VF override map iterates a dict, so a change that + # let that ordering vary would rewrite the script on every run. + - name: Assert the render is deterministic + ansible.builtin.assert: + that: + - (_rerendered_script.content | b64decode) == _script_text + fail_msg: >- + Re-applying the role with unchanged variables produced a + different script, so every run would rewrite it. + + # Turning the toggle off has to undo a previous run, or a host that + # was enabled once keeps changing GPU state at every boot. + - name: Apply the role with the toggle turned off + ansible.builtin.include_role: + name: cozystack.installer.nvidia_vgpu_host + vars: + cozystack_enable_nvidia_vgpu_host: false + + - name: Check the installed paths after disabling + ansible.builtin.stat: + path: "{{ item }}" + register: _after_disable + loop: + - "{{ _script }}" + - "{{ _unit }}" + + - name: Assert disabling the role removed what it installed + ansible.builtin.assert: + that: + - _after_disable.results | map(attribute='stat.exists') | select | list | length == 0 + fail_msg: >- + Disabling the role left its unit or script in place, so the + host no longer matches the toggle. + + - name: Remove the temporary tree + ansible.builtin.file: + path: "{{ _tmp }}" + state: absent diff --git a/tests/unit/roles/__init__.py b/tests/unit/roles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/roles/test_nvidia_vgpu_host.py b/tests/unit/roles/test_nvidia_vgpu_host.py new file mode 100644 index 0000000..e861f57 --- /dev/null +++ b/tests/unit/roles/test_nvidia_vgpu_host.py @@ -0,0 +1,590 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2026, Cozystack Contributors +# Apache License 2.0 (see LICENSE file in the repository root) + +# Structural tests for the nvidia_vgpu_host role. The invariants pinned +# here are the ones whose violation is silent on the hardware where it +# matters: a default that turns the role on, a boot unit that acts on a +# card it was not pointed at, a GPU reset re-entering the script, and +# the stage order that keeps a mode change from tearing down virtual +# functions. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import os +import re + +import yaml + + +REPO_ROOT = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "..", "..") +) + +ROLE = "roles/nvidia_vgpu_host" +DEFAULTS = ROLE + "/defaults/main.yml" +TASKS = ROLE + "/tasks/main.yml" +SCRIPT = ROLE + "/templates/cozystack-nvidia-vgpu-restore.sh.j2" +UNIT = ROLE + "/templates/cozystack-nvidia-vgpu-restore.service.j2" + +TOGGLE = "cozystack_enable_nvidia_vgpu_host" +DEVICES = "cozystack_nvidia_vgpu_devices" + +README_HEADING = "#### Opt-in: NVIDIA vGPU host state across reboots" + +PREPARE_PLAYBOOKS = ( + "examples/ubuntu/prepare-ubuntu.yml", + "examples/rhel/prepare-rhel.yml", + "examples/suse/prepare-suse.yml", +) + + +def _read(relpath): + with open(os.path.join(REPO_ROOT, relpath), "r", encoding="utf-8") as fh: + return fh.read() + + +def _load_yaml(relpath): + return yaml.safe_load(_read(relpath)) + + +def _unit_directive(name): + # The unit template still carries Jinja, so it is not ini-parseable; + # read directive values as text. + values = [] + for line in _read(UNIT).splitlines(): + line = line.strip() + if line.startswith(name + "="): + values.append(line[len(name) + 1:]) + return values + + +# ---- opt-in defaults ---- + + +def test_role_is_disabled_by_default(): + defaults = _load_yaml(DEFAULTS) + assert defaults[TOGGLE] is False, ( + "%s must default to false. The role installs a boot unit that " + "changes GPU hardware state; it may only run where an operator " + "asked for it." % TOGGLE + ) + + +def test_device_list_is_empty_by_default(): + defaults = _load_yaml(DEFAULTS) + assert defaults[DEVICES] == [], ( + "%s must default to an empty list. Every action this role takes " + "is addressed to a GPU named in that list, so an empty list is " + "what keeps the unit inert even once the role is enabled." + % DEVICES + ) + + +def _when_text(entry): + when = entry.get("when") + assert when is not None, "every top-level entry must carry a when clause" + return when if isinstance(when, str) else " ".join(when) + + +def test_tasks_are_gated_by_blocks_on_both_paths(): + tasks = _load_yaml(TASKS) + assert len(tasks) == 2, ( + "tasks/main.yml must hold exactly two top-level entries, each " + "wrapping its tasks in one gated block: the enabled path and the " + "path that undoes it. A per-task gate can be forgotten on the " + "next task added; a block gate cannot. Found %d entries." + % len(tasks) + ) + gates = [] + for entry in tasks: + assert "block" in entry, "each top-level entry must be a block" + when_text = _when_text(entry) + assert TOGGLE in when_text, ( + "each block's when must reference %s" % TOGGLE + ) + assert "default(false)" in when_text, ( + "each block's when must read the toggle through " + "default(false) so an inventory that never sets it leaves the " + "role off" + ) + gates.append(when_text) + negated = [gate for gate in gates if "not " in gate] + assert len(negated) == 1, ( + "exactly one of the two blocks must gate on the toggle being " + "false. Got gates %r" % gates + ) + + +def test_disabling_the_role_undoes_a_previous_run(): + # Without this, a host enabled once keeps the unit enabled and keeps + # changing GPU state at every boot from whatever device list was + # current then, so the host stops matching the toggle. + tasks = _load_yaml(TASKS) + disabled = [entry for entry in tasks if "not " in _when_text(entry)] + body = yaml.safe_dump(disabled[0]["block"]) + assert "enabled: false" in body, ( + "the disabled path must disable the unit so it does not run at " + "the next boot" + ) + assert "state: absent" in body, ( + "the disabled path must remove the unit file and the script" + ) + assert "/etc/systemd/system/cozystack-nvidia-vgpu-restore.service" in body + assert "/usr/local/sbin/cozystack-nvidia-vgpu-restore" in body + + +def test_role_installs_the_unit_without_starting_it(): + # Enabling MIG mode or creating virtual functions on a live host is a + # maintenance-window action. The role enables the unit for the next + # boot and leaves starting it to the operator. + text = _read(TASKS) + for forbidden in ("state: started", "state: restarted"): + assert forbidden not in text, ( + "tasks must not %r the boot unit: applying the role would " + "then change GPU state immediately, outside any maintenance " + "window. Enable it and let the next boot apply it." + % forbidden + ) + assert "enabled: true" in text, ( + "the unit must be enabled so it runs on the next boot" + ) + handlers = _read(ROLE + "/handlers/main.yml") + assert "daemon_reload: true" in handlers, ( + "dropping a unit file requires a daemon-reload to take effect. It " + "belongs in a handler rather than a task so an unchanged re-run " + "reports no change." + ) + + +def _address_patterns(): + # Pulls the real patterns out of the role's assert so this tests the + # validation rather than the presence of a task that mentions it. + for entry in _load_yaml(TASKS): + for task in entry.get("block", []): + spec = task.get("ansible.builtin.assert") + if not spec: + continue + for clause in spec.get("that") or []: + if "item.address is match" in clause: + return re.findall(r"match\('([^']+)'\)", clause) + raise AssertionError( + "no assert in tasks/main.yml validates item.address" + ) + + +def test_role_validates_device_identifiers(): + patterns = _address_patterns() + assert patterns, "the address assert must carry at least one pattern" + + def accepted(value): + # Ansible's `match` test anchors at the start via re.match. + return any(re.match(p, value) for p in patterns) + + for value in ( + "0000:41:00.0", + "00000000:41:00.0", + "41:00.0", + "GPU-12345678-1234-1234-1234-123456789abc", + ): + assert accepted(value), ( + "%r is a valid GPU identifier and must be accepted" % value + ) + for value in ( + "0", + "1", + "gpu0", + "0000:41:00.8", + "0000:41:00", + "0000:41:0.0", + "GPU-123", + "", + ): + assert not accepted(value), ( + "%r must be rejected. A bare index in particular: indices " + "shift on PCI, BIOS and kernel re-enumeration, so accepting " + "one lets a config that named the right card name a " + "different one after a firmware update." % value + ) + + +def _duplicate_check_names(): + names = [] + for entry in _load_yaml(TASKS): + for task in entry.get("block", []): + spec = task.get("ansible.builtin.assert") + if not spec: + continue + for clause in spec.get("that") or []: + if "unique" in clause: + names.append(task.get("name", "")) + return names + + +def test_role_rejects_both_kinds_of_duplicate(): + # Asserting that "unique" appears somewhere passed with either check + # deleted, because both use it. Name them individually so removing + # one is a failure. + names = " | ".join(_duplicate_check_names()) + assert "virtual-function" in names, ( + "a check must reject a virtual function named more than once " + "across the vgpu_profiles maps. Found: %s" % names + ) + assert "device addresses" in names, ( + "a check must reject a card named more than once. Found: %s" + % names + ) + + +def test_pci_normalisation_is_defined_once(): + # The two duplicate checks previously carried their own regexes and + # disagreed about the PCI domain: one canonicalised it, the other + # deleted it, so a valid multi-segment configuration was refused + # while two cards on different segments aliased onto one key. Both + # now reference the shared definition, and a new site that hardcodes + # a pattern instead fails here. + # + # Scoped to every YAML in the role, not to tasks/main.yml: splitting + # validation into a second file is ordinary growth, and a guard that + # only reads one file would pass while the claim it carries stopped + # being true. + role_dir = os.path.join(REPO_ROOT, ROLE) + offenders = [] + for root, _dirs, files in os.walk(role_dir): + for name in files: + if not name.endswith((".yml", ".yaml")): + continue + path = os.path.join(root, name) + if os.path.relpath(path, role_dir) == os.path.join("vars", "main.yml"): + continue + with open(path, "r", encoding="utf-8") as handle: + text = handle.read() + for line in text.splitlines(): + if "regex_replace" not in line: + continue + if "_cozystack_vgpu_bdf_" in line: + continue + offenders.append("%s: %s" % (name, line.strip())) + if re.search(r"\[0-9a-f(?:A-F)?\]\{2\}:\[0-9a-f(?:A-F)?\]\{2\}", + text, re.IGNORECASE): + if "match(" not in text: + offenders.append("%s: hardcoded PCI pattern" % name) + assert not offenders, ( + "PCI address normalisation must come from vars/main.yml so every " + "comparison uses one rule. These sites define their own: %s" + % offenders + ) + + +def test_shared_normalisation_keeps_the_domain(): + # The rule itself, not just its single definition. Deleting the + # domain aliases two cards on different PCI segments onto one key, + # which is how the role came to act on hardware nobody declared. + shared = _load_yaml(ROLE + "/vars/main.yml") + substitution = shared["_cozystack_vgpu_bdf_domain_sub"] + assert "1" in substitution and "2" in substitution, ( + "the domain substitution must carry both capture groups, so the " + "domain survives canonicalisation. Got %r" % substitution + ) + + +# ---- the boot unit ---- + + +def test_unit_skips_when_no_host_installed_driver_is_present(): + conditions = _unit_directive("ConditionPathExists") + assert conditions, ( + "the unit must carry ConditionPathExists on the host driver's " + "sriov-manage. That path is absent both on a host with no NVIDIA " + "driver and on one where gpu-operator manages the vGPU Manager " + "(its driver root lives elsewhere), so the condition is what " + "makes the unit a no-op there, reported as skipped, not failed." + ) + assert any("sriov" in value for value in conditions), ( + "the ConditionPathExists must point at sriov-manage, found %r" + % conditions + ) + + +def test_unit_runs_after_the_vgpu_manager(): + after = " ".join(_unit_directive("After")) + for daemon in ("nvidia-vgpud.service", "nvidia-vgpu-mgr.service"): + assert daemon in after, ( + "the unit must be ordered After=%s. sriov-manage is " + "documented to fail while the Virtual GPU Manager is still " + "initialising, so ordering before those daemons guarantees " + "that failure on the hardware where it is documented." % daemon + ) + before = " ".join(_unit_directive("Before")) + for daemon in ("nvidia-vgpud", "nvidia-vgpu-mgr"): + assert daemon not in before, ( + "the unit must not be ordered Before=%s: on an NVLink system " + "that orders the script ahead of the daemon it depends on" + % daemon + ) + + +def test_script_retries_the_virtual_function_call(): + # Naming the constant is not enough: it is also declared at the top + # of the file and used by the profile-stage wait, so an assertion on + # the name alone passes with the retry loop collapsed to one attempt. + text = _read(SCRIPT) + body = text[text.find("ensure_vfs() {"):text.find("# ---- STAGE 3")] + assert 'for attempt in $(seq 1 "$SRIOV_RETRIES")' in body, ( + "the virtual-function call must sit inside a retry loop. The " + "vendor documents it failing while the Virtual GPU Manager is " + "still initialising, and the retry is what closes that race." + ) + assert "SRIOV_RETRIES" in text, ( + "enabling virtual functions must be retried, not attempted once. " + "The vendor documents the call failing while the Virtual GPU " + "Manager initialises, and NVIDIA's own driver container retries " + "it for that reason; the retry is what closes the boot race that " + "unit ordering alone cannot." + ) + + +def test_unit_is_a_oneshot_that_does_not_block_boot(): + text = _read(UNIT) + assert "Type=oneshot" in text + assert "multi-user.target" in " ".join(_unit_directive("WantedBy")) + assert "Requires=" not in text, ( + "no Requires= on this unit: a failure must degrade GPU VMs, not " + "take a target down with it" + ) + + +# ---- the restore script ---- + + +def test_script_never_resets_a_gpu(): + # Two checks with opposite blind spots, kept together because neither + # is the property. The substring scan reads the whole file, so it + # survives a line continuation, an invocation through a variable, and + # a direct sysfs write, all of which a same-line match loses. The + # same-line flag match catches `-r`, nvidia-smi's short form of + # `--gpu-reset`, which no substring in the first list contains. + # Replacing one with the other traded three kills for one; the + # behavioural check in tests/test-nvidia-vgpu-host-stages.yml covers + # what both miss, on the paths the suite actually drives. + text = _read(SCRIPT) + for forbidden in ("--gpu-reset", "gpu_reset", "-r ALL"): + assert forbidden not in text, ( + "the script must never reset a GPU (%r found). A reset " + "destroys every consumer of the card, which on a vGPU host " + "is the tenants' running VMs." % forbidden + ) + offenders = [ + line.strip() + for line in _read(SCRIPT).splitlines() + if "nvidia-smi" in line + and re.search(r"(?:^|\s)(?:-r|--gpu-reset)(?:\s|$)", line) + ] + assert not offenders, ( + "the script must never reset a GPU. MIG mode is persistent on " + "Ampere, where setting it would need a reset, and needs no reset " + "on Hopper and later, where it is not persistent, so boot-time " + "restoration never requires one. A reset destroys every consumer " + "of the card, which on a vGPU host is the tenants' running VMs. " + "Offending lines: %r" % offenders + ) + + +def test_script_stages_mig_mode_before_virtual_functions(): + # Pins the order of the generated calls, not of the comments that + # label them: the comments could stay put while the calls moved. + text = _read(SCRIPT) + mig = text.find("\nensure_mig_mode {{") + sriov = text.find("\nensure_vfs {{") + profiles = text.find("\nensure_vf_profiles {{") + assert -1 not in (mig, sriov, profiles), ( + "the script must emit one call per declared GPU for each stage, " + "found offsets mig=%d sriov=%d profiles=%d" % (mig, sriov, profiles) + ) + assert mig < sriov < profiles, ( + "call order must be MIG mode, then virtual functions, then " + "per-VF profiles. MIG mode is a whole-GPU property and settles " + "before the GPU is subdivided, and a profile cannot be written " + "to a function that does not exist yet." + ) + + +def test_script_addresses_virtual_functions_per_device_never_all(): + # Independent of how the command is spelled: it is invoked through a + # variable, so a check anchored on the binary's name would not see + # the argument change. + text = _read(SCRIPT) + assert not re.search(r"-e[ \t]+ALL", text), ( + "the script must never pass -e ALL. Every action is addressed to " + "a GPU the operator named; ALL would act on cards that were " + "never declared, which is the failure mode that sank the " + "auto-detecting approach." + ) + assert not re.search(r"-d[ \t]+ALL", text), ( + "the script must never pass -d ALL either: tearing down every " + "card's virtual functions is not something it is asked to do." + ) + + +def test_script_exits_before_touching_hardware_when_nothing_is_declared(): + # The driver wait is a hard failure, so an inert configuration must + # never reach it. Rendering order is what guarantees that. + text = _read(SCRIPT) + no_work = text.find("no GPUs declared") + preconditions = text.find("# ---- preconditions ----") + assert no_work != -1, ( + "the script must log that nothing was declared" + ) + assert preconditions != -1, "the preconditions section must be marked" + assert no_work < preconditions, ( + "the no-work exit must be rendered before the preconditions. " + "Otherwise a host with the role enabled but no GPUs declared " + "waits for a driver it was never asked to touch and then fails." + ) + + +def test_script_logs_the_raw_virtualization_mode_it_declined(): + text = _read(SCRIPT) + assert "unrecognised" in text or "unrecognized" in text, ( + "the script must have an explicit branch for a virtualization " + "mode string it does not recognise" + ) + assert re.search(r"(observed|raw)[^\n]*mode", text, re.IGNORECASE), ( + "when the script declines because it did not recognise the " + "driver's reported mode, it must log the observed string " + "verbatim. Skipping silently on an unanticipated spelling looks " + "exactly like working correctly." + ) + + +def test_script_documents_why_the_driver_wait_is_the_only_hard_failure(): + text = _read(SCRIPT) + assert re.search( + r"#[^\n]*(only|unlike)[^\n]*(exit|fail)", text, re.IGNORECASE + ), ( + "the non-zero exit on the driver wait needs a comment saying why " + "it is the one precondition that fails loudly while every other " + "exits zero. That reasoning is a constraint the code cannot show." + ) + + +def test_script_documents_that_the_numvfs_read_is_not_capability_inference(): + text = _read(SCRIPT) + marker = text.find("sriov_numvfs") + assert marker != -1, ( + "the already-satisfied check reads sriov_numvfs from sysfs; " + "sriov-manage has no query form, so there is no alternative" + ) + window = text[max(0, marker - 1200):marker] + assert re.search( + r"#[^\n]*(not|never)[^\n]*(infer|capab)", window, re.IGNORECASE + ), ( + "the sriov_numvfs read must carry a comment stating that it only " + "suppresses a redundant write and never concludes that virtual " + "functions are missing and must be created. Deriving 'VFs are " + "missing' from sysfs counts misreads hardware that advertises " + "virtual functions but never creates them, and a reader who does " + "not know that will re-introduce it." + ) + + +# ---- wiring into the example playbooks ---- + + +def test_examples_include_the_role_unconditionally(): + # The role gates both of its own paths on the toggle. Gating the + # include as well would make the teardown unreachable through the + # documented entry point, so a host enabled once would keep + # restoring GPU state at every boot no matter how the toggle was + # set afterwards. + for relpath in PREPARE_PLAYBOOKS: + plays = _load_yaml(relpath) + includes = [ + task + for play in plays + for task in (play.get("tasks") or []) + if "ansible.builtin.include_role" in task + and task["ansible.builtin.include_role"].get("name") + == "cozystack.installer.nvidia_vgpu_host" + ] + assert len(includes) == 1, ( + "%s must include the role exactly once by its " + "collection-qualified name, found %d" + % (relpath, len(includes)) + ) + assert "when" not in includes[0], ( + "%s must not gate the include: the role decides for itself, " + "and a gate here makes turning the toggle back off a no-op" + % relpath + ) + + +def test_default_driver_paths_are_the_real_ones(): + # The test playbooks point these at a temporary tree so they never + # write to a real driver install, which means nothing else would + # notice if the shipped defaults drifted. + defaults = _load_yaml(DEFAULTS) + assert defaults["cozystack_nvidia_vgpu_sriov_manage"] == ( + "/usr/lib/nvidia/sriov-manage" + ) + assert defaults["cozystack_nvidia_vgpu_operator_driver_root"] == ( + "/run/nvidia/driver" + ) + assert defaults["cozystack_nvidia_vgpu_pci_root"] == ( + "/sys/bus/pci/devices" + ) + + +# ---- documentation drift guards ---- + + +def test_readme_documents_the_role_and_the_profile_precedence(): + text = _read("README.md") + assert "nvidia_vgpu_host" in text, "README must document the new role" + assert DEVICES in text, ( + "README must document %s, the variable that names which GPUs the " + "unit may touch" % DEVICES + ) + assert "vgpu_profiles" in text and "vgpu_profile" in text, ( + "README must document both the per-VF map and the per-PF " + "shorthand for vGPU profile assignment" + ) + assert re.search( + r"vgpu_profiles[^.\n]*(win|precede|override)", text, re.IGNORECASE + ), ( + "README must state which form wins when a VF is covered by both " + "the per-PF shorthand and the per-VF map" + ) + + +def _readme_section(heading): + text = _read("README.md") + start = text.find(heading) + assert start != -1, "README is missing the heading %r" % heading + rest = text[start + len(heading):] + end = rest.find("\n#") + return rest if end == -1 else rest[:end] + + +def test_readme_names_a_daemonset_as_the_operator_managed_answer(): + # Scoped to the role's own section: "DaemonSet" appears elsewhere in + # this README, so an unscoped search would pass without the sentence + # that matters ever being written. + section = _readme_section(README_HEADING) + assert "DaemonSet" in section, ( + "the role's README section must say that a DaemonSet remains the " + "right mechanism for operator-managed clusters. This role covers " + "the host-installed, ansible-managed path only and must not be " + "presented as the general answer to per-VF profile assignment." + ) + + +def test_changelog_documents_the_role(): + sections = _read("CHANGELOG.rst").split("Unreleased", 1) + assert len(sections) == 2, "CHANGELOG must carry an Unreleased section" + assert "nvidia_vgpu_host" in sections[1], ( + "the Unreleased section must document the new role" + )