vms-101: run heavy builds/tests on the k3s rail (run-on-rail.sh)

tools/k3s/Dockerfile.rail

@@ -0,0 +1,78 @@
+# tools/k3s/Dockerfile.rail -- the OVMX "rail" builder image.
+#
+# BUILD/TEST TOOLING ONLY. This is NOT an OVMX runtime (CLAUDE.md Rule 9): it
+# is an environment in which `cmake`/`ctest`/`tests/qemu/run_tests.sh` and the
+# QEMU-KVM smoke run on cluster hardware instead of the shared workshop host.
+# It carries no OVMX product code -- run-on-rail.sh clones the repo at a given
+# git-ref INTO a pod built from this image and runs the requested command.
+#
+# The package set is deliberately the SAME toolchain layer tests/qemu/Dockerfile
+# installs (ubuntu:24.04 + build tools + qemu + kernel pkgs + musl + cmake), so
+# anything the QEMU kernel harness needs is present, plus git (the pod clones
+# the repo) and genisoimage/xorriso (the alpha/boot ISO builders, vms-054).
+#
+# Build + push (one-time, from workshop -- this local docker build is the only
+# local-heavy step, and it is acceptable):
+#   docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.43:30500/ovmx-builder:latest .
+#   docker push 192.168.2.43:30500/ovmx-builder:latest
+# 192.168.2.43:30500 = the registry NodePort on k3s-cp, which workshop's Docker
+# daemon already lists in insecure-registries (the registry is plain HTTP, so
+# the .44/k3s-worker NodePort would need a daemon.json edit -- the .43 one is
+# already trusted). Same registry storage either way; pods pull by the
+# in-cluster Service name (see README "Registry notes").
+
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# One apt layer, one index fetch (same discipline as tests/qemu/Dockerfile).
+# The QEMU package name is arch-conditional; everything else is arch-neutral.
+# docker.io/iptables are for run-on-rail.sh --dind mode (dockerd runs INSIDE
+# this container so the docker-wrapped e2e gates -- which `docker build
+# distro/Dockerfile.bootable` then `docker run -v $REPO_ROOT/...` -- resolve
+# their bind mounts against the same filesystem the repo is cloned into. A
+# separate dind sidecar would resolve those mounts on the wrong host.
+RUN apt-get update && \
+    ARCH=$(dpkg --print-architecture) && \
+    if [ "$ARCH" = "arm64" ]; then QEMU_PKG=qemu-system-arm; else QEMU_PKG=qemu-system-x86; fi && \
+    apt-get install -y --no-install-recommends \
+        make gcc g++ libc6-dev kmod \
+        linux-headers-generic \
+        linux-image-virtual \
+        busybox-static \
+        cpio zstd xz-utils gzip bzip2 \
+        curl wget perl ca-certificates patch \
+        git \
+        cmake ninja-build pkg-config \
+        musl-tools musl-dev linux-libc-dev \
+        libelf-dev bc flex bison \
+        python3 file bsdmainutils \
+        binutils gawk strace ccache clang \
+        libssh-dev libreadline-dev \
+        docker.io iptables uidmap \
+        genisoimage xorriso \
+        "$QEMU_PKG" qemu-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+# musl doesn't ship linux/ or asm/ UAPI headers -- symlink them from
+# linux-libc-dev so OVMX_STATIC=ON (musl-gcc) builds work here exactly as they
+# do in tests/qemu/Dockerfile and distro/Dockerfile.bootable.
+RUN MUSL_INC=$(echo /usr/include/*-linux-musl) && \
+    ARCH_INC=$(echo /usr/include/*-linux-gnu) && \
+    ln -sf /usr/include/linux "$MUSL_INC/linux" && \
+    ln -sf /usr/include/asm-generic "$MUSL_INC/asm-generic" && \
+    ln -sf "$ARCH_INC/asm" "$MUSL_INC/asm"
+
+# git needs to trust any checkout dir the pod clones into (it runs as root
+# against a freshly-cloned tree owned by root -- safe.directory keeps git from
+# refusing "dubious ownership").
+RUN git config --system --add safe.directory '*'
+
+# A vmlinuz is present (linux-image-virtual) so the KVM smoke can boot a real
+# kernel under -accel kvm. Record its path for kvm-smoke.sh.
+RUN KVER=$(ls /lib/modules/ | sort -V | tail -1) && \
+    if [ ! -e /boot/vmlinuz ]; then ln -s "vmlinuz-${KVER}" /boot/vmlinuz; fi && \
+    ls -l /boot/vmlinuz
+
+WORKDIR /work
+CMD ["/bin/bash"]

tools/k3s/README.md

@@ -0,0 +1,113 @@
+# OVMX k3s rail — offload heavy builds/tests
+
+**Build/test tooling only.** Per CLAUDE.md Rule 9 the OVMX runtime is the
+kernel/QEMU path; k3s here is *only* a place to run `cmake`/`ctest`/
+`tests/qemu/run_tests.sh` and QEMU-KVM smokes on cluster hardware instead of
+the shared `workshop` host. This is never an OVMX runtime and must not be
+presented as one.
+
+## What's here
+
+| File | Purpose |
+|------|---------|
+| `run-on-rail.sh` | Run a command in the repo at a git-ref, in a pod on k3s-worker; stream logs; exit with the command's exit code. |
+| `Dockerfile.rail` | The builder image (ubuntu:24.04 + the tests/qemu toolchain + git + genisoimage). |
+| `namespace.yaml` | `ovmx-ci` namespace + ResourceQuota (~6 CPU / 40Gi) + LimitRange. |
+| `job-template.yaml` | Job manifest `run-on-rail.sh` renders per run. |
+| `kvm-smoke.sh` | Proves `/dev/kvm` passthrough / KVM acceleration in-pod. |
+
+## Usage
+
+```bash
+export KUBECONFIG=~/.kube/config          # context: default
+
+# Full build + ctest on cluster hardware:
+tools/k3s/run-on-rail.sh main \
+  "cmake -B build -DBUILD_TESTS=ON -DBUILD_TOOLS=ON && \
+   cmake --build build -j\$(nproc) && cd build && ctest --output-on-failure"
+
+# Prove KVM passthrough:
+tools/k3s/run-on-rail.sh main "bash tools/k3s/kvm-smoke.sh"
+
+# Any ref (branch/tag/SHA); --keep leaves the Job for debugging:
+tools/k3s/run-on-rail.sh --keep work/my-branch "cmake --build build -j\$(nproc)"
+```
+
+The script exits with the command's exit code, so it drops straight into CI or
+a shell `&&` chain.
+
+### Flags / env knobs
+
+- `--keep` — don't delete the Job when it finishes.
+- `--name NAME` — base name for the Job (default `ovmx-ci`).
+- `OVMX_RAIL_IMAGE` — override the builder image ref.
+- `OVMX_RAIL_REQ_CPU` / `_REQ_MEM` / `_LIM_CPU` / `_LIM_MEM` — resources.
+- `OVMX_RAIL_DEADLINE` — wall cap seconds (default 3600).
+- `OVMX_REPO_URL` — clone URL (default `https://github.com/3dl-dev/vms`, public).
+
+## Cluster facts (verified)
+
+- **Registry:** `gpu-rail-registry` (default ns), NodePort **30500** → 5000,
+  ClusterIP `10.43.10.176`. Insecure **HTTP** registry.
+  - **Working ref for BOTH push and pull:**
+    **`192.168.2.43:30500/ovmx-builder:latest`** (the k3s-cp NodePort). This is
+    the exact pattern the existing ovmx-lab StatefulSets use
+    (`192.168.2.43:30500/ovmx-vaxlab:3`), i.e. the proven-trusted address.
+  - The in-cluster Service DNS name
+    (`gpu-rail-registry.default.svc.cluster.local:5000`) does **not** work for
+    image pulls — the node's containerd/kubelet resolve image hosts via the
+    *node's* resolv.conf, not CoreDNS, so it fails with
+    `lookup ...svc.cluster.local: Try again`. Use the NodePort IP. See
+    "Registry notes" below.
+- **Target node:** `k3s-worker` (192.168.2.44, 8 CPU / 53 GB, KVM-capable bare
+  metal). The Job pins `nodeSelector kubernetes.io/hostname=k3s-worker`. It
+  never targets k3s-cp (small), k3s-mini (NotReady), or the GPU node.
+- **KVM:** `/dev/kvm` is passed through with a privileged pod + hostPath
+  `/dev/kvm` (`type: CharDevice`). Confirmed working — `kvm-smoke.sh` boots a
+  real kernel under strict `-accel kvm` and asserts the guest logs
+  `Hypervisor detected: KVM`.
+
+## Building / pushing the builder image (one-time bootstrap)
+
+This local `docker build` on workshop is the only local-heavy step; it's
+acceptable as a one-time bootstrap.
+
+```bash
+# from the repo root on workshop:
+docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.43:30500/ovmx-builder:latest .
+docker push 192.168.2.43:30500/ovmx-builder:latest
+```
+
+### Registry notes (the parts that fight you)
+
+- The registry serves **HTTP** (insecure). The k3s nodes' containerd already
+  trusts it (they pull other tenants' images from it), so **no per-node config
+  was needed for pull**.
+- **Push must use the NodePort workshop's Docker daemon already trusts.** The
+  registry is plain HTTP, so `docker push` needs the target in
+  `/etc/docker/daemon.json` `insecure-registries`. Workshop already lists
+  **`192.168.2.43:30500`** (k3s-cp) there but *not* `192.168.2.44:30500`
+  (k3s-worker) — pushing to `.44` fails with
+  `http: server gave HTTP response to HTTPS client`. Push to **`.43`** (no
+  shared-host config change) — it's the same registry storage as `.44`.
+- **Same ref for push and pull: `192.168.2.43:30500/ovmx-builder:latest`.**
+  Push goes there because workshop's daemon trusts it; pods pull the same tag
+  because the nodes' containerd trusts that address too *and* it is a bare IP
+  needing no DNS. Do **not** use the Service DNS name for the image ref — the
+  node resolves image hosts via its own resolv.conf, not CoreDNS, so
+  `gpu-rail-registry.default.svc.cluster.local:5000` ImagePullBackOffs with
+  `lookup ...: Try again` (verified). The ClusterIP form
+  `10.43.10.176:5000/...` avoids DNS but is only pullable if the nodes'
+  containerd trusts that IP as insecure too — the NodePort IP is the known-good
+  path, so that is the default (`OVMX_RAIL_IMAGE` to override).
+- `imagePullPolicy: Always` — the tag is `:latest` and mutable; always re-pull
+  so a rebuilt builder image is picked up.
+
+## How exit codes / streaming work
+
+The pod clones the repo at the ref, decodes the caller's command (passed
+base64 so arbitrary shell never touches the YAML), and runs it. The command's
+exit status is the container's exit status. `run-on-rail.sh` streams
+`kubectl logs -f`, waits for the Job to reach complete/failed, reads the
+container's `terminated.exitCode`, and re-exits with it. A failed command sets
+`backoffLimit: 0` so it is never retried — the first exit code is the verdict.

tools/k3s/job-template.yaml

@@ -0,0 +1,122 @@
+# tools/k3s/job-template.yaml -- one-shot CI Job template for run-on-rail.sh.
+#
+# BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9). run-on-rail.sh fills the ${...}
+# placeholders via envsubst and applies the result. The pod clones the OVMX
+# repo at ${GIT_REF}, then runs the caller's command (passed base64-encoded in
+# OVMX_CMD_B64, so arbitrary shell never has to be escaped into YAML). The
+# pod's exit code IS the command's exit code -- see run-on-rail.sh.
+#
+# Pinned to k3s-worker (8 CPU / 53 GB); privileged with hostPath /dev/kvm so
+# QEMU can use `-accel kvm` on this KVM-capable bare-metal node. The GPU node
+# and k3s-cp are deliberately never targeted.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${JOB_NAME}
+  namespace: ovmx-ci
+  labels:
+    app.kubernetes.io/managed-by: ovmx-run-on-rail
+spec:
+  backoffLimit: 0            # a failed command must NOT be retried -- exit code is the verdict
+  ttlSecondsAfterFinished: 3600
+  activeDeadlineSeconds: ${DEADLINE}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/managed-by: ovmx-run-on-rail
+        ovmx.dev/job: ${JOB_NAME}
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: k3s-worker
+      containers:
+        - name: rail
+          image: ${IMAGE}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true       # needed for /dev/kvm + QEMU acceleration on this trusted homelab node
+          workingDir: /work
+          env:
+            - name: GIT_REF
+              value: "${GIT_REF}"
+            - name: REPO_URL
+              value: "${REPO_URL}"
+            - name: OVMX_CMD_B64
+              value: "${OVMX_CMD_B64}"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: OVMX_DIND
+              value: "${DIND}"       # "1" enables the in-container dockerd (--dind)
+          resources:
+            requests:
+              cpu: "${REQ_CPU}"
+              memory: "${REQ_MEM}"
+            limits:
+              cpu: "${LIM_CPU}"
+              memory: "${LIM_MEM}"
+          volumeMounts:
+            - name: kvm
+              mountPath: /dev/kvm
+            # dockerd's graph on a node-backed emptyDir (not the pod overlay
+            # rootfs) so overlay2 works and the image build isn't capped by the
+            # container's writable layer. Unused (harmless) in non-dind mode.
+            - name: docker-storage
+              mountPath: /var/lib/docker
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              echo "[rail] node=${NODE_NAME} pod=$(hostname) ref=${GIT_REF} dind=${OVMX_DIND}"
+
+              # --- optional in-container dockerd (run-on-rail.sh --dind) --------
+              # dockerd runs HERE (not a sidecar) so the docker-wrapped e2e gates'
+              # `docker run -v $REPO_ROOT/...` bind mounts resolve against this
+              # same filesystem, where the repo is cloned. /dev/kvm is already in
+              # this pod, so an inner `docker run --device /dev/kvm ... -accel kvm`
+              # is KVM-accelerated (nested KVM verified).
+              if [ "${OVMX_DIND:-0}" = "1" ]; then
+                echo "[rail] starting in-container dockerd..."
+                # The in-pod dockerd must trust the same plain-HTTP registry the
+                # nodes' containerd does (192.168.2.43:30500) so an inner
+                # `docker pull`/`docker run <registry-image>` works in --dind --
+                # e.g. the nested-KVM proof that runs the builder image under
+                # `docker run --device /dev/kvm ... -accel kvm`.
+                DOCKERD_INSECURE="--insecure-registry 192.168.2.43:30500"
+                _start_dockerd() { dockerd $DOCKERD_INSECURE "$@" >/var/log/dockerd.log 2>&1 & DOCKERD_PID=$!; }
+                _wait_dockerd() {
+                  for _ in $(seq 1 60); do
+                    docker info >/dev/null 2>&1 && return 0
+                    kill -0 "$DOCKERD_PID" 2>/dev/null || return 1
+                    sleep 1
+                  done; return 1
+                }
+                _start_dockerd
+                if ! _wait_dockerd; then
+                  echo "[rail] dockerd (overlay2) did not come up; retrying with vfs"
+                  tail -20 /var/log/dockerd.log || true
+                  _start_dockerd --storage-driver=vfs
+                  _wait_dockerd || { echo "[rail] FATAL: dockerd failed to start"; tail -40 /var/log/dockerd.log; exit 1; }
+                fi
+                echo "[rail] dockerd up: $(docker version --format '{{.Server.Version}}' 2>/dev/null) driver=$(docker info --format '{{.Driver}}' 2>/dev/null)"
+              fi
+
+              echo "[rail] cloning ${REPO_URL}"
+              git clone --filter=blob:none "${REPO_URL}" repo
+              cd repo
+              git checkout "${GIT_REF}" 2>/dev/null \
+                || { git fetch --tags origin "${GIT_REF}" && git checkout FETCH_HEAD; }
+              echo "[rail] HEAD=$(git rev-parse HEAD)  ($(git log -1 --pretty=%s))"
+              echo "${OVMX_CMD_B64}" | base64 -d > /tmp/ovmx-cmd.sh
+              echo "[rail] --- command start ---"
+              # The command's exit status becomes the container's exit status,
+              # which run-on-rail.sh reads back and re-exits with.
+              bash /tmp/ovmx-cmd.sh
+      volumes:
+        - name: kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice
+        - name: docker-storage
+          emptyDir: {}

tools/k3s/kvm-smoke.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# tools/k3s/kvm-smoke.sh -- prove QEMU hardware acceleration (/dev/kvm) works
+# in-pod on the rail. BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9).
+#
+# Run via: tools/k3s/run-on-rail.sh <ref> "bash tools/k3s/kvm-smoke.sh"
+#
+# It boots the builder image's stock kernel under QEMU with STRICT `-accel kvm`
+# (no `:tcg` fallback -- QEMU aborts if KVM is unavailable, so a TCG fallback
+# can never masquerade as a pass) and asserts the *guest* detected the KVM
+# hypervisor. Linux prints "Hypervisor detected: KVM" only when it is actually
+# running on KVM-accelerated vCPUs, so that line is positive proof the vCPU is
+# KVM-backed -- not merely that /dev/kvm exists.
+
+set -euo pipefail
+
+echo "=== /dev/kvm ==="
+ls -l /dev/kvm
+
+ARCH="$(uname -m)"
+case "$ARCH" in
+  x86_64)  QEMU=qemu-system-x86_64;  MACHINE=q35 ;;
+  aarch64) QEMU=qemu-system-aarch64; MACHINE=virt ;;
+  *) echo "kvm-smoke: unsupported arch $ARCH" >&2; exit 2 ;;
+esac
+command -v "$QEMU" >/dev/null || { echo "kvm-smoke: $QEMU not installed" >&2; exit 127; }
+
+KERNEL=/boot/vmlinuz
+[ -e "$KERNEL" ] || KERNEL="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1)"
+[ -n "$KERNEL" ] && [ -e "$KERNEL" ] || { echo "kvm-smoke: no kernel image found" >&2; exit 1; }
+echo "=== kernel: $KERNEL ==="
+
+echo "=== accelerators QEMU sees ==="
+"$QEMU" -accel help 2>&1 | sed 's/^/  /'
+
+LOG="$(mktemp)"
+echo "=== booting under STRICT -accel kvm (no tcg fallback) ==="
+# No rootfs is staged: the kernel boots the vCPU (the point -- proves KVM), then
+# panics unable to mount root. panic=-1 + -no-reboot make it exit promptly.
+# STRICT `-accel kvm`: if KVM were unavailable QEMU would abort here, non-zero.
+timeout 90 "$QEMU" \
+  -accel kvm \
+  -M "$MACHINE" \
+  -cpu host \
+  -m 512 \
+  -smp 2 \
+  -nographic -no-reboot \
+  -kernel "$KERNEL" \
+  -append "console=ttyS0 panic=-1 loglevel=8" \
+  ${QEMU_EXTRA:-} \
+  > "$LOG" 2>&1 || true    # kernel panic (no root) is expected; we grade the log
+
+echo "=== guest serial (head) ==="
+sed -n '1,40p' "$LOG"
+
+echo "======================================================================"
+if grep -qi "Hypervisor detected: KVM" "$LOG"; then
+  echo "PASS: guest detected KVM hypervisor -- vCPU is KVM-accelerated (not TCG)."
+  exit 0
+fi
+
+# Fallback positive signal: some kernels word it differently; accept an
+# explicit kvm-clock / KVM paravirt line too. But NEVER accept a TCG boot.
+if grep -qiE "kvm-clock|kvm-guest|KVM setup" "$LOG"; then
+  echo "PASS: guest shows KVM paravirt (kvm-clock/kvm-guest) -- KVM-accelerated."
+  exit 0
+fi
+
+echo "FAIL: no KVM hypervisor signal in guest boot -- acceleration not proven."
+echo "----- full log -----"
+cat "$LOG"
+exit 1