From 1324380974ed5e2a28c5eaf17e3f631477efc2cd Mon Sep 17 00:00:00 2001
From: alice <baron@3dl.dev>
Date: Thu, 20 Aug 2026 18:48:13 +0000
Subject: [PATCH 1/5] vms-101: run heavy builds/tests on the k3s rail
 (run-on-rail.sh)

Offload cmake/ctest/QEMU-KVM smokes from the shared workshop host onto the
k3s rail. BUILD/TEST TOOLING ONLY (Rule 9) -- never an OVMX runtime.

- tools/k3s/run-on-rail.sh <git-ref> <command...>: clone the repo at the ref
  in a pod on k3s-worker, stream logs live, exit with the command's exit code.
  --keep leaves the Job for debugging.
- tools/k3s/Dockerfile.rail: builder image (ubuntu:24.04 + the tests/qemu
  toolchain + git + genisoimage), pushed to the in-cluster registry.
- tools/k3s/namespace.yaml: ovmx-ci namespace + ResourceQuota (~6 CPU / 40Gi)
  + LimitRange so CI never starves the shared tenants.
- tools/k3s/job-template.yaml: privileged Job pinned to k3s-worker with
  hostPath /dev/kvm for KVM-accelerated QEMU; command passed base64 so
  arbitrary shell never touches the YAML; pod exit code = command exit code.
- tools/k3s/kvm-smoke.sh: boots a kernel under strict -accel kvm and asserts
  the guest logs "Hypervisor detected: KVM" (KVM proof, not TCG fallback).
- tools/k3s/README.md: registry push/pull ref split + KVM notes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tools/k3s/Dockerfile.rail   |  66 ++++++++++++++++
 tools/k3s/README.md         | 101 ++++++++++++++++++++++++
 tools/k3s/job-template.yaml |  80 +++++++++++++++++++
 tools/k3s/kvm-smoke.sh      |  71 +++++++++++++++++
 tools/k3s/namespace.yaml    |  56 +++++++++++++
 tools/k3s/run-on-rail.sh    | 152 ++++++++++++++++++++++++++++++++++++
 6 files changed, 526 insertions(+)
 create mode 100644 tools/k3s/Dockerfile.rail
 create mode 100644 tools/k3s/README.md
 create mode 100644 tools/k3s/job-template.yaml
 create mode 100755 tools/k3s/kvm-smoke.sh
 create mode 100644 tools/k3s/namespace.yaml
 create mode 100755 tools/k3s/run-on-rail.sh

diff --git a/tools/k3s/Dockerfile.rail b/tools/k3s/Dockerfile.rail
new file mode 100644
index 000000000..87e6b30cb
--- /dev/null
+++ b/tools/k3s/Dockerfile.rail
@@ -0,0 +1,66 @@
+# tools/k3s/Dockerfile.rail -- the OVMX "rail" builder image.
+#
+# BUILD/TEST TOOLING ONLY. This is NOT an OVMX runtime (CLAUDE.md Rule 9): it
+# is an environment in which `cmake`/`ctest`/`tests/qemu/run_tests.sh` and the
+# QEMU-KVM smoke run on cluster hardware instead of the shared workshop host.
+# It carries no OVMX product code -- run-on-rail.sh clones the repo at a given
+# git-ref INTO a pod built from this image and runs the requested command.
+#
+# The package set is deliberately the SAME toolchain layer tests/qemu/Dockerfile
+# installs (ubuntu:24.04 + build tools + qemu + kernel pkgs + musl + cmake), so
+# anything the QEMU kernel harness needs is present, plus git (the pod clones
+# the repo) and genisoimage/xorriso (the alpha/boot ISO builders, vms-054).
+#
+# Build + push (one-time, from workshop -- this local docker build is the only
+# local-heavy step, and it is acceptable):
+#   docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.44:30500/ovmx-builder:latest .
+#   docker push 192.168.2.44:30500/ovmx-builder:latest
+# (192.168.2.44:30500 = the in-cluster registry's NodePort on k3s-worker.)
+
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# One apt layer, one index fetch (same discipline as tests/qemu/Dockerfile).
+# The QEMU package name is arch-conditional; everything else is arch-neutral.
+RUN apt-get update && \
+    ARCH=$(dpkg --print-architecture) && \
+    if [ "$ARCH" = "arm64" ]; then QEMU_PKG=qemu-system-arm; else QEMU_PKG=qemu-system-x86; fi && \
+    apt-get install -y --no-install-recommends \
+        make gcc g++ libc6-dev kmod \
+        linux-headers-generic \
+        linux-image-virtual \
+        busybox-static \
+        cpio zstd xz-utils gzip bzip2 \
+        curl wget perl ca-certificates patch \
+        git \
+        cmake ninja-build pkg-config \
+        musl-tools musl-dev linux-libc-dev \
+        libelf-dev bc flex bison \
+        python3 file bsdmainutils \
+        genisoimage xorriso \
+        "$QEMU_PKG" qemu-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+# musl doesn't ship linux/ or asm/ UAPI headers -- symlink them from
+# linux-libc-dev so OVMX_STATIC=ON (musl-gcc) builds work here exactly as they
+# do in tests/qemu/Dockerfile and distro/Dockerfile.bootable.
+RUN MUSL_INC=$(echo /usr/include/*-linux-musl) && \
+    ARCH_INC=$(echo /usr/include/*-linux-gnu) && \
+    ln -sf /usr/include/linux "$MUSL_INC/linux" && \
+    ln -sf /usr/include/asm-generic "$MUSL_INC/asm-generic" && \
+    ln -sf "$ARCH_INC/asm" "$MUSL_INC/asm"
+
+# git needs to trust any checkout dir the pod clones into (it runs as root
+# against a freshly-cloned tree owned by root -- safe.directory keeps git from
+# refusing "dubious ownership").
+RUN git config --system --add safe.directory '*'
+
+# A vmlinuz is present (linux-image-virtual) so the KVM smoke can boot a real
+# kernel under -accel kvm. Record its path for kvm-smoke.sh.
+RUN KVER=$(ls /lib/modules/ | sort -V | tail -1) && \
+    if [ ! -e /boot/vmlinuz ]; then ln -s "vmlinuz-${KVER}" /boot/vmlinuz; fi && \
+    ls -l /boot/vmlinuz
+
+WORKDIR /work
+CMD ["/bin/bash"]
diff --git a/tools/k3s/README.md b/tools/k3s/README.md
new file mode 100644
index 000000000..4af10a7e2
--- /dev/null
+++ b/tools/k3s/README.md
@@ -0,0 +1,101 @@
+# OVMX k3s rail — offload heavy builds/tests
+
+**Build/test tooling only.** Per CLAUDE.md Rule 9 the OVMX runtime is the
+kernel/QEMU path; k3s here is *only* a place to run `cmake`/`ctest`/
+`tests/qemu/run_tests.sh` and QEMU-KVM smokes on cluster hardware instead of
+the shared `workshop` host. This is never an OVMX runtime and must not be
+presented as one.
+
+## What's here
+
+| File | Purpose |
+|------|---------|
+| `run-on-rail.sh` | Run a command in the repo at a git-ref, in a pod on k3s-worker; stream logs; exit with the command's exit code. |
+| `Dockerfile.rail` | The builder image (ubuntu:24.04 + the tests/qemu toolchain + git + genisoimage). |
+| `namespace.yaml` | `ovmx-ci` namespace + ResourceQuota (~6 CPU / 40Gi) + LimitRange. |
+| `job-template.yaml` | Job manifest `run-on-rail.sh` renders per run. |
+| `kvm-smoke.sh` | Proves `/dev/kvm` passthrough / KVM acceleration in-pod. |
+
+## Usage
+
+```bash
+export KUBECONFIG=~/.kube/config          # context: default
+
+# Full build + ctest on cluster hardware:
+tools/k3s/run-on-rail.sh main \
+  "cmake -B build -DBUILD_TESTS=ON -DBUILD_TOOLS=ON && \
+   cmake --build build -j\$(nproc) && cd build && ctest --output-on-failure"
+
+# Prove KVM passthrough:
+tools/k3s/run-on-rail.sh main "bash tools/k3s/kvm-smoke.sh"
+
+# Any ref (branch/tag/SHA); --keep leaves the Job for debugging:
+tools/k3s/run-on-rail.sh --keep work/my-branch "cmake --build build -j\$(nproc)"
+```
+
+The script exits with the command's exit code, so it drops straight into CI or
+a shell `&&` chain.
+
+### Flags / env knobs
+
+- `--keep` — don't delete the Job when it finishes.
+- `--name NAME` — base name for the Job (default `ovmx-ci`).
+- `OVMX_RAIL_IMAGE` — override the builder image ref.
+- `OVMX_RAIL_REQ_CPU` / `_REQ_MEM` / `_LIM_CPU` / `_LIM_MEM` — resources.
+- `OVMX_RAIL_DEADLINE` — wall cap seconds (default 3600).
+- `OVMX_REPO_URL` — clone URL (default `https://github.com/3dl-dev/vms`, public).
+
+## Cluster facts (verified)
+
+- **Registry:** `gpu-rail-registry` (default ns), NodePort **30500** → 5000,
+  ClusterIP `10.43.10.176`. Insecure HTTP registry, trusted cluster-wide.
+  - **Push from workshop:** via a node NodePort, e.g.
+    `192.168.2.44:30500` (k3s-worker) or `192.168.2.43:30500` (k3s-cp).
+  - **Pull in-cluster:** the working ref is
+    **`gpu-rail-registry.default.svc.cluster.local:5000/ovmx-builder:latest`**
+    (see "Registry notes" below for why this one and not the NodePort).
+- **Target node:** `k3s-worker` (192.168.2.44, 8 CPU / 53 GB, KVM-capable bare
+  metal). The Job pins `nodeSelector kubernetes.io/hostname=k3s-worker`. It
+  never targets k3s-cp (small), k3s-mini (NotReady), or the GPU node.
+- **KVM:** `/dev/kvm` is passed through with a privileged pod + hostPath
+  `/dev/kvm` (`type: CharDevice`). Confirmed working — `kvm-smoke.sh` boots a
+  real kernel under strict `-accel kvm` and asserts the guest logs
+  `Hypervisor detected: KVM`.
+
+## Building / pushing the builder image (one-time bootstrap)
+
+This local `docker build` on workshop is the only local-heavy step; it's
+acceptable as a one-time bootstrap.
+
+```bash
+# from the repo root on workshop:
+docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.44:30500/ovmx-builder:latest .
+docker push 192.168.2.44:30500/ovmx-builder:latest
+```
+
+### Registry notes (the parts that fight you)
+
+- The registry serves **HTTP** (insecure). Workshop's Docker already pushes to
+  it (other tenants do), and the k3s nodes' containerd already trusts it (they
+  pull other tenants' images from it), so **no per-node TLS/insecure config was
+  needed** for either push or pull.
+- **Push tag vs. pull ref differ on purpose.** You push to a *NodePort*
+  (`192.168.2.44:30500`) because that's reachable from workshop. Pods pull with
+  the *in-cluster* Service DNS name
+  (`gpu-rail-registry.default.svc.cluster.local:5000`), which resolves to the
+  Service ClusterIP `10.43.10.176:5000`. Both names address the same registry
+  storage, so an image pushed at the NodePort tag is pullable at the in-cluster
+  tag — they are the same blob under two names. If DNS ever misbehaves, the
+  ClusterIP form `10.43.10.176:5000/ovmx-builder:latest` works identically
+  (set `OVMX_RAIL_IMAGE`).
+- `imagePullPolicy: Always` — the tag is `:latest` and mutable; always re-pull
+  so a rebuilt builder image is picked up.
+
+## How exit codes / streaming work
+
+The pod clones the repo at the ref, decodes the caller's command (passed
+base64 so arbitrary shell never touches the YAML), and runs it. The command's
+exit status is the container's exit status. `run-on-rail.sh` streams
+`kubectl logs -f`, waits for the Job to reach complete/failed, reads the
+container's `terminated.exitCode`, and re-exits with it. A failed command sets
+`backoffLimit: 0` so it is never retried — the first exit code is the verdict.
diff --git a/tools/k3s/job-template.yaml b/tools/k3s/job-template.yaml
new file mode 100644
index 000000000..5ae8c2a84
--- /dev/null
+++ b/tools/k3s/job-template.yaml
@@ -0,0 +1,80 @@
+# tools/k3s/job-template.yaml -- one-shot CI Job template for run-on-rail.sh.
+#
+# BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9). run-on-rail.sh fills the ${...}
+# placeholders via envsubst and applies the result. The pod clones the OVMX
+# repo at ${GIT_REF}, then runs the caller's command (passed base64-encoded in
+# OVMX_CMD_B64, so arbitrary shell never has to be escaped into YAML). The
+# pod's exit code IS the command's exit code -- see run-on-rail.sh.
+#
+# Pinned to k3s-worker (8 CPU / 53 GB); privileged with hostPath /dev/kvm so
+# QEMU can use `-accel kvm` on this KVM-capable bare-metal node. The GPU node
+# and k3s-cp are deliberately never targeted.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${JOB_NAME}
+  namespace: ovmx-ci
+  labels:
+    app.kubernetes.io/managed-by: ovmx-run-on-rail
+spec:
+  backoffLimit: 0            # a failed command must NOT be retried -- exit code is the verdict
+  ttlSecondsAfterFinished: 3600
+  activeDeadlineSeconds: ${DEADLINE}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/managed-by: ovmx-run-on-rail
+        ovmx.dev/job: ${JOB_NAME}
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: k3s-worker
+      containers:
+        - name: rail
+          image: ${IMAGE}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true       # needed for /dev/kvm + QEMU acceleration on this trusted homelab node
+          workingDir: /work
+          env:
+            - name: GIT_REF
+              value: "${GIT_REF}"
+            - name: REPO_URL
+              value: "${REPO_URL}"
+            - name: OVMX_CMD_B64
+              value: "${OVMX_CMD_B64}"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: "${REQ_CPU}"
+              memory: "${REQ_MEM}"
+            limits:
+              cpu: "${LIM_CPU}"
+              memory: "${LIM_MEM}"
+          volumeMounts:
+            - name: kvm
+              mountPath: /dev/kvm
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              echo "[rail] node=${NODE_NAME} pod=$(hostname) ref=${GIT_REF}"
+              echo "[rail] cloning ${REPO_URL}"
+              git clone --filter=blob:none "${REPO_URL}" repo
+              cd repo
+              git checkout "${GIT_REF}" 2>/dev/null \
+                || { git fetch --tags origin "${GIT_REF}" && git checkout FETCH_HEAD; }
+              echo "[rail] HEAD=$(git rev-parse HEAD)  ($(git log -1 --pretty=%s))"
+              echo "${OVMX_CMD_B64}" | base64 -d > /tmp/ovmx-cmd.sh
+              echo "[rail] --- command start ---"
+              # The command's exit status becomes the container's exit status,
+              # which run-on-rail.sh reads back and re-exits with.
+              bash /tmp/ovmx-cmd.sh
+      volumes:
+        - name: kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice
diff --git a/tools/k3s/kvm-smoke.sh b/tools/k3s/kvm-smoke.sh
new file mode 100755
index 000000000..2afa04643
--- /dev/null
+++ b/tools/k3s/kvm-smoke.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# tools/k3s/kvm-smoke.sh -- prove QEMU hardware acceleration (/dev/kvm) works
+# in-pod on the rail. BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9).
+#
+# Run via: tools/k3s/run-on-rail.sh <ref> "bash tools/k3s/kvm-smoke.sh"
+#
+# It boots the builder image's stock kernel under QEMU with STRICT `-accel kvm`
+# (no `:tcg` fallback -- QEMU aborts if KVM is unavailable, so a TCG fallback
+# can never masquerade as a pass) and asserts the *guest* detected the KVM
+# hypervisor. Linux prints "Hypervisor detected: KVM" only when it is actually
+# running on KVM-accelerated vCPUs, so that line is positive proof the vCPU is
+# KVM-backed -- not merely that /dev/kvm exists.
+
+set -euo pipefail
+
+echo "=== /dev/kvm ==="
+ls -l /dev/kvm
+
+ARCH="$(uname -m)"
+case "$ARCH" in
+  x86_64)  QEMU=qemu-system-x86_64;  MACHINE=q35 ;;
+  aarch64) QEMU=qemu-system-aarch64; MACHINE=virt ;;
+  *) echo "kvm-smoke: unsupported arch $ARCH" >&2; exit 2 ;;
+esac
+command -v "$QEMU" >/dev/null || { echo "kvm-smoke: $QEMU not installed" >&2; exit 127; }
+
+KERNEL=/boot/vmlinuz
+[ -e "$KERNEL" ] || KERNEL="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1)"
+[ -n "$KERNEL" ] && [ -e "$KERNEL" ] || { echo "kvm-smoke: no kernel image found" >&2; exit 1; }
+echo "=== kernel: $KERNEL ==="
+
+echo "=== accelerators QEMU sees ==="
+"$QEMU" -accel help 2>&1 | sed 's/^/  /'
+
+LOG="$(mktemp)"
+echo "=== booting under STRICT -accel kvm (no tcg fallback) ==="
+# No rootfs is staged: the kernel boots the vCPU (the point -- proves KVM), then
+# panics unable to mount root. panic=-1 + -no-reboot make it exit promptly.
+# STRICT `-accel kvm`: if KVM were unavailable QEMU would abort here, non-zero.
+timeout 90 "$QEMU" \
+  -accel kvm \
+  -M "$MACHINE" \
+  -cpu host \
+  -m 512 \
+  -smp 2 \
+  -nographic -no-reboot \
+  -kernel "$KERNEL" \
+  -append "console=ttyS0 panic=-1 loglevel=8" \
+  ${QEMU_EXTRA:-} \
+  > "$LOG" 2>&1 || true    # kernel panic (no root) is expected; we grade the log
+
+echo "=== guest serial (head) ==="
+sed -n '1,40p' "$LOG"
+
+echo "======================================================================"
+if grep -qi "Hypervisor detected: KVM" "$LOG"; then
+  echo "PASS: guest detected KVM hypervisor -- vCPU is KVM-accelerated (not TCG)."
+  exit 0
+fi
+
+# Fallback positive signal: some kernels word it differently; accept an
+# explicit kvm-clock / KVM paravirt line too. But NEVER accept a TCG boot.
+if grep -qiE "kvm-clock|kvm-guest|KVM setup" "$LOG"; then
+  echo "PASS: guest shows KVM paravirt (kvm-clock/kvm-guest) -- KVM-accelerated."
+  exit 0
+fi
+
+echo "FAIL: no KVM hypervisor signal in guest boot -- acceleration not proven."
+echo "----- full log -----"
+cat "$LOG"
+exit 1
diff --git a/tools/k3s/namespace.yaml b/tools/k3s/namespace.yaml
new file mode 100644
index 000000000..01c3c6a7c
--- /dev/null
+++ b/tools/k3s/namespace.yaml
@@ -0,0 +1,56 @@
+# tools/k3s/namespace.yaml -- the ovmx-ci namespace and its guardrails.
+#
+# BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9). This namespace exists so heavy
+# OVMX builds/tests run on the k3s rail without ever starving the shared
+# tenants (meilisearch in enterprise-ai, the mm-* batch jobs, the GPU rail).
+#
+# The ResourceQuota caps the WHOLE namespace at ~6 CPU / 40Gi of REQUESTS, so
+# the sum of everything CI schedules can never exceed that regardless of how
+# many run-on-rail.sh jobs pile up -- a second concurrent job that would push
+# the total over the cap is refused admission and queues, rather than being
+# packed onto the node beside a tenant workload.
+#
+# The LimitRange gives every CI container a sane default request/limit (so a
+# manifest that forgets to set them still counts against the quota) and a hard
+# per-container ceiling that keeps a single job from claiming the entire cap.
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ovmx-ci
+  labels:
+    app.kubernetes.io/managed-by: ovmx-run-on-rail
+    ovmx.dev/purpose: build-test-tooling
+---
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: ovmx-ci-quota
+  namespace: ovmx-ci
+spec:
+  hard:
+    requests.cpu: "6"
+    requests.memory: 40Gi
+    limits.cpu: "6"
+    limits.memory: 40Gi
+    pods: "8"
+---
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: ovmx-ci-limits
+  namespace: ovmx-ci
+spec:
+  limits:
+    - type: Container
+      # Default request/limit applied to a container that names neither.
+      defaultRequest:
+        cpu: "2"
+        memory: 4Gi
+      default:
+        cpu: "4"
+        memory: 12Gi
+      # Hard ceiling: no single CI container may claim the whole namespace cap.
+      max:
+        cpu: "6"
+        memory: 32Gi
diff --git a/tools/k3s/run-on-rail.sh b/tools/k3s/run-on-rail.sh
new file mode 100755
index 000000000..7d8615411
--- /dev/null
+++ b/tools/k3s/run-on-rail.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+# tools/k3s/run-on-rail.sh -- run a heavy OVMX build/test on the k3s rail.
+#
+# BUILD/TEST TOOLING ONLY (CLAUDE.md Rule 9). This offloads `cmake`/`ctest`/
+# `tests/qemu/run_tests.sh` and the QEMU-KVM smoke from the shared workshop
+# host onto cluster hardware (k3s-worker). It is NOT an OVMX runtime.
+#
+# Usage:
+#   tools/k3s/run-on-rail.sh [--keep] [--name NAME] <git-ref> <command...>
+#
+#   <git-ref>    branch, tag, or full SHA of https://github.com/3dl-dev/vms
+#   <command>    shell run in the repo root inside the pod; its exit code
+#                becomes this script's exit code.
+#
+# Flags:
+#   --keep       do not delete the Job when it finishes (for debugging)
+#   --name NAME  base name for the Job (default: ovmx-ci)
+#
+# Examples:
+#   tools/k3s/run-on-rail.sh main \
+#     "cmake -B build -DBUILD_TESTS=ON -DBUILD_TOOLS=ON && \
+#      cmake --build build -j\$(nproc) && cd build && ctest --output-on-failure"
+#
+#   tools/k3s/run-on-rail.sh main "bash tools/k3s/kvm-smoke.sh"
+#
+# Requires: kubectl (KUBECONFIG pointing at the rail), envsubst (gettext-base).
+
+set -euo pipefail
+
+# --- config -----------------------------------------------------------------
+NS=ovmx-ci
+REPO_URL="${OVMX_REPO_URL:-https://github.com/3dl-dev/vms}"
+IMAGE="${OVMX_RAIL_IMAGE:-gpu-rail-registry.default.svc.cluster.local:5000/ovmx-builder:latest}"
+DEADLINE="${OVMX_RAIL_DEADLINE:-3600}"    # activeDeadlineSeconds (wall cap)
+REQ_CPU="${OVMX_RAIL_REQ_CPU:-4}"
+REQ_MEM="${OVMX_RAIL_REQ_MEM:-8Gi}"
+LIM_CPU="${OVMX_RAIL_LIM_CPU:-6}"
+LIM_MEM="${OVMX_RAIL_LIM_MEM:-24Gi}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# --- args -------------------------------------------------------------------
+KEEP=0
+NAME_BASE=ovmx-ci
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --keep) KEEP=1; shift ;;
+    --name) NAME_BASE="$2"; shift 2 ;;
+    --)     shift; break ;;
+    -h|--help) sed -n '2,40p' "$0"; exit 0 ;;
+    -*)     echo "run-on-rail: unknown flag $1" >&2; exit 2 ;;
+    *)      break ;;
+  esac
+done
+
+if [ $# -lt 2 ]; then
+  echo "usage: run-on-rail.sh [--keep] [--name NAME] <git-ref> <command...>" >&2
+  exit 2
+fi
+
+GIT_REF="$1"; shift
+CMD="$*"
+
+command -v kubectl >/dev/null || { echo "run-on-rail: kubectl not found" >&2; exit 127; }
+command -v envsubst >/dev/null || { echo "run-on-rail: envsubst not found (apt install gettext-base)" >&2; exit 127; }
+
+# --- ensure namespace + quota + limits exist --------------------------------
+kubectl apply -f "$SCRIPT_DIR/namespace.yaml" >/dev/null
+
+# --- render the Job manifest ------------------------------------------------
+# The command is base64-encoded so arbitrary shell never touches the YAML.
+OVMX_CMD_B64="$(printf '%s' "$CMD" | base64 | tr -d '\n')"
+JOB_NAME="${NAME_BASE}-$(date +%s)-${RANDOM}"
+
+export JOB_NAME IMAGE GIT_REF REPO_URL OVMX_CMD_B64 DEADLINE \
+       REQ_CPU REQ_MEM LIM_CPU LIM_MEM
+MANIFEST="$(envsubst \
+  '$JOB_NAME $IMAGE $GIT_REF $REPO_URL $OVMX_CMD_B64 $DEADLINE $REQ_CPU $REQ_MEM $LIM_CPU $LIM_MEM' \
+  < "$SCRIPT_DIR/job-template.yaml")"
+
+cleanup() {
+  if [ "$KEEP" -eq 1 ]; then
+    echo "[run-on-rail] --keep: leaving Job $JOB_NAME (kubectl -n $NS delete job $JOB_NAME to remove)" >&2
+  else
+    kubectl -n "$NS" delete job "$JOB_NAME" --wait=false >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+echo "[run-on-rail] ref=$GIT_REF  job=$JOB_NAME  image=$IMAGE" >&2
+echo "[run-on-rail] command: $CMD" >&2
+printf '%s\n' "$MANIFEST" | kubectl apply -f - >/dev/null
+
+# --- wait for the pod to exist and reach a streamable state -----------------
+echo "[run-on-rail] waiting for pod to schedule on k3s-worker..." >&2
+POD=""
+for _ in $(seq 1 120); do
+  POD="$(kubectl -n "$NS" get pods -l "ovmx.dev/job=$JOB_NAME" \
+          -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+  [ -n "$POD" ] && break
+  sleep 1
+done
+[ -n "$POD" ] || { echo "[run-on-rail] pod never appeared" >&2; exit 1; }
+
+# Wait until the pod is out of Pending (Running or already terminated), so
+# `logs -f` has something to stream. Surface scheduling failures early.
+for _ in $(seq 1 300); do
+  PHASE="$(kubectl -n "$NS" get pod "$POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+  case "$PHASE" in
+    Running|Succeeded|Failed) break ;;
+  esac
+  # Show pull/schedule trouble instead of hanging silently.
+  WAITMSG="$(kubectl -n "$NS" get pod "$POD" \
+    -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
+  case "$WAITMSG" in
+    ErrImagePull|ImagePullBackOff|CreateContainerError|InvalidImageName)
+      echo "[run-on-rail] pod stuck: $WAITMSG" >&2
+      kubectl -n "$NS" describe pod "$POD" 2>&1 | sed -n '/Events:/,$p' >&2
+      exit 1 ;;
+  esac
+  sleep 1
+done
+
+# --- stream logs ------------------------------------------------------------
+echo "[run-on-rail] pod=$POD phase=$PHASE -- streaming logs:" >&2
+echo "----------------------------------------------------------------------" >&2
+kubectl -n "$NS" logs -f "$POD" 2>/dev/null || true
+
+# --- wait for terminal state, then read the real exit code ------------------
+kubectl -n "$NS" wait --for=condition=complete "job/$JOB_NAME" --timeout="${DEADLINE}s" >/dev/null 2>&1 &
+W1=$!
+kubectl -n "$NS" wait --for=condition=failed "job/$JOB_NAME" --timeout="${DEADLINE}s" >/dev/null 2>&1 &
+W2=$!
+wait -n "$W1" "$W2" 2>/dev/null || true
+kill "$W1" "$W2" 2>/dev/null || true
+
+# The container's terminated exitCode is the command's exit code (see template).
+EXIT_CODE=""
+for _ in $(seq 1 30); do
+  EXIT_CODE="$(kubectl -n "$NS" get pod "$POD" \
+    -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || true)"
+  [ -n "$EXIT_CODE" ] && break
+  sleep 1
+done
+
+echo "----------------------------------------------------------------------" >&2
+if [ -z "$EXIT_CODE" ]; then
+  echo "[run-on-rail] could not read container exit code; treating as failure" >&2
+  exit 1
+fi
+echo "[run-on-rail] pod=$POD node=$(kubectl -n "$NS" get pod "$POD" -o jsonpath='{.spec.nodeName}' 2>/dev/null) exit=$EXIT_CODE" >&2
+exit "$EXIT_CODE"

From 2f28562f43baf99009f61c29b13437835fc216c3 Mon Sep 17 00:00:00 2001
From: alice <baron@3dl.dev>
Date: Thu, 20 Aug 2026 19:10:53 +0000
Subject: [PATCH 2/5] =?UTF-8?q?vms-101:=20v0=20working=20on=20the=20rail?=
 =?UTF-8?q?=20=E2=80=94=20correct=20image=20ref,=20scheduling=20fit,=20gen?=
 =?UTF-8?q?erous=20quota?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Verified end-to-end on k3s-worker: clone-at-ref, run command, stream logs,
propagate exit code (exit 7 -> script exit 7).

- Image ref fixed to 192.168.2.43:30500/ovmx-builder:latest (the k3s-cp
  NodePort). The in-cluster Service DNS name ImagePullBackOffs — node
  containerd resolves image hosts via node resolv.conf, not CoreDNS. Push must
  also use .43 (workshop daemon trusts only that NodePort as insecure HTTP).
- Per-pod request lowered to 1 CPU / 4Gi so jobs schedule on k3s-worker
  (shared tenants already hold ~5 of its 8 CPU); build still bursts to the
  6-CPU limit.
- run-on-rail.sh surfaces FailedScheduling / pull failures instead of hanging,
  and never streams logs from a still-Pending pod.
- ResourceQuota made generous (operator: rail capacity is ample) so it never
  gates a wave of parallel jobs; node free-CPU still bounds real concurrency.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tools/k3s/Dockerfile.rail | 10 +++++---
 tools/k3s/README.md       | 54 ++++++++++++++++++++++++---------------
 tools/k3s/namespace.yaml  | 15 +++++++----
 tools/k3s/run-on-rail.sh  | 33 +++++++++++++++++++-----
 4 files changed, 76 insertions(+), 36 deletions(-)

diff --git a/tools/k3s/Dockerfile.rail b/tools/k3s/Dockerfile.rail
index 87e6b30cb..428b0cbb4 100644
--- a/tools/k3s/Dockerfile.rail
+++ b/tools/k3s/Dockerfile.rail
@@ -13,9 +13,13 @@
 #
 # Build + push (one-time, from workshop -- this local docker build is the only
 # local-heavy step, and it is acceptable):
-#   docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.44:30500/ovmx-builder:latest .
-#   docker push 192.168.2.44:30500/ovmx-builder:latest
-# (192.168.2.44:30500 = the in-cluster registry's NodePort on k3s-worker.)
+#   docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.43:30500/ovmx-builder:latest .
+#   docker push 192.168.2.43:30500/ovmx-builder:latest
+# 192.168.2.43:30500 = the registry NodePort on k3s-cp, which workshop's Docker
+# daemon already lists in insecure-registries (the registry is plain HTTP, so
+# the .44/k3s-worker NodePort would need a daemon.json edit -- the .43 one is
+# already trusted). Same registry storage either way; pods pull by the
+# in-cluster Service name (see README "Registry notes").
 
 FROM ubuntu:24.04
 
diff --git a/tools/k3s/README.md b/tools/k3s/README.md
index 4af10a7e2..b348b7dfd 100644
--- a/tools/k3s/README.md
+++ b/tools/k3s/README.md
@@ -48,12 +48,17 @@ a shell `&&` chain.
 ## Cluster facts (verified)
 
 - **Registry:** `gpu-rail-registry` (default ns), NodePort **30500** → 5000,
-  ClusterIP `10.43.10.176`. Insecure HTTP registry, trusted cluster-wide.
-  - **Push from workshop:** via a node NodePort, e.g.
-    `192.168.2.44:30500` (k3s-worker) or `192.168.2.43:30500` (k3s-cp).
-  - **Pull in-cluster:** the working ref is
-    **`gpu-rail-registry.default.svc.cluster.local:5000/ovmx-builder:latest`**
-    (see "Registry notes" below for why this one and not the NodePort).
+  ClusterIP `10.43.10.176`. Insecure **HTTP** registry.
+  - **Working ref for BOTH push and pull:**
+    **`192.168.2.43:30500/ovmx-builder:latest`** (the k3s-cp NodePort). This is
+    the exact pattern the existing ovmx-lab StatefulSets use
+    (`192.168.2.43:30500/ovmx-vaxlab:3`), i.e. the proven-trusted address.
+  - The in-cluster Service DNS name
+    (`gpu-rail-registry.default.svc.cluster.local:5000`) does **not** work for
+    image pulls — the node's containerd/kubelet resolve image hosts via the
+    *node's* resolv.conf, not CoreDNS, so it fails with
+    `lookup ...svc.cluster.local: Try again`. Use the NodePort IP. See
+    "Registry notes" below.
 - **Target node:** `k3s-worker` (192.168.2.44, 8 CPU / 53 GB, KVM-capable bare
   metal). The Job pins `nodeSelector kubernetes.io/hostname=k3s-worker`. It
   never targets k3s-cp (small), k3s-mini (NotReady), or the GPU node.
@@ -69,25 +74,32 @@ acceptable as a one-time bootstrap.
 
 ```bash
 # from the repo root on workshop:
-docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.44:30500/ovmx-builder:latest .
-docker push 192.168.2.44:30500/ovmx-builder:latest
+docker build -f tools/k3s/Dockerfile.rail -t 192.168.2.43:30500/ovmx-builder:latest .
+docker push 192.168.2.43:30500/ovmx-builder:latest
 ```
 
 ### Registry notes (the parts that fight you)
 
-- The registry serves **HTTP** (insecure). Workshop's Docker already pushes to
-  it (other tenants do), and the k3s nodes' containerd already trusts it (they
-  pull other tenants' images from it), so **no per-node TLS/insecure config was
-  needed** for either push or pull.
-- **Push tag vs. pull ref differ on purpose.** You push to a *NodePort*
-  (`192.168.2.44:30500`) because that's reachable from workshop. Pods pull with
-  the *in-cluster* Service DNS name
-  (`gpu-rail-registry.default.svc.cluster.local:5000`), which resolves to the
-  Service ClusterIP `10.43.10.176:5000`. Both names address the same registry
-  storage, so an image pushed at the NodePort tag is pullable at the in-cluster
-  tag — they are the same blob under two names. If DNS ever misbehaves, the
-  ClusterIP form `10.43.10.176:5000/ovmx-builder:latest` works identically
-  (set `OVMX_RAIL_IMAGE`).
+- The registry serves **HTTP** (insecure). The k3s nodes' containerd already
+  trusts it (they pull other tenants' images from it), so **no per-node config
+  was needed for pull**.
+- **Push must use the NodePort workshop's Docker daemon already trusts.** The
+  registry is plain HTTP, so `docker push` needs the target in
+  `/etc/docker/daemon.json` `insecure-registries`. Workshop already lists
+  **`192.168.2.43:30500`** (k3s-cp) there but *not* `192.168.2.44:30500`
+  (k3s-worker) — pushing to `.44` fails with
+  `http: server gave HTTP response to HTTPS client`. Push to **`.43`** (no
+  shared-host config change) — it's the same registry storage as `.44`.
+- **Same ref for push and pull: `192.168.2.43:30500/ovmx-builder:latest`.**
+  Push goes there because workshop's daemon trusts it; pods pull the same tag
+  because the nodes' containerd trusts that address too *and* it is a bare IP
+  needing no DNS. Do **not** use the Service DNS name for the image ref — the
+  node resolves image hosts via its own resolv.conf, not CoreDNS, so
+  `gpu-rail-registry.default.svc.cluster.local:5000` ImagePullBackOffs with
+  `lookup ...: Try again` (verified). The ClusterIP form
+  `10.43.10.176:5000/...` avoids DNS but is only pullable if the nodes'
+  containerd trusts that IP as insecure too — the NodePort IP is the known-good
+  path, so that is the default (`OVMX_RAIL_IMAGE` to override).
 - `imagePullPolicy: Always` — the tag is `:latest` and mutable; always re-pull
   so a rebuilt builder image is picked up.
 
diff --git a/tools/k3s/namespace.yaml b/tools/k3s/namespace.yaml
index 01c3c6a7c..7b23787f5 100644
--- a/tools/k3s/namespace.yaml
+++ b/tools/k3s/namespace.yaml
@@ -28,12 +28,17 @@ metadata:
   name: ovmx-ci-quota
   namespace: ovmx-ci
 spec:
+  # Generous headroom (operator: rail capacity is ample, don't let the quota
+  # gate a wave of parallel jobs). Actual concurrency is still bounded by
+  # k3s-worker's real free CPU -- the scheduler queues what doesn't fit rather
+  # than failing it -- so this cap only exists to keep a runaway from claiming
+  # the entire node against the shared tenants, not to serialize normal CI.
   hard:
-    requests.cpu: "6"
-    requests.memory: 40Gi
-    limits.cpu: "6"
-    limits.memory: 40Gi
-    pods: "8"
+    requests.cpu: "12"
+    requests.memory: 96Gi
+    limits.cpu: "64"
+    limits.memory: 192Gi
+    pods: "16"
 ---
 apiVersion: v1
 kind: LimitRange
diff --git a/tools/k3s/run-on-rail.sh b/tools/k3s/run-on-rail.sh
index 7d8615411..f8e956c13 100755
--- a/tools/k3s/run-on-rail.sh
+++ b/tools/k3s/run-on-rail.sh
@@ -30,10 +30,20 @@ set -euo pipefail
 # --- config -----------------------------------------------------------------
 NS=ovmx-ci
 REPO_URL="${OVMX_REPO_URL:-https://github.com/3dl-dev/vms}"
-IMAGE="${OVMX_RAIL_IMAGE:-gpu-rail-registry.default.svc.cluster.local:5000/ovmx-builder:latest}"
+# The working image ref for BOTH push and pull is the k3s-cp registry NodePort
+# (192.168.2.43:30500): the nodes' containerd trusts it as insecure HTTP and it
+# is a bare IP so it needs no cluster-DNS resolution (kubelet/containerd resolve
+# image hosts via the NODE's resolv.conf, not CoreDNS -- the in-cluster Service
+# DNS name does NOT work for image pulls). See README "Registry notes".
+IMAGE="${OVMX_RAIL_IMAGE:-192.168.2.43:30500/ovmx-builder:latest}"
 DEADLINE="${OVMX_RAIL_DEADLINE:-3600}"    # activeDeadlineSeconds (wall cap)
-REQ_CPU="${OVMX_RAIL_REQ_CPU:-4}"
-REQ_MEM="${OVMX_RAIL_REQ_MEM:-8Gi}"
+# Requests are kept modest so the pod actually SCHEDULES on k3s-worker, which
+# runs shared tenants (~5 of its 8 CPU are already requested). The limit still
+# lets the build burst wide (cmake -j$(nproc)); CFS hands it spare CPU when the
+# node has it. Bump OVMX_RAIL_REQ_CPU if you want a guaranteed-wide build and
+# the node has headroom.
+REQ_CPU="${OVMX_RAIL_REQ_CPU:-1}"
+REQ_MEM="${OVMX_RAIL_REQ_MEM:-4Gi}"
 LIM_CPU="${OVMX_RAIL_LIM_CPU:-6}"
 LIM_MEM="${OVMX_RAIL_LIM_MEM:-24Gi}"
 
@@ -103,17 +113,18 @@ done
 [ -n "$POD" ] || { echo "[run-on-rail] pod never appeared" >&2; exit 1; }
 
 # Wait until the pod is out of Pending (Running or already terminated), so
-# `logs -f` has something to stream. Surface scheduling failures early.
-for _ in $(seq 1 300); do
+# `logs -f` has something to stream. Surface pull/schedule failures instead of
+# hanging silently, and never fall through to `logs -f` on a still-Pending pod.
+PHASE=""
+for _ in $(seq 1 600); do
   PHASE="$(kubectl -n "$NS" get pod "$POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
   case "$PHASE" in
     Running|Succeeded|Failed) break ;;
   esac
-  # Show pull/schedule trouble instead of hanging silently.
   WAITMSG="$(kubectl -n "$NS" get pod "$POD" \
     -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
   case "$WAITMSG" in
-    ErrImagePull|ImagePullBackOff|CreateContainerError|InvalidImageName)
+    ErrImagePull|ImagePullBackOff|CreateContainerError|InvalidImageName|RunContainerError)
       echo "[run-on-rail] pod stuck: $WAITMSG" >&2
       kubectl -n "$NS" describe pod "$POD" 2>&1 | sed -n '/Events:/,$p' >&2
       exit 1 ;;
@@ -121,6 +132,14 @@ for _ in $(seq 1 300); do
   sleep 1
 done
 
+case "$PHASE" in
+  Running|Succeeded|Failed) ;;
+  *)
+    echo "[run-on-rail] pod still $PHASE after wait -- not started. Recent events:" >&2
+    kubectl -n "$NS" describe pod "$POD" 2>&1 | sed -n '/Events:/,$p' >&2
+    exit 1 ;;
+esac
+
 # --- stream logs ------------------------------------------------------------
 echo "[run-on-rail] pod=$POD phase=$PHASE -- streaming logs:" >&2
 echo "----------------------------------------------------------------------" >&2

From a000da07f9f6702cdd260adf88d4e1ac67ccf63a Mon Sep 17 00:00:00 2001
From: alice <baron@3dl.dev>
Date: Thu, 20 Aug 2026 19:34:48 +0000
Subject: [PATCH 3/5] vms-101: complete builder image deps so full ctest is
 green on the rail

Verified on k3s-worker: full build (232 targets) + `ctest` = 100% passed,
0 failed out of 195, exit 0. Real exit-code propagation also proven (a red
ctest returned rc 8 -> script exit 8).

Added the deps the plain-ctest path needs that the tests/qemu toolchain layer
lacked (the authenticity symbol-scan gate compiles every product source, so a
missing build header is a hard FAIL, not a skip):
  - libssh-dev, libreadline-dev  (src/vmsssh, DCL readline)
  - strace, gawk, binutils       (integration gates: monitor/show-device/register)
  - ccache, clang                (match ci.yml's ctest job env)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tools/k3s/Dockerfile.rail | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/k3s/Dockerfile.rail b/tools/k3s/Dockerfile.rail
index 428b0cbb4..d76cfe391 100644
--- a/tools/k3s/Dockerfile.rail
+++ b/tools/k3s/Dockerfile.rail
@@ -42,6 +42,8 @@ RUN apt-get update && \
         musl-tools musl-dev linux-libc-dev \
         libelf-dev bc flex bison \
         python3 file bsdmainutils \
+        binutils gawk strace ccache clang \
+        libssh-dev libreadline-dev \
         genisoimage xorriso \
         "$QEMU_PKG" qemu-utils \
     && rm -rf /var/lib/apt/lists/*

From 97ba6f35e5b52d1e4718cd3b253af2aa7350a3fd Mon Sep 17 00:00:00 2001
From: alice <baron@3dl.dev>
Date: Thu, 20 Aug 2026 20:16:46 +0000
Subject: [PATCH 4/5] vms-101: add --dind mode so docker-wrapped QEMU e2e run
 on the rail

The reap's remaining e2e verification is docker-wrapped (docker build
distro/Dockerfile.bootable then docker run ... qemu) and SKIP(77)s in the plain
builder pod for lack of docker. --dind starts an in-container dockerd so those
gates run on cluster hardware.

- run-on-rail.sh --dind: sets OVMX_DIND=1; the pod starts dockerd (overlay2,
  vfs fallback) and waits for it before running the command. Non-dind path
  unchanged.
- dockerd runs IN the same container as the repo (not a sidecar) on purpose:
  the e2e gates use `docker run -v $REPO_ROOT/...` bind mounts, which resolve on
  the DAEMON's filesystem -- a sidecar daemon would resolve them on the wrong
  host and break the mounts.
- job-template.yaml: OVMX_DIND env + a node-backed emptyDir at /var/lib/docker
  (so overlay2 works off the pod rootfs) + gated dockerd preamble. /dev/kvm is
  already in the pod, so an inner `docker run --device /dev/kvm ... -accel kvm`
  is KVM-accelerated (nested KVM).
- Dockerfile.rail: add docker.io iptables uidmap.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tools/k3s/Dockerfile.rail   |  6 ++++++
 tools/k3s/job-template.yaml | 38 ++++++++++++++++++++++++++++++++++++-
 tools/k3s/run-on-rail.sh    | 14 +++++++++++---
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/tools/k3s/Dockerfile.rail b/tools/k3s/Dockerfile.rail
index d76cfe391..d7cf17ed2 100644
--- a/tools/k3s/Dockerfile.rail
+++ b/tools/k3s/Dockerfile.rail
@@ -27,6 +27,11 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # One apt layer, one index fetch (same discipline as tests/qemu/Dockerfile).
 # The QEMU package name is arch-conditional; everything else is arch-neutral.
+# docker.io/iptables are for run-on-rail.sh --dind mode (dockerd runs INSIDE
+# this container so the docker-wrapped e2e gates -- which `docker build
+# distro/Dockerfile.bootable` then `docker run -v $REPO_ROOT/...` -- resolve
+# their bind mounts against the same filesystem the repo is cloned into. A
+# separate dind sidecar would resolve those mounts on the wrong host.
 RUN apt-get update && \
     ARCH=$(dpkg --print-architecture) && \
     if [ "$ARCH" = "arm64" ]; then QEMU_PKG=qemu-system-arm; else QEMU_PKG=qemu-system-x86; fi && \
@@ -44,6 +49,7 @@ RUN apt-get update && \
         python3 file bsdmainutils \
         binutils gawk strace ccache clang \
         libssh-dev libreadline-dev \
+        docker.io iptables uidmap \
         genisoimage xorriso \
         "$QEMU_PKG" qemu-utils \
     && rm -rf /var/lib/apt/lists/*
diff --git a/tools/k3s/job-template.yaml b/tools/k3s/job-template.yaml
index 5ae8c2a84..11d836b58 100644
--- a/tools/k3s/job-template.yaml
+++ b/tools/k3s/job-template.yaml
@@ -47,6 +47,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: OVMX_DIND
+              value: "${DIND}"       # "1" enables the in-container dockerd (--dind)
           resources:
             requests:
               cpu: "${REQ_CPU}"
@@ -57,11 +59,43 @@ spec:
           volumeMounts:
             - name: kvm
               mountPath: /dev/kvm
+            # dockerd's graph on a node-backed emptyDir (not the pod overlay
+            # rootfs) so overlay2 works and the image build isn't capped by the
+            # container's writable layer. Unused (harmless) in non-dind mode.
+            - name: docker-storage
+              mountPath: /var/lib/docker
           command: ["/bin/bash", "-c"]
           args:
             - |
               set -euo pipefail
-              echo "[rail] node=${NODE_NAME} pod=$(hostname) ref=${GIT_REF}"
+              echo "[rail] node=${NODE_NAME} pod=$(hostname) ref=${GIT_REF} dind=${OVMX_DIND}"
+
+              # --- optional in-container dockerd (run-on-rail.sh --dind) --------
+              # dockerd runs HERE (not a sidecar) so the docker-wrapped e2e gates'
+              # `docker run -v $REPO_ROOT/...` bind mounts resolve against this
+              # same filesystem, where the repo is cloned. /dev/kvm is already in
+              # this pod, so an inner `docker run --device /dev/kvm ... -accel kvm`
+              # is KVM-accelerated (nested KVM verified).
+              if [ "${OVMX_DIND:-0}" = "1" ]; then
+                echo "[rail] starting in-container dockerd..."
+                _start_dockerd() { dockerd "$@" >/var/log/dockerd.log 2>&1 & DOCKERD_PID=$!; }
+                _wait_dockerd() {
+                  for _ in $(seq 1 60); do
+                    docker info >/dev/null 2>&1 && return 0
+                    kill -0 "$DOCKERD_PID" 2>/dev/null || return 1
+                    sleep 1
+                  done; return 1
+                }
+                _start_dockerd
+                if ! _wait_dockerd; then
+                  echo "[rail] dockerd (overlay2) did not come up; retrying with vfs"
+                  tail -20 /var/log/dockerd.log || true
+                  _start_dockerd --storage-driver=vfs
+                  _wait_dockerd || { echo "[rail] FATAL: dockerd failed to start"; tail -40 /var/log/dockerd.log; exit 1; }
+                fi
+                echo "[rail] dockerd up: $(docker version --format '{{.Server.Version}}' 2>/dev/null) driver=$(docker info --format '{{.Driver}}' 2>/dev/null)"
+              fi
+
               echo "[rail] cloning ${REPO_URL}"
               git clone --filter=blob:none "${REPO_URL}" repo
               cd repo
@@ -78,3 +112,5 @@ spec:
           hostPath:
             path: /dev/kvm
             type: CharDevice
+        - name: docker-storage
+          emptyDir: {}
diff --git a/tools/k3s/run-on-rail.sh b/tools/k3s/run-on-rail.sh
index f8e956c13..6d6df3c10 100755
--- a/tools/k3s/run-on-rail.sh
+++ b/tools/k3s/run-on-rail.sh
@@ -6,7 +6,7 @@
 # host onto cluster hardware (k3s-worker). It is NOT an OVMX runtime.
 #
 # Usage:
-#   tools/k3s/run-on-rail.sh [--keep] [--name NAME] <git-ref> <command...>
+#   tools/k3s/run-on-rail.sh [--keep] [--dind] [--name NAME] <git-ref> <command...>
 #
 #   <git-ref>    branch, tag, or full SHA of https://github.com/3dl-dev/vms
 #   <command>    shell run in the repo root inside the pod; its exit code
@@ -14,6 +14,12 @@
 #
 # Flags:
 #   --keep       do not delete the Job when it finishes (for debugging)
+#   --dind       start an in-container dockerd first, so the command can use
+#                `docker build`/`docker run` -- required for the docker-wrapped
+#                QEMU e2e gates (run_install_boot_e2e.sh, sysboot-cluster-params,
+#                etc.) that otherwise SKIP(77) for lack of docker. /dev/kvm is
+#                present so an inner `docker run --device /dev/kvm ... -accel kvm`
+#                is KVM-accelerated (nested KVM).
 #   --name NAME  base name for the Job (default: ovmx-ci)
 #
 # Examples:
@@ -51,10 +57,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # --- args -------------------------------------------------------------------
 KEEP=0
+DIND=0
 NAME_BASE=ovmx-ci
 while [ $# -gt 0 ]; do
   case "$1" in
     --keep) KEEP=1; shift ;;
+    --dind) DIND=1; shift ;;
     --name) NAME_BASE="$2"; shift 2 ;;
     --)     shift; break ;;
     -h|--help) sed -n '2,40p' "$0"; exit 0 ;;
@@ -83,9 +91,9 @@ OVMX_CMD_B64="$(printf '%s' "$CMD" | base64 | tr -d '\n')"
 JOB_NAME="${NAME_BASE}-$(date +%s)-${RANDOM}"
 
 export JOB_NAME IMAGE GIT_REF REPO_URL OVMX_CMD_B64 DEADLINE \
-       REQ_CPU REQ_MEM LIM_CPU LIM_MEM
+       REQ_CPU REQ_MEM LIM_CPU LIM_MEM DIND
 MANIFEST="$(envsubst \
-  '$JOB_NAME $IMAGE $GIT_REF $REPO_URL $OVMX_CMD_B64 $DEADLINE $REQ_CPU $REQ_MEM $LIM_CPU $LIM_MEM' \
+  '$JOB_NAME $IMAGE $GIT_REF $REPO_URL $OVMX_CMD_B64 $DEADLINE $REQ_CPU $REQ_MEM $LIM_CPU $LIM_MEM $DIND' \
   < "$SCRIPT_DIR/job-template.yaml")"
 
 cleanup() {

From cbe2c0f5de67aaf7495a37dc9c50b7f878b5e15e Mon Sep 17 00:00:00 2001
From: alice <baron@3dl.dev>
Date: Thu, 20 Aug 2026 23:42:42 +0000
Subject: [PATCH 5/5] vms-101: dind trusts the insecure registry so inner
 docker run works

The --dind image now ships dockerd (Dockerfile.rail already carried
docker.io; the registry image just predated it and has been rebuilt +
pushed). This wires the in-pod dockerd to the same plain-HTTP registry
the nodes' containerd trusts (192.168.2.43:30500) via
--insecure-registry, so an inner `docker pull`/`docker run
<registry-image>` resolves in --dind -- needed for the nested-KVM proof
that boots the builder image under `docker run --device /dev/kvm ...
-accel kvm`.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tools/k3s/job-template.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/k3s/job-template.yaml b/tools/k3s/job-template.yaml
index 11d836b58..e26f2f4c2 100644
--- a/tools/k3s/job-template.yaml
+++ b/tools/k3s/job-template.yaml
@@ -78,7 +78,13 @@ spec:
               # is KVM-accelerated (nested KVM verified).
               if [ "${OVMX_DIND:-0}" = "1" ]; then
                 echo "[rail] starting in-container dockerd..."
-                _start_dockerd() { dockerd "$@" >/var/log/dockerd.log 2>&1 & DOCKERD_PID=$!; }
+                # The in-pod dockerd must trust the same plain-HTTP registry the
+                # nodes' containerd does (192.168.2.43:30500) so an inner
+                # `docker pull`/`docker run <registry-image>` works in --dind --
+                # e.g. the nested-KVM proof that runs the builder image under
+                # `docker run --device /dev/kvm ... -accel kvm`.
+                DOCKERD_INSECURE="--insecure-registry 192.168.2.43:30500"
+                _start_dockerd() { dockerd $DOCKERD_INSECURE "$@" >/var/log/dockerd.log 2>&1 & DOCKERD_PID=$!; }
                 _wait_dockerd() {
                   for _ in $(seq 1 60); do
                     docker info >/dev/null 2>&1 && return 0