ci(e2e): run Qwen and DS4 end to end on lucebox3

.github/ci/e2e/README.md

@@ -0,0 +1,47 @@
+# Model e2e on lucebox3
+
+`model-e2e.yml` runs on lucebox3's self-hosted runner: Qwen3.8-27B on the R9700
+(gfx1201, HIP index 0) and DeepSeek V4 Flash on the Strix Halo (gfx1151, HIP
+index 1). Each model's files, flags and GPU are in `select_models.py`.
+
+## What lucebox3 needs
+
+- **The models** in `/opt/models` (or the repository variable
+  `LUCEBOX_MODELS_DIR`):
+  - `Qwen3.8-27B-UD-IQ4_XS.gguf`
+  - `qwen38-dflash2-q8_0.gguf` (the draft)
+  - `DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf`
+
+  A missing file skips a PR's job and fails a baseline run.
+- **A readable kernel log**, so GPU faults during the run are caught: passwordless
+  `sudo dmesg` for the runner user (as `gpu-tests-amd` already uses), or
+  `kernel.dmesg_restrict=0`. Without it every run warns that GPU errors were not
+  checked.
+- **ccache** (optional): cold builds take ~100 s without it. The build directory
+  and remembered passes live in the runner user's `~/.cache/lucebox-e2e`.
+
+The job sees every user's GPU processes through `/sys/class/kfd/kfd/proc`, which
+any user can read.
+
+## Baselines
+
+Every merge to main runs the models whose code changed since their baseline
+and uploads each result as the artifact `model-e2e-baseline-<model>-<device>`
+(kept 90 days) unless it fails. Every job compares with the newest one, so a
+merged change that alters the output becomes the reference for the next PRs. To
+refresh a baseline by hand, e.g. after a ROCm upgrade, run the workflow on main
+with `update_baseline` ticked. Until the first baseline exists, jobs still fail on
+crashes, hangs and failed checks but cannot detect changed output.
+
+## Benchmarking by hand on lucebox3
+
+A job waits up to 4 minutes for other GPU users, then skips with a warning. To
+keep jobs off the machine for longer, stop its runner service
+(`sudo ./svc.sh stop` in the runner directory) and start it again when you're
+done. This also pauses `gpu-tests-amd`.
+
+## Running the tests
+
+```bash
+cd .github/ci/e2e && uv run --with pytest --no-project python -m pytest -q
+```

.github/ci/e2e/find_baseline.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Find each model's baseline: the newest baseline artifact from a main run.
+
+Baseline runs on main (every merge, or dispatched with update_baseline) upload
+their result as the artifact `model-e2e-baseline-<model>-<device>`. This reads
+the select job's matrix on stdin and adds to each entry the `baseline_run` that
+holds its baseline, or "" when there is none yet.
+
+With --only-changed HEAD, it keeps only the models whose code changed between
+their baseline's commit and HEAD (or that have no baseline): a merge that
+cannot change a model's output does not need a new baseline. Measuring from the
+baseline rather than the previous commit means a merge whose run was skipped
+or failed is picked up by the next one.
+
+Only artifacts from pushes to or dispatched runs on this repository's main
+count: pull request code runs in the same workflow and could upload an artifact
+with the same name.
+
+    python3 find_baseline.py --repo OWNER/REPO [--only-changed SHA] < matrix.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.parse
+import urllib.request
+from collections.abc import Callable
+
+from select_models import models_for
+
+WORKFLOW = ".github/workflows/model-e2e.yml"
+TRUSTED_EVENTS = ("push", "workflow_dispatch")
+# The compare API lists at most this many files; a longer diff counts as all changed.
+COMPARE_FILE_LIMIT = 300
+
+Api = Callable[[str], dict]
+
+
+def artifact_name(entry: dict) -> str:
+    return f"model-e2e-baseline-{entry['model']}-{entry['device']}"
+
+
+def github_api(token: str) -> Api:
+    def get(path: str) -> dict:
+        request = urllib.request.Request(
+            f"https://api.github.com/{path}",
+            headers={
+                "Accept": "application/vnd.github+json",
+                "Authorization": f"Bearer {token}",
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+        )
+        with urllib.request.urlopen(request, timeout=30) as response:
+            return json.load(response)
+
+    return get
+
+
+def trusted(run: dict, repo: str) -> bool:
+    return (
+        run.get("event") in TRUSTED_EVENTS
+        and run.get("head_branch") == "main"
+        and (run.get("head_repository") or {}).get("full_name") == repo
+        and (run.get("path") or "").split("@")[0] == WORKFLOW
+    )
+
+
+def find_run(api: Api, repo: str, name: str) -> dict:
+    """The newest trusted run that uploaded `name`, or {}."""
+    query = urllib.parse.urlencode({"name": name, "per_page": 50})
+    artifacts = api(f"repos/{repo}/actions/artifacts?{query}").get("artifacts", [])
+    artifacts = [a for a in artifacts if a.get("name") == name and not a.get("expired")]
+    artifacts.sort(key=lambda a: a.get("created_at") or "", reverse=True)
+    for artifact in artifacts:
+        run_id = (artifact.get("workflow_run") or {}).get("id")
+        if run_id:
+            run = api(f"repos/{repo}/actions/runs/{run_id}")
+            if trusted(run, repo):
+                return run
+    return {}
+
+
+def add_baselines(matrix: dict, api: Api, repo: str) -> dict:
+    found: dict[str, dict] = {}
+    for entry in matrix["include"]:
+        name = artifact_name(entry)
+        if name not in found:
+            found[name] = find_run(api, repo, name)
+        entry["baseline_run"] = str(found[name].get("id") or "")
+        entry["baseline_commit"] = found[name].get("head_sha") or ""
+    return matrix
+
+
+def keep_changed(matrix: dict, api: Api, repo: str, head: str) -> dict:
+    """Drop the entries whose model code is unchanged since their baseline."""
+    kept = []
+    for entry in matrix["include"]:
+        base = entry["baseline_commit"]
+        if base:
+            files = api(f"repos/{repo}/compare/{base}...{head}").get("files", [])
+            paths = [f["filename"] for f in files]
+            if len(files) < COMPARE_FILE_LIMIT and entry["model"] not in models_for(paths):
+                continue
+        kept.append(entry)
+    return {"include": kept}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+    parser.add_argument("--repo", required=True, help="OWNER/REPO")
+    parser.add_argument(
+        "--only-changed",
+        metavar="SHA",
+        help="keep only the models whose code changed between their baseline and SHA",
+    )
+    args = parser.parse_args(argv)
+    token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") or ""
+    api = github_api(token)
+    matrix = add_baselines(json.load(sys.stdin), api, args.repo)
+    for entry in matrix["include"]:
+        where = f"run {entry['baseline_run']}" if entry["baseline_run"] else "none yet"
+        print(f"Baseline for {entry['model']}: {where}", file=sys.stderr)
+    if args.only_changed:
+        matrix = keep_changed(matrix, api, args.repo, args.only_changed)
+        kept = [e["model"] for e in matrix["include"]]
+        print(f"Changed since their baseline: {', '.join(kept) or 'none'}", file=sys.stderr)
+    print(json.dumps(matrix, separators=(",", ":")))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/ci/e2e/gpu_wait.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Wait until no process holds an AMD GPU, so the e2e run does not share the
+# machine with someone's manual server or benchmark.
+#
+# Any KFD process counts: /dev/kfd is shared by every AMD GPU, and a model run
+# on the other GPU still competes for host memory and CPU. The holders come
+# from the kernel's KFD process list, /sys/class/kfd/kfd/proc/<pid>, which
+# lists every user's processes and which any user can read.
+#
+# The last line printed is state=free|busy. Exits 1 when the process list
+# cannot be read, since a busy machine would then look free.
+#
+# Usage: gpu_wait.sh [max seconds to wait, default 300]
+set -u
+
+deadline=$((SECONDS + ${1:-300}))
+# Overridable for the tests.
+kfd_procs=${KFD_PROC_DIR:-/sys/class/kfd/kfd/proc}
+
+if [ ! -r "$kfd_procs" ] || [ ! -x "$kfd_procs" ]; then
+  echo "::error title=GPU check unavailable::Cannot read $kfd_procs to see who holds the GPUs"
+  exit 1
+fi
+
+while :; do
+  # An entry outlives its process for a moment while the driver frees the GPU
+  # memory; count it until it is gone.
+  pids=$(ls -A "$kfd_procs")
+  if [ -z "$pids" ]; then
+    echo "No process holds the GPUs; they are free."
+    echo "state=free"
+    exit 0
+  fi
+  if [ "$SECONDS" -ge "$deadline" ]; then
+    echo "::warning title=GPU busy::Other processes still hold the GPUs; skipping the model e2e run."
+    for pid in $pids; do
+      if ! line=$(ps -o pid=,user=,etime=,args= -p "$pid"); then
+        line="$pid (exited, but the GPU driver still holds its state)"
+      fi
+      echo "${line:0:200}"
+    done
+    echo "state=busy"
+    exit 0
+  fi
+  echo "GPU busy (PIDs: $(echo "$pids" | tr '\n' ' ')); waiting..."
+  left=$((deadline - SECONDS))
+  sleep $((left < 20 ? left : 20))
+done

.github/ci/e2e/prompts.json

@@ -0,0 +1,117 @@
+[
+  {
+    "id": "arith",
+    "messages": [{"role": "user", "content": "What is 17 * 23? Reply with only the number."}],
+    "max_tokens": 16,
+    "check": {"number": 391}
+  },
+  {
+    "id": "capital",
+    "messages": [{"role": "user", "content": "What is the capital of France? Reply with one word."}],
+    "max_tokens": 16,
+    "check": {"contains": ["paris"]}
+  },
+  {
+    "id": "primes",
+    "messages": [{"role": "user", "content": "List the first five prime numbers, separated by commas. Reply with only the list."}],
+    "max_tokens": 32,
+    "check": {"regex": "2\\D+3\\D+5\\D+7\\D+11"}
+  },
+  {
+    "id": "word_problem",
+    "messages": [{"role": "user", "content": "A train travels 60 km in 1.5 hours. What is its average speed in km/h? Reply with only the number."}],
+    "max_tokens": 16,
+    "check": {"number": 40}
+  },
+  {
+    "id": "count",
+    "messages": [{"role": "user", "content": "Count from 1 to 40, separated by single spaces. Reply with only the numbers."}],
+    "max_tokens": 160,
+    "check": {"sequence": 40}
+  },
+  {
+    "id": "code",
+    "messages": [{"role": "user", "content": "Write a Python function is_even(n) that returns True when n is even. Reply with only the code."}],
+    "max_tokens": 96,
+    "check": {"contains": ["def is_even", "% 2"]}
+  },
+  {
+    "id": "json",
+    "messages": [{"role": "user", "content": "Return a JSON object with the key \"name\" set to \"lucebox\" and the key \"version\" set to 3. Reply with only the JSON."}],
+    "max_tokens": 48,
+    "check": {"json": {"name": "lucebox", "version": 3}}
+  },
+  {
+    "id": "translate",
+    "messages": [{"role": "user", "content": "Translate \"good morning\" into Italian. Reply with only the translation."}],
+    "max_tokens": 16,
+    "check": {"any": ["buongiorno", "buon giorno"]}
+  },
+  {
+    "id": "unicode",
+    "messages": [{"role": "user", "content": "Repeat this text exactly: Città 東京 🚀"}],
+    "max_tokens": 24,
+    "check": {"contains": ["città", "東京", "🚀"]}
+  },
+  {
+    "id": "multi_turn",
+    "messages": [
+      {"role": "system", "content": "You are a concise assistant."},
+      {"role": "user", "content": "My name is Alice and I live in Lisbon."},
+      {"role": "assistant", "content": "Nice to meet you, Alice."},
+      {"role": "user", "content": "In which city do I live? Reply with one word."}
+    ],
+    "max_tokens": 16,
+    "check": {"contains": ["lisbon"]}
+  },
+  {
+    "id": "tool_call",
+    "messages": [{"role": "user", "content": "What is the weather in Rome right now?"}],
+    "tools": [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "description": "Get the current weather for a city.",
+          "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string"}},
+            "required": ["city"]
+          }
+        }
+      }
+    ],
+    "max_tokens": 160,
+    "check": {"tool_call": {"name": "get_weather", "arguments_contain": {"city": "rome"}}}
+  },
+  {
+    "id": "needle",
+    "generate": "needle",
+    "max_tokens": 16,
+    "check": {"contains": ["7429"]}
+  },
+  {
+    "id": "stream",
+    "stream": true,
+    "messages": [{"role": "user", "content": "What is the capital of Japan? Reply with one word."}],
+    "max_tokens": 16,
+    "check": {"contains": ["tokyo"]}
+  },
+  {
+    "id": "thinking",
+    "thinking": true,
+    "messages": [{"role": "user", "content": "What is 12 + 30?"}],
+    "max_tokens": 768,
+    "check": {"number": 42, "field": "any"}
+  },
+  {
+    "id": "story",
+    "messages": [{"role": "user", "content": "Write a short paragraph about a lighthouse keeper."}],
+    "max_tokens": 192,
+    "check": {"min_words": 40, "no_loop": true}
+  },
+  {
+    "id": "arith_repeat",
+    "repeat_of": "arith"
+  }
+]