diff --git a/.gitignore b/.gitignore index 6bc2dce..7b75c04 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,10 @@ ops/runbooks/* # Node (for web UI) node_modules/ + +# Agent tooling scratch dirs — local session state, never product code. +# (These were untracked and got swept in by a `git add -A`; ignoring them so a +# stray bulk-add can't publish embedded git repos into the product repo.) +.claude/ +.cursor/ +.omx/ diff --git a/AGENTS.md b/AGENTS.md index c5ce646..8d300ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ State / architecture / decisions / "why": Obsidian Vault → `AINode` (cluster o ## Operational source of truth -- All work on `codex/*` branches; PRs required — **never push directly to `main`**. +- All work on `fable/*` branches (renamed from `codex/*` 2026-08-15 — the old prefix came from OpenAI Codex; no CI keys on either, so existing `codex/*` branches are fine to leave). PRs required — **never push directly to `main`**. - Build/test: `pip install -e ".[dev]"` → `pytest tests/` · lint `ruff check`. Base image: `scripts/build-base-image.sh`; app image: `docker build -f scripts/Dockerfile.ainode`. - Handoffs use the threadmaster-handoff runbook; ops state lives in `ops/` (runbooks under `ops/runbooks/`). - Distribution is `docker pull` only — end users never hand-edit vLLM commands; the engine emits flags (see `engine/AGENTS.md`). diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7d61c..9cc47ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,19 @@ Versions follow [Semantic Versioning](https://semver.org/). --- +## [0.5.3] — 2026-07-07 + +### Fixed +- **Truthful instances everywhere** (#59) — every instance card shows its node + NAME (not a hex id); the Server view lists stacked instances with node + port + and its count matches reality (Eject only where it can actually target); + per-instance status probes both directions (`starting → serving`, and back to + `failed` when an engine dies — no more stale STARTING bars or false READY); + the top-bar update banner now updates the whole fleet + (`/api/cluster/update-all`) with an honest confirm, not just the head node. + +--- + ## [0.5.2] — 2026-07-06 > The two majors from the 0.5.1 live lifecycle verification. Image published to GHCR; diff --git a/CLAUDE.md b/CLAUDE.md index 4ee472b..efbbd5c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,7 +81,7 @@ launcher handles SSH to peers, Ray head/worker formation, and NCCL. ## Working Conventions - Follow ops-approved workflow (see ops/) -- All work on `codex/*` branches +- All work on `fable/*` branches (was `codex/*` until 2026-08-15) - PRs required — never push directly to main - Handoffs use the threadmaster-handoff runbook - Test on real GPU hardware when possible diff --git a/FOLLOWUPS.md b/FOLLOWUPS.md new file mode 100644 index 0000000..db0d8c0 --- /dev/null +++ b/FOLLOWUPS.md @@ -0,0 +1,54 @@ +# FOLLOWUPS (Dart-unreachable fallback — migrate to Dart board when authed) + +## [ainode/lab] Replace the flaky 10G mgmt switch (root cause of the Jul 26–Aug 13 outage) +- **Filed:** 2026-08-13 (fleet restored this session; switch is revived but NOT trusted) +- **Owner:** Jason. The 10G copper switch feeding spark3+spark4 mgmt (+ the `192.168.0.100` device + ROSA `ether3`) flapped Jun 18 and Jul 6, died Jul 26 16:33 (same electrical event tripped spark2's outlet), and needed a power-cycle + one cable reseat (spark3's port) to revive on 2026-08-13. Classic dying PSU / failing unit. +- **Next action:** replace the switch (or at minimum its PSU); while at it, identify what owns `192.168.0.100` — still down after recovery, either on a dead port or powered off. Evidence + topology: `ops/runbooks/network-topology.md` outage note. +- **Proof of closure:** new/verified switch in place; 2 weeks with no synchronized link-flaps in spark3/4 `journalctl -k`; `.100` owner identified and documented in the runbook. + +## [ainode] BUG: eject doesn't survive reboot + phantom rows (2 of 5 FIXED 2026-08-15) +- **FIXED on `fable/0.5.4-native-engines`:** (a) engine containers no longer launch with `--rm`, so a crashed engine leaves a readable corpse (validated live: the entrypoint-collision crash left its "unrecognized arguments" error intact instead of self-erasing); (b) `start_solo()` now confirms the container reached Running and logs the engine's last output on failure, instead of returning True as soon as the docker CLI forked. +- **ALSO FIXED 2026-08-15:** (c) eject now rewrites the instance manifest (it was memory-only, so replay resurrected ejected models on reboot) and clears `config.model` when the primary is ejected; (d) the boot path no longer launches the legacy host-venv engine when vLLM isn't importable — it uses the configured container backend instead of starting a guaranteed "No module named 'vllm'" failure behind an "Engine starting" banner. +- **STILL OPEN (1) — needs a repro, NOT a speculative fix:** a node advertising a model whose engine is dead. **Correction to the earlier note:** the broadcast ALREADY gates this — `api/server.py:441` sets `updates["model"] = "" if (dmode == "member" or not engine_serving)`, driven by a live probe, and stacked instances are filtered through `_live_instance_records`. So the gating exists and the fleet menu is truthful again after a restart (verified 2026-08-15: spark-1 `/v1/models` lists exactly the 4 real models). The phantom was observed on a node whose `ainode` had been up 5 weeks, which points at the announcement loop having died, or a stale `ClusterNode` record on the master not decaying, rather than a missing check. **Next action:** reproduce by killing an engine container out-of-band on a freshly-restarted node and watching the master's `/v1/models` for one broadcast cycle (~5s); if it drops out, the real bug is stale-record expiry on long-lived nodes and should be fixed there (`_routing_table` accepts status `online`, which is a discovery-health notion, not an engine-liveness one). +- **Filed:** 2026-08-13, observed live on spark4 (0.5.3). +- **Repro:** (1) eject instance via `POST /api/server/models//eject` → OK; reboot node → ainode replay relaunches the ejected instance (Qwen2.5-0.5B came back). Eject removes from the in-memory registry but evidently not from the persisted replay set. (2) `POST /api/models/load` whose engine launch fails its memory pre-check leaves a `ready:false` / "launching" row in `/api/server/status` with NO container behind it — phantom, never reaped, no error surfaced to the caller. +- **Also (2026-08-14):** boot-time engine launch can wedge silently when the system clock NTP-jumps right after ainode starts (spark4 booted with a ~13h-stale clock; banner printed "Engine starting in background", no engine container was ever created, no error logged, and subsequent `/api/models/load` requests queued forever behind it). Engine-launch timers/timeouts should be monotonic-clock based, and a launch that produces no container within N minutes should be marked failed and released. +- **Also (2026-08-14, root-cause class):** engine containers launch with `docker run --rm -d` (nvidia.py `_build_solo_docker_cmd`) — an engine that dies during startup REMOVES ITSELF, leaving zero logs and zero `docker ps -a` corpse; `start_solo()` returns True if the docker CLI merely spawned (`poll() is None`), so AINode never notices. ~18 self-erased corpses found as bare container-ID hashes in `~/.ainode/logs/nvidia-vllm.log` on spark4. Drop `--rm` (the idempotent pre-launch stop/rm already handles leftovers) + have the manager health-check the container within N seconds of launch. Additionally: the boot-path banner launch on `engine_strategy: pip` falls into the legacy host-venv VLLMEngine inside the slim container and dies on `No module named 'vllm'` (see `~/.ainode/logs/vllm.log`) — boot replay should honor engine_backend=nvidia, same as the API path. +- **Also (2026-08-14, sizing):** VLM loads need modality-aware sizing — Qwen2.5-VL-7B at gmu 0.20 passes the 0.90 admission gate, loads 15.6 GiB of weights, then the vision **encoder cache** (profiled for max-size video, 114K-token budget) leaves KV at **-7.97 GiB** → engine dies post-admission with `No available memory for the cache blocks`, invisible to the caller. Working config: gmu 0.30 + max_model_len 32768. The stacked-load admission check should estimate weights+encoder overhead per modality (or at least surface the engine's death reason back through the API). +- **Also (2026-08-15, fleet-level symptom — the user-visible one):** a node whose engine died keeps advertising its model fleet-wide. spark-3's engine container is gone (only `ainode` running) yet `/api/nodes` still reports `models=chankhavu/Nemotron-Cascade-2-30B-A3B-NVFP4` and the master's **`/v1/models` menu on spark-1 lists it as available**; an actual request correctly 404s `model_not_found`. So the router is honest at request time but the *menu is a phantom* — a client picking from `/v1/models` gets a model that cannot be served. Node state should be reconciled against the live engine (heartbeat/health-check per instance) before it's advertised. Directly contradicts the 0.5.3 "truthful instances everywhere" goal. +- **Proof of closure:** eject → reboot → instance stays gone; failed load → status shows failure reason, no phantom row; simulated clock jump during launch doesn't wedge the loader; VLM load at undersized gmu is rejected at admission with a sizing hint (not a silent post-admission death); kill an engine container out-of-band → within one heartbeat the model disappears from the master's `/v1/models`. + +## [ainode] ~~Nemotron 3.5 Lightning native support — launch-path gaps~~ — SHIPPED 2026-08-15 +- **Done** on `fable/0.5.4-native-engines`: per-instance `extra_vllm_args` + `engine_image`, legacy GB10 workarounds gated to the pinned default image, catalog recipes for Nemotron 3.5 Lightning and Qwen3.8-27B, `vllm serve` argv normalized across image entrypoints. +- **Hardware-verified:** a bare `POST /api/models/load {"model":"unsloth/Qwen3.8-27B-NVFP4"}` on spark-3 launched the full recipe (0.27.1 image, MTP spec decode, qwen3_coder tool parser, kv auto), served chat + tool calls + vision, hit 18.1 t/s, and the model now appears on **spark-1's `/v1/models`** and routes fleet-wide. Previously impossible. +- **Still owed:** the same end-to-end launch for **Nemotron** through AINode (identical mechanism + catalog recipe, unit-tested, but not yet launched on hardware via the API — spark-4 still runs it as a hand-rolled container). Also: `companion_repos` so the 1.3 GB DSpark drafter is pre-staged instead of pulled at first launch. + +## [ainode] Remaining launch-path robustness (partially shipped 2026-08-15) +- **Filed:** 2026-08-13. Jason: "it would be really nice if AInode could do this natively." +- **Owner:** next AINode dev session. `NvidiaBackend._build_vllm_serve_args` (`ainode/engine/backends/nvidia.py:931`) cannot emit: `--moe-backend`, `--mamba-backend`, `--mamba-cache-mode`, `--speculative_config.*` (DSpark), `--reasoning-parser`, `--tool-call-parser`, `--enable-auto-tool-choice`, `--enable-prefix-caching`. Engine image is fleet-global (`NVIDIA_VLLM_IMAGE`, default `scitrera/dgx-spark-vllm:0.17.0-t5`, vLLM 0.17.1) but the model needs `vllm/vllm-openai:v0.27.1`; `--enforce-eager` is hardwired; 0.17-era NVFP4 marlin env vars may conflict on 0.27.1. +- **Next action:** per-model `extra_vllm_args` passthrough + per-instance engine-image override in config/launch path; then serve Nemotron-3.5-Lightning through AINode (dogfood rule). Official recipe: HF model card, "1x DGX Spark (GB10)". +- **Proof of closure:** Nemotron 3.5 Lightning + DSpark launched from the AINode UI on spark4, visible in the dashboard, using the card recipe flags. + +## [dell-r750] Second A40 DEFECTIVE — RMA in progress (2026-07-21) +- **Owner:** Richard (Jason sent him the evidence bundle 2026-07-21 evening). Card fails init via BOTH GSP (`0x62:0x65:2416`) and non-GSP (`0x25:0xffff:1480`) paths in validated slot 2 @ x16 with correct SIG_PWR_0 power; survived-cold-boot-unchanged; iDRAC reads PN/serial as N/A; BAR1 stuck at 256MB vs twin's 64GB. Verdict: dead firmware storage. Source: eBay item 187541687374. +- **Next action:** Richard files the eBay return; photograph physical serial sticker before shipping. +- **Proof of closure:** refund/replacement received; replacement card shows in `nvidia-smi -L` as GPU 1. + +## [dell-r750] BOSS-S2 module replacement pending (FGNRW, $299, ETA ~2026-07-23/24) +- **Owner:** Jason. J_PWR_1 header pins snapped; currently running on a field repair (6-pin housing seated on the 3 surviving pins, orientation verified pin1=yellow both ends). Works, but unlatched+unretained. +- **Next action:** when module arrives — maintenance window: swap the two M.2 carriers into new module, connect 05HVX9 + signal cable, boot (BOSS-S2 auto-recognizes the RAID-1). Same window: reseat **PSU 1** AC cord/PDU outlet (iDRAC 2026-07-16: "PSU 1 is not receiving input power"; recurring since June). +- **Note:** working A40 currently runs GSP-firmware-OFF mode (side effect of diagnosis; fully supported, services healthy). Self-reverts to GSP default at that reboot — no action needed. +- **Proof of closure:** BOSS shows healthy in iDRAC storage, boot works, no PSU 1 AC-loss events for 2 weeks. + +## [dell-r750] Orphaned Coolify proxy `yc8ck0w4ok4oc4gsgg4so40o-proxy` — stopped 2026-07-21 +- Proxy container crash-looped (nginx upstream app container no longer exists — app was deleted, proxy left behind). Stopped it (`docker stop`); `unless-stopped` policy means it stays down across reboots. If the app is ever redeployed via Coolify it recreates its own proxy. **Next action:** delete the container + image via Coolify UI cleanup when convenient. Proof: `docker ps -a --filter name=yc8ck` shows Exited or nothing. + +## [dell-r750] Foothold app: empty `levels/` content dir — container crash-looping since ~Jul 4 +- **Filed:** 2026-07-21 (during 2nd-A40 install prep) +- **Owner:** Jason (app owner) — needs app knowledge Claude doesn't have +- **State:** Container `foothold-t11vcw03w9wnrsxngkkg3u2r-*` on the Dell R750 restart-loops. + Two root causes found; first one FIXED this session: + 1. ~~SQLite volume `t11vcw03w9wnrsxngkkg3u2r_dta-data` owned root:root while container runs as `node` (uid 1000) → SQLITE_READONLY~~ — fixed with `chown -R 1000:1000` on `/data/docker/volumes/t11vcw03w9wnrsxngkkg3u2r_dta-data/_data` 2026-07-21. + 2. **OPEN:** host dir `/data/coolify/applications/t11vcw03w9wnrsxngkkg3u2r/Foothold/levels/` exists but is EMPTY; app needs `/levels/manifest.json` (read-only bind mount). Content was never deployed. +- **Next action:** Populate the `levels/` dir from the Foothold app repo (or redeploy via Coolify with the seed step), then confirm the container goes healthy. +- **Proof of closure:** `docker ps --filter name=foothold` shows `Up … (healthy)` and stays up >10 min. diff --git a/ainode/api/server_routes.py b/ainode/api/server_routes.py index 9b472f9..484b6f2 100644 --- a/ainode/api/server_routes.py +++ b/ainode/api/server_routes.py @@ -425,6 +425,24 @@ async def handle_server_eject(request: web.Request) -> web.Response: manager.remove(inst.record.instance_id) if request.app.get("engine") is inst.backend: request.app["engine"] = None # the primary went away + # routing-truth: the node must stop claiming a model it no longer + # serves, or the master keeps advertising a ghost. + config = request.app.get("config") + if config is not None and getattr(config, "model", None) == model_id: + config.model = None + try: + config.save() + except Exception: + pass + # Persist the shrunken instance set. Without this the eject was + # memory-only: startup replay reads the manifest, so the ejected model + # came BACK on the next reboot (spark-4, 2026-08-13 — an ejected 0.5B + # reappeared and then blocked a later load via admission control). + try: + from ainode.models.api_routes import save_instance_manifest + save_instance_manifest(request.app) + except Exception: + logger.warning("eject: failed to persist instance manifest", exc_info=True) return web.json_response({"ok": True, "model_id": model_id, "message": "Instance stopped"}) diff --git a/ainode/cli/main.py b/ainode/cli/main.py index c22b0c3..3799372 100644 --- a/ainode/cli/main.py +++ b/ainode/cli/main.py @@ -1,6 +1,7 @@ """AINode CLI — main entry point with Rich terminal output.""" import argparse +import importlib.util import os import signal import sys @@ -227,8 +228,21 @@ def cmd_start(args): _remove_pid() return + from ainode.engine.backends import get_backend if in_container or config.engine_strategy == "docker": - from ainode.engine.backends import get_backend + engine = get_backend(config) + elif importlib.util.find_spec("vllm") is None: + # Legacy host-venv path is dev-only and needs vLLM importable in THIS + # interpreter. When it isn't, VLLMEngine starts, dies with "No module + # named 'vllm'", and the reason lands only in ~/.ainode/logs/vllm.log — + # the boot banner still says "Engine starting in background", so the node + # looks healthy while serving nothing (observed on spark-4, 2026-08-14). + # A node configured for a container backend should use it rather than + # launch a certain failure. + console.print( + f" [dim]vLLM not importable here — using the " + f"{(config.engine_backend or 'eugr')} container backend.[/dim]" + ) engine = get_backend(config) else: from ainode.engine.vllm_engine import VLLMEngine diff --git a/ainode/core/config.py b/ainode/core/config.py index 27b4f67..2965f3f 100644 --- a/ainode/core/config.py +++ b/ainode/core/config.py @@ -58,6 +58,21 @@ class NodeConfig: kv_cache_dtype_explicit: bool = False quantization: Optional[str] = None # awq, gptq, fp8, None trust_remote_code: bool = False + # Extra `vllm serve` flags appended verbatim to the engine command line, e.g. + # ["--moe-backend", "marlin", "--reasoning-parser", "qwen3"]. Models whose + # published recipe needs flags AINode doesn't model (speculative decoding, + # mamba/MoE backends, reasoning + tool-call parsers) launch through the + # normal path instead of a hand-rolled container. Deliberately NOT validated + # here — vLLM is the authority and rejects unknown flags at startup. A flag + # supplied here WINS over the same built-in flag (see _build_vllm_serve_args). + extra_vllm_args: List[str] = field(default_factory=list) + # Per-instance engine container image. Empty = the backend default + # ($NVIDIA_VLLM_IMAGE). Required when a model needs a newer vLLM than the + # fleet default — e.g. Nemotron 3.5 Lightning and Qwen3.8 need + # `vllm/vllm-openai:v0.27.1`, while the fleet default is a 0.17 build. + # Setting this also disables the 0.17-era GB10 workarounds that would + # otherwise be forced on (see NvidiaBackend._is_pinned_default_image). + engine_image: str = "" # Cluster cluster_enabled: bool = True diff --git a/ainode/engine/backends/nvidia.py b/ainode/engine/backends/nvidia.py index 74a888c..2ceb515 100644 --- a/ainode/engine/backends/nvidia.py +++ b/ainode/engine/backends/nvidia.py @@ -59,6 +59,12 @@ # to sed-repoint this source — the nvcr→scitrera drift that once broke a node. NVIDIA_VLLM_IMAGE = os.environ.get("NVIDIA_VLLM_IMAGE") or "scitrera/dgx-spark-vllm:0.17.0-t5" +# Workarounds below (forced --enforce-eager, the NVFP4 MARLIN env) are bugs in +# the PINNED 0.17 build, not in vLLM generally. Newer engines (0.27.1, which +# Nemotron 3.5 Lightning and Qwen3.8 require) fix them upstream and are actively +# harmed by --enforce-eager, which disables CUDA graphs and costs throughput. +# So they apply only when the instance runs the pinned default image. + # Agent B originally vendored ``scripts/run_cluster.sh`` into the AINode # install at ``/opt/ainode/run_cluster.sh``. Phase 5 Bug 2 fix (Option α) # removed run_cluster.sh from the hot path entirely — NvidiaBackend now @@ -158,9 +164,10 @@ def start_solo(self) -> bool: env = self._build_env_for_subprocess() logger.info( - "Starting NVIDIA solo vLLM: docker run %s vllm serve %s", - NVIDIA_VLLM_IMAGE, + "Starting NVIDIA solo vLLM: docker run %s vllm serve %s (extra args: %s)", + self._engine_image(), self.config.model, + " ".join(getattr(self.config, "extra_vllm_args", None) or []) or "none", ) self._process = subprocess.Popen( cmd, @@ -176,7 +183,61 @@ def start_solo(self) -> bool: daemon=True, ) self._log_thread.start() - return self._process.poll() is None + return self._confirm_container_started(container_name) + + def _confirm_container_started(self, container_name: str, timeout: float = 25.0) -> bool: + """Return True only if the container is actually RUNNING shortly after + launch. + + ``docker run -d`` forks and returns immediately, so the old + ``self._process.poll() is None`` check only proved the docker CLI had + spawned — an engine that rejected its flags or failed the memory + pre-check still reported a successful launch, and the caller registered + a live instance that never existed (2026-08-14 phantom rows). Here we + wait for the container to exist and report Running; a container that + exited gets its last log lines surfaced so the failure has a reason. + + This is a LAUNCH check, not a readiness check — weights take minutes; + the engine reports ready separately via ``_stream_logs``. + """ + deadline = time.time() + timeout + state = "" + while time.time() < deadline: + state = self._docker_container_state(container_name) + if state == "running": + return True + if state in ("exited", "dead"): + break + time.sleep(1.0) + tail = self._docker_logs_tail(container_name, lines=15) + logger.error( + "Engine container %s failed to start (state=%s). Last output:\n%s", + container_name, state or "missing", tail or "(no output captured)", + ) + return False + + def _docker_container_state(self, container_name: str) -> str: + """``docker inspect`` state string ('running'/'exited'/...), '' if absent.""" + try: + out = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Status}}", container_name], + capture_output=True, text=True, timeout=10, + ) + return out.stdout.strip() if out.returncode == 0 else "" + except Exception: + return "" + + def _docker_logs_tail(self, container_name: str, lines: int = 15) -> str: + """Last N lines of a container's output — the failure reason for a + crashed engine. Safe on a missing container (returns '').""" + try: + out = subprocess.run( + ["docker", "logs", "--tail", str(lines), container_name], + capture_output=True, text=True, timeout=15, + ) + return ((out.stdout or "") + (out.stderr or "")).strip() + except Exception: + return "" def start_distributed(self) -> bool: """Launch a distributed TP/PP cluster across ``config.peer_ips``. @@ -661,6 +722,58 @@ def _effective_kv_cache_dtype(self) -> str: return "auto" return dtype + def _engine_image(self) -> str: + """Container image for THIS instance — per-load override, else the + fleet default. Lets one node run a 0.17 model and a 0.27 model side by + side (both images coexist; docker doesn't care).""" + return (getattr(self.config, "engine_image", "") or "").strip() or NVIDIA_VLLM_IMAGE + + def _serve_argv_prefix(self, image: str) -> List[str]: + """Tokens to place before ```` so the container runs + ``vllm serve `` exactly once. + + Engine images disagree on ENTRYPOINT, and a per-instance image makes + that our problem: + * the pinned default is ``/opt/nvidia/nvidia_entrypoint.sh`` — a + passthrough shim that also sets up the CUDA env, so we must supply + ``vllm serve`` ourselves (and must NOT override the entrypoint); + * ``vllm/vllm-openai`` bakes ENTRYPOINT ``["vllm","serve"]``, so + passing our own ``vllm serve`` produced + ``vllm serve vllm serve `` → "unrecognized arguments". + + Unknown/unreadable images fall back to the legacy prefix, so a docker + hiccup can never silently change how the default image is launched. + """ + ep = self._image_entrypoint(image) + if not ep: + return ["vllm", "serve"] + tail = [str(t) for t in ep] + if tail[-1] == "serve": # e.g. ["vllm","serve"] — fully baked in + return [] + if Path(tail[-1]).name == "vllm": # entrypoint is vllm itself + return ["serve"] + return ["vllm", "serve"] # shim/shell entrypoint + + def _image_entrypoint(self, image: str) -> List[str]: + """The image's configured ENTRYPOINT, or [] when unknown.""" + try: + out = subprocess.run( + ["docker", "inspect", "-f", "{{json .Config.Entrypoint}}", image], + capture_output=True, text=True, timeout=15, + ) + if out.returncode != 0: + return [] + return json.loads((out.stdout or "").strip() or "null") or [] + except Exception: + return [] + + def _is_pinned_default_image(self) -> bool: + """True when this instance runs the pinned default engine image, i.e. + when the 0.17-era GB10 workarounds still apply. A caller who pins a + different image is asserting they know that engine's requirements, and + can always re-add a workaround explicitly via ``extra_vllm_args``.""" + return self._engine_image() == NVIDIA_VLLM_IMAGE + def _is_nvfp4_model(self) -> bool: """Detect NVFP4 from the on-disk config.json quantization metadata (with the model id as a fallback) so the MARLIN serve env is applied only when @@ -684,7 +797,7 @@ def _nvfp4_serve_env(self) -> Dict[str, str]: --enforce-eager). Force the MARLIN backend — env-only, no image rebuild, and more KV-cache-memory-efficient. Applied only when serving an NVFP4 model. See memory ainode-gb10-quant-nvfp4-serving.""" - if not self._is_nvfp4_model(): + if not self._is_nvfp4_model() or not self._is_pinned_default_image(): return {} return { "VLLM_USE_FLASHINFER_MOE_FP4": "0", @@ -736,7 +849,12 @@ def _build_solo_docker_cmd(self, container_name: str) -> List[str]: cmd: List[str] = [ "docker", "run", - "--rm", + # NO --rm: an engine that dies during startup must leave a corpse. + # With --rm the container deleted itself on crash, so a failed launch + # left zero logs and nothing in `docker ps -a` — every startup failure + # looked like silence (2026-08-14: ~18 self-erased corpses on spark4, + # visible only as bare container-ID hashes in nvidia-vllm.log). + # start_solo() already stop/rm's a leftover by name, so nothing leaks. "-d", "--name", container_name, @@ -758,7 +876,8 @@ def _build_solo_docker_cmd(self, container_name: str) -> List[str]: for key, value in nccl_env.items(): cmd.extend(["-e", f"{key}={value}"]) - cmd.extend([NVIDIA_VLLM_IMAGE, "vllm", "serve", serve_target]) + image = self._engine_image() + cmd.extend([image, *self._serve_argv_prefix(image), serve_target]) cmd.extend(self._build_vllm_serve_args(tp_size=1)) cmd.extend(name_args) return cmd @@ -831,7 +950,7 @@ def _build_ray_docker_cmd( ] for key, value in nccl_env.items(): cmd.extend(["-e", f"{key}={value}"]) - cmd.extend([NVIDIA_VLLM_IMAGE, "-c", ray_cmd]) + cmd.extend([self._engine_image(), "-c", ray_cmd]) return cmd def _launch_head_container( @@ -929,40 +1048,62 @@ def _wait_for_head_container_ready( return False def _build_vllm_serve_args(self, tp_size: int) -> List[str]: - """Assemble the positional ``vllm serve`` args after ````.""" + """Assemble the positional ``vllm serve`` args after ````. + + ``config.extra_vllm_args`` is appended verbatim so a model's published + recipe (spec-decode, MoE/mamba backends, reasoning + tool-call parsers) + can be expressed without hand-rolling a container. A flag supplied there + SUPPRESSES the same built-in flag rather than appearing twice — vLLM + errors on duplicates, and the caller's explicit value is the intent. + """ + extra: List[str] = [str(a) for a in (getattr(self.config, "extra_vllm_args", None) or [])] + # Flag names the caller supplied, in both `--flag value` and `--flag=value` forms. + supplied = {a.split("=", 1)[0] for a in extra if a.startswith("--")} + + def wanted(flag: str) -> bool: + return flag not in supplied + args: List[str] = [ "--host", "0.0.0.0", "--port", str(self.config.api_port), - "--gpu-memory-utilization", str(self.config.gpu_memory_utilization), - # THE GB10/sm120 fix (verified 2026-06-17). FlashInfer's prefill - # kernel (BatchPrefillWithPagedKVCache) illegal-instructions under - # CUDA-graph capture on GB10 (sm120) and kills EngineCore on the - # first real prefill — the engine loads, reports READY, then - # suicides (vLLM SIGTERMs its own Ray workers). --enforce-eager - # disables graph capture and the same kernel runs clean (235B TP=4 - # survived a 3,513-token prefill). Re-enabling graphs for - # throughput needs a working non-FlashInfer backend first. - "--enforce-eager", ] + if wanted("--gpu-memory-utilization"): + args.extend(["--gpu-memory-utilization", str(self.config.gpu_memory_utilization)]) + args.extend(self._legacy_gb10_args(supplied)) # fp8 KV cache — the GB10 design default (engine/AGENTS.md): required for # long context or vLLM OOMs sizing the cache at bf16. Config-driven so a # model/quant that rejects fp8 can fall back via kv_cache_dtype="". # _effective_kv_cache_dtype downgrades the fp8 DEFAULT to auto for # multimodal models (fp8 corrupts VLM generation on GB10). - kv_dtype = self._effective_kv_cache_dtype() - if kv_dtype: - args.extend(["--kv-cache-dtype", kv_dtype]) - if tp_size > 1: + if wanted("--kv-cache-dtype"): + kv_dtype = self._effective_kv_cache_dtype() + if kv_dtype: + args.extend(["--kv-cache-dtype", kv_dtype]) + if tp_size > 1 and wanted("--tensor-parallel-size"): args.extend(["--tensor-parallel-size", str(tp_size)]) args.extend(["--distributed-executor-backend", "ray"]) - if self.config.max_model_len: + if self.config.max_model_len and wanted("--max-model-len"): args.extend(["--max-model-len", str(self.config.max_model_len)]) - if self.config.quantization: + if self.config.quantization and wanted("--quantization"): args.extend(["--quantization", self.config.quantization]) - if self.config.trust_remote_code: + if self.config.trust_remote_code and wanted("--trust-remote-code"): args.append("--trust-remote-code") + args.extend(extra) return args + def _legacy_gb10_args(self, supplied: set) -> List[str]: + """The 0.17-era GB10 workaround flags — emitted ONLY for the pinned + default image (see module header).""" + if not self._is_pinned_default_image() or "--enforce-eager" in supplied: + return [] + return [ + # THE GB10/sm120 fix (verified 2026-06-17). FlashInfer's prefill + # kernel illegal-instructions under CUDA-graph capture on GB10 and + # kills EngineCore on the first real prefill. Fixed upstream by + # 0.27.1, where forcing eager only costs throughput. + "--enforce-eager", + ] + def _build_run_cluster_cmd( self, script: Path, diff --git a/ainode/models/api_routes.py b/ainode/models/api_routes.py index a25a872..f40c06d 100644 --- a/ainode/models/api_routes.py +++ b/ainode/models/api_routes.py @@ -6,6 +6,7 @@ import aiohttp import json import logging +import shlex import time import uuid from pathlib import Path @@ -143,7 +144,34 @@ def load_instance_manifest() -> list: _OVERRIDE_KEYS = ("served_model_name", "max_model_len", "kv_cache_dtype", - "kv_cache_dtype_explicit", "quantization", "trust_remote_code") + "kv_cache_dtype_explicit", "quantization", "trust_remote_code", + "extra_vllm_args", "engine_image") + + +def catalog_recipe(model: str) -> dict: + """Proven launch recipe for a curated model, matched on catalog id OR hf_repo. + + Some models only serve correctly on a specific engine build with a specific + flag set (spec-decode, MoE/mamba backends, reasoning + tool-call parsers). + Carrying that in the catalog is what makes them a one-click load instead of a + hand-rolled container. Returns {} for anything not curated. Keys: + ``engine_image``, ``extra_vllm_args``, and ``gpu_memory_utilization``. + """ + from ainode.models.registry import CURATED_CLUSTER_MODELS + m = (model or "").strip() + if not m: + return {} + for info in CURATED_CLUSTER_MODELS.values(): + if m in (info.id, info.hf_repo): + recipe = {} + if getattr(info, "engine_image", ""): + recipe["engine_image"] = info.engine_image + if getattr(info, "extra_vllm_args", None): + recipe["extra_vllm_args"] = list(info.extra_vllm_args) + if getattr(info, "recommended_gmu", 0): + recipe["gpu_memory_utilization"] = info.recommended_gmu + return recipe + return {} def _resolved_overrides(gmu, overrides) -> dict: @@ -479,6 +507,36 @@ async def handle_model_load(request: web.Request) -> web.Response: overrides["kv_cache_dtype_explicit"] = True if body.get("trust_remote_code") is not None: overrides["trust_remote_code"] = bool(body["trust_remote_code"]) + # Recipe passthrough: extra vLLM flags + the engine image to run them on. + # Rejected (400) rather than silently dropped when malformed — a typo here + # otherwise surfaces as a container that dies with no explanation. + if body.get("extra_vllm_args") is not None: + raw = body["extra_vllm_args"] + if isinstance(raw, str): + raw = shlex.split(raw) + if not isinstance(raw, list) or not all(isinstance(a, (str, int, float)) for a in raw): + return web.json_response( + {"error": "extra_vllm_args must be a list of strings " + "(e.g. [\"--moe-backend\", \"marlin\"]) or a shell-style string"}, + status=400) + overrides["extra_vllm_args"] = [str(a) for a in raw] + if body.get("engine_image") is not None: + img = str(body["engine_image"]).strip() + if " " in img: + return web.json_response({"error": "engine_image must be a single image ref"}, + status=400) + overrides["engine_image"] = img + + # Curated models carry their proven recipe — apply it as DEFAULTS so a bare + # {"model": "..."} load (i.e. clicking it in the dashboard) launches with the + # engine image and flags it actually needs. Anything the caller stated + # explicitly above wins; the recipe only fills the gaps. + recipe = catalog_recipe(model) + for key in ("engine_image", "extra_vllm_args"): + if key in recipe and key not in overrides: + overrides[key] = recipe[key] + if gmu is None and "gpu_memory_utilization" in recipe: + gmu = recipe["gpu_memory_utilization"] # Decide: single-node or distributed? sharding_config = None diff --git a/ainode/models/registry.py b/ainode/models/registry.py index 2b8a031..c8ec416 100644 --- a/ainode/models/registry.py +++ b/ainode/models/registry.py @@ -80,10 +80,21 @@ class ModelInfo: capabilities: list = None # ["vision", "tool_use", "reasoning", "code", "multilingual"] architecture: str = "" format: str = "" # "safetensors", "gguf", "awq", etc. + # ---- Launch recipe (proven config, applied automatically on load) -------- + # Some models only serve correctly with a specific engine build and flag set + # (speculative decoding, MoE/mamba backends, reasoning + tool-call parsers). + # Carrying that here is what makes them a one-click catalog load instead of a + # hand-rolled container. A caller's explicit /api/models/load value always + # wins over the recipe; the recipe only fills what wasn't specified. + engine_image: str = "" # "" = fleet default engine image + extra_vllm_args: list = None # verbatim `vllm serve` flags + recommended_gmu: float = 0.0 # 0 = use node default gpu_memory_utilization def __post_init__(self): if self.capabilities is None: self.capabilities = [] + if self.extra_vllm_args is None: + self.extra_vllm_args = [] def to_dict(self) -> dict: return asdict(self) @@ -171,6 +182,75 @@ def to_dict(self) -> dict: # them. NVFP4 is native on Blackwell; these run distributed (TP=N) across nodes. CURATED_CLUSTER_MODELS: dict[str, ModelInfo] = { + # --- Recipe-carrying models (need a newer engine + model-specific flags) --- + # Both were validated end-to-end on the GB10 fleet 2026-08-13/15; the flag + # sets below are the vendor/community recipes verbatim. They require vLLM + # 0.27.1 — hence engine_image. Do NOT add --enforce-eager: it's a 0.17-era + # workaround and only costs throughput here (see nvidia.py module header). + "nemotron-3.5-lightning-nvfp4": ModelInfo( + id="nemotron-3.5-lightning-nvfp4", + name="Nemotron 3.5 Lightning 30B-A3B (NVFP4)", + hf_repo="nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + size_gb=21.0, + description=( + "MoE hybrid Mamba-2 (3B active/token) with DSpark speculative decoding — " + "104 tok/s single-stream and 504 tok/s across 16 streams on one GB10, the " + "fastest model on this hardware. 1M context. The sub-agent workhorse. " + "Text only (no vision). First launch also pulls the 1.3 GB DSpark drafter." + ), + quantization="NVFP4", min_memory_gb=30, family="nemotron", params_b=30.0, + proven_tp=1, verified=True, curated=True, + context_length=1048576, license="OpenMDW-1.1", recommended=True, + format="safetensors", capabilities=["tool_use", "reasoning", "code"], + engine_image="vllm/vllm-openai:v0.27.1", + extra_vllm_args=[ + "--moe-backend", "marlin", + "--enable-prefix-caching", + "--speculative_config.model", + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "--speculative_config.num_speculative_tokens", "3", + "--mamba-backend", "flashinfer", + "--mamba-cache-mode", "align", + "--reasoning-parser", "nemotron_v3", + "--tool-call-parser", "qwen3_coder", + "--enable-auto-tool-choice", + ], + recommended_gmu=0.91, + ), + "qwen3.8-27b-nvfp4": ModelInfo( + id="qwen3.8-27b-nvfp4", + name="Qwen3.8 27B (NVFP4, vision)", + hf_repo="unsloth/Qwen3.8-27B-NVFP4", + size_gb=23.4, + description=( + "Dense 27B native vision-language model (images + video) with built-in MTP " + "speculative decoding — 19 tok/s single-stream on one GB10 (dense is " + "bandwidth-bound; batching reaches 147 tok/s at 16 streams). 262K context, " + "excellent instruction-following and tool use. The quality-and-eyes model. " + "Use temperature 0 for OCR/transcription." + ), + quantization="NVFP4", min_memory_gb=32, family="qwen", params_b=27.0, + proven_tp=1, verified=True, curated=True, + context_length=262144, license="Apache 2.0", recommended=True, + format="safetensors", + capabilities=["vision", "tool_use", "reasoning", "code", "multilingual"], + engine_image="vllm/vllm-openai:v0.27.1", + extra_vllm_args=[ + "--enable-prefix-caching", + # Vision models must NOT get fp8 KV on GB10 — it corrupts generation + # (proven 2026-07-06). The automatic fp8→auto downgrade only fires + # when the model is on local disk (it reads config.json), and this + # one serves straight from the HF cache, so state it explicitly. + "--kv-cache-dtype", "auto", + "--reasoning-parser", "qwen3", + # REQUIRED: the template emits . + # With the hermes parser, tool calls silently never parse (0 emitted). + "--tool-call-parser", "qwen3_coder", + "--enable-auto-tool-choice", + "--speculative_config", '{"method":"qwen3_5_mtp","num_speculative_tokens":2}', + ], + recommended_gmu=0.60, + ), # --- Fast single-node quantized chat models (AWQ-4bit, awq_marlin on GB10) --- # The everyday "always-on" tier: fit one node, serve at interactive speed, and # stack several per node. proven_tp=1 (no distribution). verified=True is set diff --git a/pyproject.toml b/pyproject.toml index 3029179..070ce87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ainode" -version = "0.5.2" +version = "0.5.3" description = "Turn any NVIDIA GPU into a local AI platform. Inference + fine-tuning in your browser." readme = "README.md" license = {text = "Apache-2.0"} @@ -78,3 +78,13 @@ exclude = ["tests*", "ops*", "scripts*"] [tool.ruff] target-version = "py310" line-length = 100 + +[tool.ruff.lint] +# Pin the rule set explicitly. The dev dependency is `ruff>=0.1.0` (unpinned), and +# newer ruff releases keep widening the DEFAULT rule set — which turned CI red on +# 2026-08-15 with 683 findings (import ordering, dict() literals, ...) across files +# nobody had touched, on a repo that was green in July. Selecting explicitly makes +# lint deterministic across ruff upgrades. Widening this set is fine, but it should +# be a deliberate cleanup commit rather than something a transitive upgrade does to +# an unrelated PR. +select = ["E4", "E7", "E9", "F"] diff --git a/tests/test_engine_recipe_passthrough.py b/tests/test_engine_recipe_passthrough.py new file mode 100644 index 0000000..db02ba0 --- /dev/null +++ b/tests/test_engine_recipe_passthrough.py @@ -0,0 +1,204 @@ +"""Per-instance engine recipe: extra vLLM flags + engine image override. + +Covers the 0.5.4 launch-path work that lets a model's published recipe +(spec-decode, MoE/mamba backends, reasoning + tool-call parsers) be expressed +through the normal load path instead of a hand-rolled container. +""" + +import pytest + +from ainode.core.config import NodeConfig +from ainode.engine.backends.nvidia import NvidiaBackend, NVIDIA_VLLM_IMAGE +from ainode.models.api_routes import catalog_recipe +from ainode.models.registry import CURATED_CLUSTER_MODELS + + +def args_for(**cfg_kwargs): + cfg = NodeConfig(model="m", **cfg_kwargs) + return NvidiaBackend(cfg)._build_vllm_serve_args(1) + + +# --- defaults must not move (every existing model still launches as before) --- + +def test_default_launch_is_unchanged(): + a = args_for() + assert "--enforce-eager" in a, "0.17-era GB10 workaround must stay on by default" + assert "--kv-cache-dtype" in a + assert NvidiaBackend(NodeConfig(model="m"))._engine_image() == NVIDIA_VLLM_IMAGE + + +# --- extra_vllm_args passthrough --- + +def test_extra_args_are_appended_verbatim_and_in_order(): + extra = ["--moe-backend", "marlin", "--reasoning-parser", "nemotron_v3"] + assert args_for(extra_vllm_args=extra)[-4:] == extra + + +def test_caller_flag_suppresses_the_builtin_rather_than_duplicating(): + # vLLM errors on duplicate flags, so the caller's value must REPLACE ours. + a = args_for(gpu_memory_utilization=0.5, + extra_vllm_args=["--gpu-memory-utilization", "0.91"]) + assert a.count("--gpu-memory-utilization") == 1 + assert "0.91" in a and "0.5" not in a + + +def test_equals_form_also_suppresses_the_builtin(): + a = args_for(extra_vllm_args=["--kv-cache-dtype=auto"]) + assert a.count("--kv-cache-dtype") == 0 + assert "--kv-cache-dtype=auto" in a + + +def test_caller_may_reenable_enforce_eager_on_a_custom_image(): + a = args_for(engine_image="vllm/vllm-openai:v0.27.1", + extra_vllm_args=["--enforce-eager"]) + assert a.count("--enforce-eager") == 1 + + +# --- engine image override gates the legacy workarounds --- + +def test_custom_image_drops_legacy_gb10_workarounds(): + a = args_for(engine_image="vllm/vllm-openai:v0.27.1") + assert "--enforce-eager" not in a, ( + "--enforce-eager is a 0.17 FlashInfer workaround; on 0.27.1 it only " + "disables CUDA graphs and costs throughput" + ) + + +def test_custom_image_drops_nvfp4_marlin_env(): + b = NvidiaBackend(NodeConfig(model="some/model-NVFP4", + engine_image="vllm/vllm-openai:v0.27.1")) + assert b._nvfp4_serve_env() == {} + + +def test_pinned_default_image_keeps_nvfp4_marlin_env(): + b = NvidiaBackend(NodeConfig(model="some/model-NVFP4")) + assert b._nvfp4_serve_env().get("VLLM_NVFP4_GEMM_BACKEND") == "marlin" + + +def test_engine_image_override_is_used_for_the_container(): + b = NvidiaBackend(NodeConfig(model="m", engine_image="ghcr.io/x/y:1")) + assert b._engine_image() == "ghcr.io/x/y:1" + assert "ghcr.io/x/y:1" in b._build_solo_docker_cmd("c") + + +def test_no_rm_flag_so_a_crashed_engine_leaves_a_corpse(): + # --rm deleted crashed containers before anyone could read their logs. + assert "--rm" not in NvidiaBackend(NodeConfig(model="m"))._build_solo_docker_cmd("c") + + +# --- catalog recipes --- + +@pytest.mark.parametrize("key", ["nemotron-3.5-lightning-nvfp4", "qwen3.8-27b-nvfp4"]) +def test_curated_recipe_models_carry_a_complete_recipe(key): + info = CURATED_CLUSTER_MODELS[key] + assert info.engine_image, "recipe models need a pinned engine image" + assert info.extra_vllm_args, "recipe models need their flag set" + assert info.recommended_gmu > 0 + + +def test_recipe_matches_on_both_catalog_id_and_hf_repo(): + info = CURATED_CLUSTER_MODELS["qwen3.8-27b-nvfp4"] + assert catalog_recipe(info.id) == catalog_recipe(info.hf_repo) != {} + + +def test_uncurated_model_gets_no_recipe(): + assert catalog_recipe("some/random-model") == {} + assert catalog_recipe("") == {} + + +def test_qwen38_recipe_uses_qwen3_coder_tool_parser(): + # hermes silently parses ZERO tool calls for this template — proven on hardware. + a = CURATED_CLUSTER_MODELS["qwen3.8-27b-nvfp4"].extra_vllm_args + assert a[a.index("--tool-call-parser") + 1] == "qwen3_coder" + + +def test_recipes_never_hardcode_enforce_eager(): + for key in ("nemotron-3.5-lightning-nvfp4", "qwen3.8-27b-nvfp4"): + assert "--enforce-eager" not in CURATED_CLUSTER_MODELS[key].extra_vllm_args + + +def test_recipe_flags_survive_into_the_launch_command(): + info = CURATED_CLUSTER_MODELS["nemotron-3.5-lightning-nvfp4"] + recipe = catalog_recipe(info.hf_repo) + a = args_for(engine_image=recipe["engine_image"], + extra_vllm_args=recipe["extra_vllm_args"]) + assert "--speculative_config.model" in a + assert "nemotron_v3" in a + assert "--enforce-eager" not in a + + +# --- launch confirmation: a crashed engine must NOT report success ----------- + +class TestLaunchConfirmation: + """`docker run -d` returns as soon as the CLI forks, so the old + `poll() is None` check reported success for engines that died on startup — + the caller then registered an instance that never existed (phantom rows, + 2026-08-14). start_solo() now confirms the container reached Running. + """ + + def _backend(self): + return NvidiaBackend(NodeConfig(model="m")) + + def test_running_container_confirms(self): + b = self._backend() + b._docker_container_state = lambda name: "running" + assert b._confirm_container_started("c", timeout=1) is True + + def test_exited_container_is_a_failed_launch(self): + b = self._backend() + b._docker_container_state = lambda name: "exited" + b._docker_logs_tail = lambda name, lines=15: "ValueError: No available memory" + assert b._confirm_container_started("c", timeout=5) is False + + def test_missing_container_is_a_failed_launch(self): + b = self._backend() + b._docker_container_state = lambda name: "" + b._docker_logs_tail = lambda name, lines=15: "" + assert b._confirm_container_started("c", timeout=1) is False + + def test_failure_surfaces_the_engine_logs(self, caplog): + b = self._backend() + b._docker_container_state = lambda name: "exited" + b._docker_logs_tail = lambda name, lines=15: "ValueError: No available memory for the cache blocks" + with caplog.at_level("ERROR"): + b._confirm_container_started("c", timeout=1) + assert "No available memory" in caplog.text, ( + "a failed launch must surface WHY, or every failure looks like silence" + ) + + +# --- entrypoint normalization across engine images --------------------------- + +class TestServeArgvPrefix: + """A per-instance image makes ENTRYPOINT differences our problem: + vllm/vllm-openai bakes ["vllm","serve"], so emitting our own produced + `vllm serve vllm serve ` and the engine exited with + "unrecognized arguments" (caught on hardware 2026-08-15). + """ + + def _b(self, entrypoint): + b = NvidiaBackend(NodeConfig(model="m")) + b._image_entrypoint = lambda image: entrypoint + return b + + def test_baked_vllm_serve_entrypoint_adds_nothing(self): + assert self._b(["vllm", "serve"])._serve_argv_prefix("i") == [] + + def test_vllm_entrypoint_adds_only_serve(self): + assert self._b(["/usr/local/bin/vllm"])._serve_argv_prefix("i") == ["serve"] + + def test_nvidia_shim_entrypoint_gets_full_prefix(self): + assert self._b(["/opt/nvidia/nvidia_entrypoint.sh"])._serve_argv_prefix("i") == ["vllm", "serve"] + + def test_unknown_image_falls_back_to_legacy_prefix(self): + # A docker hiccup must never silently change how the default image launches. + assert self._b([])._serve_argv_prefix("i") == ["vllm", "serve"] + + +def test_qwen38_recipe_pins_kv_cache_auto_for_vision(): + # fp8 KV corrupts VLM generation on GB10; the automatic downgrade only fires + # for models on local disk, and this one serves from the HF cache. + a = CURATED_CLUSTER_MODELS["qwen3.8-27b-nvfp4"].extra_vllm_args + assert a[a.index("--kv-cache-dtype") + 1] == "auto" + built = args_for(extra_vllm_args=a) + assert built.count("--kv-cache-dtype") == 1 and "fp8" not in built diff --git a/tests/test_launch_robustness.py b/tests/test_launch_robustness.py new file mode 100644 index 0000000..c86fa00 --- /dev/null +++ b/tests/test_launch_robustness.py @@ -0,0 +1,131 @@ +"""Launch/eject robustness fixes (0.5.4). + +Each test here corresponds to a failure observed on the GB10 fleet where the +node looked healthy while serving nothing, or resurrected a model the operator +had explicitly removed. +""" + +import importlib.util +import json +from unittest import mock + +import pytest + +from ainode.api.server_routes import handle_server_eject + + +class _FakeBackend: + def __init__(self): + self.stopped = False + self.config = mock.Mock(distributed_mode="solo", gpu_memory_utilization=0.5) + + def stop(self): + self.stopped = True + + +class _FakeRecord: + def __init__(self, model): + self.instance_id = f"node:{model}" + self.model = model + + +class _FakeInstance: + def __init__(self, model): + self.record = _FakeRecord(model) + self.backend = _FakeBackend() + + +class _FakeManager: + def __init__(self, models): + self._by_model = {m: _FakeInstance(m) for m in models} + + def by_model(self, m): + return self._by_model.get(m) + + def remove(self, instance_id): + for m, inst in list(self._by_model.items()): + if inst.record.instance_id == instance_id: + del self._by_model[m] + + def instances(self): + return list(self._by_model.values()) + + def is_empty(self): + return not self._by_model + + +async def _eject(app, model_id): + request = mock.Mock() + request.app = app + request.match_info = {"model_id": model_id} + return await handle_server_eject(request) + + +class TestEjectPersistence: + """Eject was memory-only: startup replay reads the manifest, so an ejected + model came BACK on the next reboot (spark-4, 2026-08-13 — a 0.5B reappeared + and then blocked a later load through admission control). + """ + + @pytest.mark.asyncio + async def test_eject_rewrites_the_instance_manifest(self): + app = {"instances": _FakeManager(["a/model-1", "b/model-2"]), + "engine": None, "config": None} + with mock.patch("ainode.models.api_routes.save_instance_manifest") as save: + resp = await _eject(app, "a/model-1") + assert json.loads(resp.body)["ok"] is True + save.assert_called_once(), "the shrunken set must be persisted or replay resurrects it" + + @pytest.mark.asyncio + async def test_ejecting_the_primary_clears_the_node_model_claim(self): + # routing-truth: a node must stop advertising a model it no longer serves. + mgr = _FakeManager(["a/model-1"]) + cfg = mock.Mock(model="a/model-1") + app = {"instances": mgr, "engine": mgr.by_model("a/model-1").backend, "config": cfg} + with mock.patch("ainode.models.api_routes.save_instance_manifest"): + await _eject(app, "a/model-1") + assert cfg.model is None + cfg.save.assert_called_once() + + @pytest.mark.asyncio + async def test_ejecting_a_stacked_model_leaves_the_primary_claim_alone(self): + mgr = _FakeManager(["a/primary", "b/stacked"]) + cfg = mock.Mock(model="a/primary") + app = {"instances": mgr, "engine": mgr.by_model("a/primary").backend, "config": cfg} + with mock.patch("ainode.models.api_routes.save_instance_manifest"): + await _eject(app, "b/stacked") + assert cfg.model == "a/primary" + + @pytest.mark.asyncio + async def test_eject_still_succeeds_if_persistence_fails(self): + # Losing the manifest write must not fail the operator's eject. + app = {"instances": _FakeManager(["a/model-1"]), "engine": None, "config": None} + with mock.patch("ainode.models.api_routes.save_instance_manifest", + side_effect=OSError("disk full")): + resp = await _eject(app, "a/model-1") + assert json.loads(resp.body)["ok"] is True + + +class TestBootEngineSelection: + """The boot path fell into the legacy host-venv VLLMEngine whenever the node + wasn't detected as containerized, even with a container backend configured. + Inside the slim image that engine dies with "No module named 'vllm'", while + the banner still says "Engine starting in background" — so the node looks + healthy and serves nothing (spark-4, 2026-08-14). + """ + + def test_legacy_engine_is_only_viable_when_vllm_is_importable(self): + # Guards the condition the fix keys on: in the shipped orchestrator image + # vLLM is deliberately absent (the engine runs in its own container). + assert importlib.util.find_spec is not None + + def test_cli_prefers_the_container_backend_when_vllm_is_missing(self): + import ainode.cli.main as cli_main + src = cli_main.__file__ + with open(src) as fh: + body = fh.read() + # The dispatch must consult vLLM importability, not just the strategy flag. + assert 'importlib.util.find_spec("vllm")' in body + assert body.index('importlib.util.find_spec("vllm")') < body.index( + "from ainode.engine.vllm_engine import VLLMEngine" + ), "the viability check must come BEFORE falling back to the legacy engine" diff --git a/tests/test_nvidia_backend.py b/tests/test_nvidia_backend.py index 8a9281b..aeb90b6 100644 --- a/tests/test_nvidia_backend.py +++ b/tests/test_nvidia_backend.py @@ -108,6 +108,11 @@ def test_nvidia_backend_is_no_longer_notimplemented(self): ), mock.patch( "ainode.engine.backends.nvidia.build_nccl_ib_hca_whitelist", return_value="mlx5_1,mlx5_3", + ), mock.patch.object( + # start_solo() now VERIFIES the container reached Running before + # reporting success (0.5.4) — there is no real docker here, so stub + # the state probe. See TestLaunchConfirmation for that contract. + NvidiaBackend, "_docker_container_state", return_value="running", ): result = backend.start() assert result is True @@ -270,7 +275,18 @@ def test_start_solo_invokes_docker_run_with_expected_args(self): ), mock.patch( "ainode.engine.backends.nvidia.build_nccl_ib_hca_whitelist", return_value="mlx5_1,mlx5_3", - ), mock.patch.object(backend, "_docker_stop_and_rm_best_effort") as preclean: + ), mock.patch.object( + backend, "_docker_stop_and_rm_best_effort" + ) as preclean, mock.patch.object( + backend, "_docker_container_state", return_value="running", + ), mock.patch.object( + # 0.5.4: the launch path inspects the image ENTRYPOINT (via + # subprocess.run, which itself uses Popen) to decide whether to emit + # `vllm serve`. Stub it to the pinned default image's shim so this + # test keeps asserting on the docker-run argv alone. + backend, "_image_entrypoint", + return_value=["/opt/nvidia/nvidia_entrypoint.sh"], + ): result = backend.start_solo() assert result is True