Four launch-path fixes found benchmarking on a real node

ainode/api/server.py

@@ -50,6 +50,23 @@
 
 logger = logging.getLogger(__name__)
 
+def _client_max_bytes(config) -> int:
+    """Inbound request-body ceiling for the API server, in bytes.
+
+    NOT optional: aiohttp defaults to 1 MB, which rejects long-context prompts
+    at the proxy. A model advertising 262k context can only be fed ~190k tokens
+    through our own endpoint before the caller gets an opaque 413 that says
+    nothing about which hop refused it (found 2026-08-25 benchmarking decode
+    against context depth). A bad value falls back to the default rather than
+    producing a server that rejects every body.
+    """
+    try:
+        mb = int(getattr(config, "max_request_mb", 64))
+    except (TypeError, ValueError):
+        mb = 64
+    return max(1, mb) * 1024 * 1024
+
+
 def create_app(
     config: Optional[NodeConfig] = None,
     engine=None,
@@ -71,7 +88,10 @@ def create_app(
 
     auth_config = AuthConfig.load()
 
-    app = web.Application(middlewares=[cors_middleware, request_log_middleware, auth_middleware])
+    app = web.Application(
+        middlewares=[cors_middleware, request_log_middleware, auth_middleware],
+        client_max_size=_client_max_bytes(config),
+    )
     init_server_state(app)
     # Instantiate shared services
     collector = MetricsCollector()

ainode/core/config.py

@@ -4,7 +4,7 @@
 import json
 from pathlib import Path
 from dataclasses import dataclass, asdict, field
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 AINODE_HOME = Path(os.environ.get("AINODE_HOME", Path.home() / ".ainode"))
 CONFIG_FILE = AINODE_HOME / "config.json"
@@ -73,6 +73,20 @@ class NodeConfig:
     # Setting this also disables the 0.17-era GB10 workarounds that would
     # otherwise be forced on (see NvidiaBackend._is_pinned_default_image).
     engine_image: str = ""
+    # Per-instance environment for the engine container. Some engine features
+    # are selected by env var, not by a `vllm serve` flag — the b12x FP4 kernel
+    # path is VLLM_NVFP4_GEMM_BACKEND + friends, with no CLI equivalent. Merged
+    # OVER the computed NCCL env at launch, so a recipe can also correct an
+    # autodetected NCCL value when a model needs it. Deliberately unvalidated,
+    # same as extra_vllm_args: the engine is the authority on what it accepts.
+    extra_env: Dict[str, str] = field(default_factory=dict)
+    # Max inbound request body for the API server, in MB. aiohttp defaults to
+    # 1 MB, which silently caps a 262k-context model at roughly 190k tokens of
+    # prompt: the proxy 413s the request before the engine ever sees it, and the
+    # caller gets "Request Entity Too Large" with nothing pointing at us. Sized
+    # for a 1M-token context (~5 MB of text) plus base64 image/video parts on
+    # the multimodal models, with headroom.
+    max_request_mb: int = 64
 
     # Cluster
     cluster_enabled: bool = True

ainode/engine/backends/nvidia.py

@@ -93,6 +93,9 @@
 
 # NCCL tuning from Phase 1 floor verification — see
 # ops/slices/nvidia-vllm-engine/runbooks/01-nccl-floor-verification.md.
+# A cold engine image is ~20 GB; a first pull on a slow link needs real room.
+IMAGE_PULL_TIMEOUT = 3600
+
 NCCL_IB_GID_INDEX = "3"
 MASTER_PORT = "29501"
 
@@ -160,6 +163,12 @@ def start_solo(self) -> bool:
         # Conflict. The head path already does this (see _launch_head_container);
         # solo needs it too.
         self._docker_stop_and_rm_best_effort(container_name)
+        # Before _build_solo_docker_cmd, because the argv prefix is derived from
+        # the image's ENTRYPOINT and inspecting a missing image yields nothing.
+        if not self.ensure_image(self._engine_image()):
+            logger.error("Engine image %s unavailable; not launching %s",
+                         self._engine_image(), self.config.model)
+            return False
         cmd = self._build_solo_docker_cmd(container_name)
         env = self._build_env_for_subprocess()
 
@@ -722,6 +731,20 @@ def _effective_kv_cache_dtype(self) -> str:
             return "auto"
         return dtype
 
+    def _engine_env(self, nccl_env: Dict[str, str]) -> Dict[str, str]:
+        """Env for the engine container: computed NCCL env + per-instance extras.
+
+        ``extra_env`` is applied LAST so a recipe wins over an autodetected
+        value. Some engine features have no CLI flag at all (the b12x FP4 path
+        is selected purely by VLLM_NVFP4_GEMM_BACKEND and friends), so without
+        this they can only be reached by hand-rolling a container — which is
+        exactly what the launch path exists to avoid.
+        """
+        env = dict(nccl_env or {})
+        for key, value in (getattr(self.config, "extra_env", None) or {}).items():
+            env[str(key)] = str(value)
+        return env
+
     def _engine_image(self) -> str:
         """Container image for THIS instance — per-load override, else the
         fleet default. Lets one node run a 0.17 model and a 0.27 model side by
@@ -754,6 +777,46 @@ def _serve_argv_prefix(self, image: str) -> List[str]:
             return ["serve"]
         return ["vllm", "serve"]           # shim/shell entrypoint
 
+    def _image_present(self, image: str) -> bool:
+        """True if the image is already on this host."""
+        try:
+            out = subprocess.run(["docker", "image", "inspect", image],
+                                 capture_output=True, text=True, timeout=20)
+            return out.returncode == 0
+        except Exception:
+            return False
+
+    def ensure_image(self, image: str, timeout: float = IMAGE_PULL_TIMEOUT) -> bool:
+        """Make sure ``image`` is on this host, pulling it if it isn't.
+
+        Per-model ``engine_image`` means a node can be asked for an image it has
+        never run. Without this the launch just fails: docker starts an implicit
+        pull, the launch confirmation times out underneath it, and the caller
+        gets a bare "Failed to launch engine" with no mention of an image. The
+        entrypoint probe degrades too, since ``docker inspect`` on a missing
+        image returns nothing and we silently fall back to a default prefix.
+        Observed 2026-08-25 loading a recipe model onto a fresh node.
+        """
+        if self._image_present(image):
+            return True
+        logger.info("Engine image %s not present; pulling (this can take several "
+                    "minutes for a ~20 GB image)", image)
+        try:
+            out = subprocess.run(["docker", "pull", image],
+                                 capture_output=True, text=True, timeout=timeout)
+        except subprocess.TimeoutExpired:
+            logger.error("Timed out pulling engine image %s after %ss", image, timeout)
+            return False
+        except Exception as exc:
+            logger.error("Could not pull engine image %s: %s", image, exc)
+            return False
+        if out.returncode != 0:
+            logger.error("Failed to pull engine image %s: %s", image,
+                         (out.stderr or out.stdout or "").strip()[-400:])
+            return False
+        logger.info("Pulled engine image %s", image)
+        return True
+
     def _image_entrypoint(self, image: str) -> List[str]:
         """The image's configured ENTRYPOINT, or [] when unknown."""
         try:
@@ -873,7 +936,7 @@ def _build_solo_docker_cmd(self, container_name: str) -> List[str]:
             "-v",
             f"{models_src}:{self.MODELS_MOUNT}:ro",
         ]
-        for key, value in nccl_env.items():
+        for key, value in self._engine_env(nccl_env).items():
             cmd.extend(["-e", f"{key}={value}"])
 
         image = self._engine_image()
@@ -948,7 +1011,7 @@ def _build_ray_docker_cmd(
             # peer's home-dir cache, which isn't under AINODE_HOME.
             "-v", f"{self._host_path(hf_cache_dir)}:/root/.cache/huggingface",
         ]
-        for key, value in nccl_env.items():
+        for key, value in self._engine_env(nccl_env).items():
             cmd.extend(["-e", f"{key}={value}"])
         cmd.extend([self._engine_image(), "-c", ray_cmd])
         return cmd
@@ -1134,7 +1197,7 @@ def _build_run_cluster_cmd(
             f"--{role}",
             hf_cache_dir,
         ]
-        for key, value in nccl_env.items():
+        for key, value in self._engine_env(nccl_env).items():
             cmd.extend(["-e", f"{key}={value}"])
         return cmd
 

ainode/engine/instance_manager.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import socket
 from dataclasses import dataclass
 from typing import Dict, List, Optional
 
@@ -55,10 +56,35 @@ def is_empty(self) -> bool:
     def used_ports(self) -> set:
         return {i.record.api_port for i in self._instances.values()}
 
-    def allocate_port(self) -> int:
-        """Lowest free port from base_port up, not held by an existing instance."""
+    @staticmethod
+    def _port_bindable(port: int, host: str = "0.0.0.0") -> bool:
+        """True if nothing on the HOST is already listening on this port.
+
+        The engine container runs on the host network, so a port owned by any
+        other process (not just another AINode instance) collides. Without this
+        check the engine launches, fails deep in startup with
+        ``OSError: [Errno 98] Address already in use``, and the caller only sees
+        a generic launch failure. Observed 2026-08-25 on a node where an
+        unrelated service had held 8000 for weeks.
+        """
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                sock.bind((host, port))
+            return True
+        except OSError:
+            return False
+
+    def allocate_port(self, probe: bool = True) -> int:
+        """Lowest free port from base_port up.
+
+        Skips ports held by an existing instance AND, unless ``probe`` is off,
+        ports already bound by anything else on the host.
+        """
         used = self.used_ports()
         port = self._base_port
-        while port in used:
+        for _ in range(256):
+            if port not in used and (not probe or self._port_bindable(port)):
+                return port
             port += 1
         return port

ainode/models/api_routes.py

@@ -145,7 +145,7 @@ def load_instance_manifest() -> list:
 
 _OVERRIDE_KEYS = ("served_model_name", "max_model_len", "kv_cache_dtype",
                   "kv_cache_dtype_explicit", "quantization", "trust_remote_code",
-                  "extra_vllm_args", "engine_image")
+                  "extra_vllm_args", "engine_image", "extra_env")
 
 
 def catalog_recipe(model: str) -> dict:
@@ -155,7 +155,8 @@ def catalog_recipe(model: str) -> dict:
     flag set (spec-decode, MoE/mamba backends, reasoning + tool-call parsers).
     Carrying that in the catalog is what makes them a one-click load instead of a
     hand-rolled container. Returns {} for anything not curated. Keys:
-    ``engine_image``, ``extra_vllm_args``, and ``gpu_memory_utilization``.
+    ``engine_image``, ``extra_vllm_args``, ``extra_env``, and
+    ``gpu_memory_utilization``.
     """
     from ainode.models.registry import CURATED_CLUSTER_MODELS
     m = (model or "").strip()
@@ -168,6 +169,8 @@ def catalog_recipe(model: str) -> dict:
                 recipe["engine_image"] = info.engine_image
             if getattr(info, "extra_vllm_args", None):
                 recipe["extra_vllm_args"] = list(info.extra_vllm_args)
+            if getattr(info, "extra_env", None):
+                recipe["extra_env"] = dict(info.extra_env)
             if getattr(info, "recommended_gmu", 0):
                 recipe["gpu_memory_utilization"] = info.recommended_gmu
             return recipe
@@ -568,6 +571,16 @@ async def handle_model_load(request: web.Request) -> web.Response:
                           "(e.g. [\"--moe-backend\", \"marlin\"]) or a shell-style string"},
                 status=400)
         overrides["extra_vllm_args"] = [str(a) for a in raw]
+    if body.get("extra_env") is not None:
+        raw = body["extra_env"]
+        if not isinstance(raw, dict) or not all(
+                isinstance(k, str) and k and isinstance(v, (str, int, float, bool))
+                for k, v in raw.items()):
+            return web.json_response(
+                {"error": "extra_env must be an object of NAME -> value "
+                          "(e.g. {\"VLLM_NVFP4_GEMM_BACKEND\": \"flashinfer-b12x\"})"},
+                status=400)
+        overrides["extra_env"] = {k: str(v) for k, v in raw.items()}
     if body.get("engine_image") is not None:
         img = str(body["engine_image"]).strip()
         if " " in img:
@@ -580,7 +593,7 @@ async def handle_model_load(request: web.Request) -> web.Response:
     # engine image and flags it actually needs. Anything the caller stated
     # explicitly above wins; the recipe only fills the gaps.
     recipe = catalog_recipe(model)
-    for key in ("engine_image", "extra_vllm_args"):
+    for key in ("engine_image", "extra_vllm_args", "extra_env"):
         if key in recipe and key not in overrides:
             overrides[key] = recipe[key]
     if gmu is None and "gpu_memory_utilization" in recipe:

ainode/models/registry.py

@@ -88,13 +88,16 @@ class ModelInfo:
     # wins over the recipe; the recipe only fills what wasn't specified.
     engine_image: str = ""          # "" = fleet default engine image
     extra_vllm_args: list = None    # verbatim `vllm serve` flags
+    extra_env: dict = None          # engine-container env (e.g. b12x kernel selection)
     recommended_gmu: float = 0.0    # 0 = use node default gpu_memory_utilization
 
     def __post_init__(self):
         if self.capabilities is None:
             self.capabilities = []
         if self.extra_vllm_args is None:
             self.extra_vllm_args = []
+        if self.extra_env is None:
+            self.extra_env = {}
 
     def to_dict(self) -> dict:
         return asdict(self)