diff --git a/package.json b/package.json index 37956f7c..b86afbbc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "workspaces": [], "scripts": { "dev": "NODE_ENV=development NODE_OPTIONS=--max-old-space-size=16384 node --import tsx src/server/index.ts --dev", - "dev:watch": "NODE_ENV=development node --watch --watch-path=src --import tsx src/server/index.ts --dev", + "dev:watch": "NODE_ENV=development NODE_OPTIONS=--max-old-space-size=16384 node --watch --watch-path=src --import tsx src/server/index.ts --dev", "build": "next build", "start": "NODE_ENV=production node --import tsx src/server/index.ts", "lint": "eslint", diff --git a/src/components/layout/breadcrumbResolvers.ts b/src/components/layout/breadcrumbResolvers.ts index 9625dee2..aab3248a 100644 --- a/src/components/layout/breadcrumbResolvers.ts +++ b/src/components/layout/breadcrumbResolvers.ts @@ -228,6 +228,16 @@ const STANDARD_RESOLVERS: BreadcrumbResolver[] = [ buildUrl: (name) => `/api/tracing/agents/${encodeURIComponent(name)}/overview`, pickLabel: (b) => wrappedName(b, 'agent', 'label', 'name', 'key'), }), + createStandardResolver({ + path: ['gpu-fleet', 'hosts'], + buildUrl: (id) => `/api/gpu-fleet/hosts/${encodeURIComponent(id)}`, + pickLabel: (b) => wrappedName(b, 'host'), + }), + createStandardResolver({ + path: ['gpu-fleet', 'deployments'], + buildUrl: (id) => `/api/gpu-fleet/deployments/${encodeURIComponent(id)}`, + pickLabel: (b) => wrappedName(b, 'deployment'), + }), ]; /* ----- Custom resolvers --------------------------------------------------- */ diff --git a/src/config/gpu-model-library.json b/src/config/gpu-model-library.json index 12bc0994..0e904127 100644 --- a/src/config/gpu-model-library.json +++ b/src/config/gpu-model-library.json @@ -100,6 +100,19 @@ "healthPath": "/api/tags", "openaiCompatible": true, "openaiBasePath": "/v1" + }, + "ray": { + "image": "rayproject/ray-llm:2.47.1-py311-cu124", + "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM. `dtype: half` is explicit (not 'auto') because the vLLM 0.8.5 bundled in this Ray image does not auto-downgrade bfloat16 to float16 on <8.0 compute-capability GPUs the way the newer standalone vllm/vllm-openai image does — Qwen3's HF config defaults to bfloat16, which hard-crashes on T4/V100 without this override. `serve.start(http_options={'host': '0.0.0.0'})` before serve.run() is mandatory: Ray Serve's HTTP proxy binds to localhost by default, which is unreachable through Docker's published port — without this the container looks 'starting' forever with every request getting Connection reset by peer.", + "args": [ + "python3", "-c", + "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; serve.start(http_options={'host': '0.0.0.0', 'port': 8000}); app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'Qwen/Qwen3-8B', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" + ], + "port": 8000, + "healthPath": "/-/healthz", + "openaiCompatible": true, + "openaiBasePath": "/v1", + "secretEnv": ["HUGGING_FACE_HUB_TOKEN"] } } }, diff --git a/src/lib/database/provider/types.extended.ts b/src/lib/database/provider/types.extended.ts index 9f391e99..bb633e0b 100644 --- a/src/lib/database/provider/types.extended.ts +++ b/src/lib/database/provider/types.extended.ts @@ -529,7 +529,7 @@ export interface IGpuSlice { updatedAt?: Date; } -export type LlmDeploymentRuntime = 'vllm' | 'tgi' | 'ollama' | 'custom'; +export type LlmDeploymentRuntime = 'vllm' | 'tgi' | 'ollama' | 'ray' | 'custom'; export type LlmDeploymentDesiredState = 'running' | 'stopped'; @@ -573,6 +573,12 @@ export interface ILlmDeployment { readOnly?: boolean; }>; restart: 'no' | 'on-failure' | 'always' | 'unless-stopped'; + /** + * `/dev/shm` size in bytes for the container, or `null` to use the + * agent's per-runtime default (see `resolveShmSizeBytes` in the agent's + * `docker.ts`). Operator-chosen at deploy time; not auto-inferred. + */ + shmSizeBytes: number | null; desiredState: LlmDeploymentDesiredState; actualState: LlmDeploymentActualState; /** Container id from the agent's last report. */ diff --git a/src/lib/database/sqlite/base.ts b/src/lib/database/sqlite/base.ts index e24e0bab..663c4f7e 100644 --- a/src/lib/database/sqlite/base.ts +++ b/src/lib/database/sqlite/base.ts @@ -437,6 +437,16 @@ export class SQLiteProviderBase { 'projectId', 'projectId TEXT', ); + // Explicit `/dev/shm` override for a deployment's container, falling + // back to the agent's per-runtime default when null (added 2026-07-26, + // alongside the Ray Serve runtime — vLLM/Ray need more than Docker's + // 64 MiB default to avoid a silent engine-startup hang). + this.ensureTableColumn( + db, + TABLES.llmDeployments, + 'shmSizeBytes', + 'shmSizeBytes INTEGER', + ); // Sandbox instance per-instance env (added later). Safe to ensure on boot. this.ensureTableColumn(db, 'sandbox_instances', 'env', 'env TEXT'); this.ensureTableColumn( diff --git a/src/lib/database/sqlite/schema.ts b/src/lib/database/sqlite/schema.ts index 31febe1f..e1d3a317 100644 --- a/src/lib/database/sqlite/schema.ts +++ b/src/lib/database/sqlite/schema.ts @@ -1915,6 +1915,7 @@ export const TENANT_SCHEMA_SQL = ` healthPath TEXT NOT NULL DEFAULT '/health', volumes TEXT NOT NULL DEFAULT '[]', restart TEXT NOT NULL DEFAULT 'unless-stopped', + shmSizeBytes INTEGER, desiredState TEXT NOT NULL DEFAULT 'running', actualState TEXT NOT NULL DEFAULT 'pending', containerId TEXT,