From 09d6339453290898dbb5edc4ecdb9a0e30a02ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Sun, 26 Jul 2026 13:04:28 +0300 Subject: [PATCH 1/7] =?UTF-8?q?=E2=9C=A8=20Add=20Ray=20deployment=20option?= =?UTF-8?q?=20to=20LLM=20runtime=20and=20update=20GPU=20model=20library=20?= =?UTF-8?q?configuration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/gpu-model-library.json | 13 +++++++++++++ src/lib/database/provider/types.extended.ts | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/config/gpu-model-library.json b/src/config/gpu-model-library.json index 12bc0994..2d24d544 100644 --- a/src/config/gpu-model-library.json +++ b/src/config/gpu-model-library.json @@ -100,6 +100,19 @@ "healthPath": "/api/tags", "openaiCompatible": true, "openaiBasePath": "/v1" + }, + "ray": { + "image": "rayproject/ray-llm:2.47.1-py311-cu124", + "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM.", + "args": [ + "python3", "-c", + "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}})]}); serve.run(app, blocking=True)" + ], + "port": 8000, + "healthPath": "/-/healthz", + "openaiCompatible": true, + "openaiBasePath": "/v1", + "secretEnv": ["HUGGING_FACE_HUB_TOKEN"] } } }, diff --git a/src/lib/database/provider/types.extended.ts b/src/lib/database/provider/types.extended.ts index 9f391e99..75a4d4e1 100644 --- a/src/lib/database/provider/types.extended.ts +++ b/src/lib/database/provider/types.extended.ts @@ -529,7 +529,7 @@ export interface IGpuSlice { updatedAt?: Date; } -export type LlmDeploymentRuntime = 'vllm' | 'tgi' | 'ollama' | 'custom'; +export type LlmDeploymentRuntime = 'vllm' | 'tgi' | 'ollama' | 'ray' | 'custom'; export type LlmDeploymentDesiredState = 'running' | 'stopped'; From 420c196b072593b57f20fc29ecfe80b0a4b7eb9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Sun, 26 Jul 2026 15:24:08 +0300 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=94=A8=20Add=20memory=20limit=20to=20?= =?UTF-8?q?dev:watch=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 37956f7c..b86afbbc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "workspaces": [], "scripts": { "dev": "NODE_ENV=development NODE_OPTIONS=--max-old-space-size=16384 node --import tsx src/server/index.ts --dev", - "dev:watch": "NODE_ENV=development node --watch --watch-path=src --import tsx src/server/index.ts --dev", + "dev:watch": "NODE_ENV=development NODE_OPTIONS=--max-old-space-size=16384 node --watch --watch-path=src --import tsx src/server/index.ts --dev", "build": "next build", "start": "NODE_ENV=production node --import tsx src/server/index.ts", "lint": "eslint", From 195335c83588923accb2ef232d2489dddbfd2a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Sun, 26 Jul 2026 16:16:49 +0300 Subject: [PATCH 3/7] =?UTF-8?q?=E2=9C=A8=20Add=20shmSizeBytes=20configurat?= =?UTF-8?q?ion=20for=20LLM=20deployments=20and=20update=20SQLite=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/database/provider/types.extended.ts | 6 ++++++ src/lib/database/sqlite/base.ts | 10 ++++++++++ src/lib/database/sqlite/schema.ts | 1 + 3 files changed, 17 insertions(+) diff --git a/src/lib/database/provider/types.extended.ts b/src/lib/database/provider/types.extended.ts index 75a4d4e1..bb633e0b 100644 --- a/src/lib/database/provider/types.extended.ts +++ b/src/lib/database/provider/types.extended.ts @@ -573,6 +573,12 @@ export interface ILlmDeployment { readOnly?: boolean; }>; restart: 'no' | 'on-failure' | 'always' | 'unless-stopped'; + /** + * `/dev/shm` size in bytes for the container, or `null` to use the + * agent's per-runtime default (see `resolveShmSizeBytes` in the agent's + * `docker.ts`). Operator-chosen at deploy time; not auto-inferred. + */ + shmSizeBytes: number | null; desiredState: LlmDeploymentDesiredState; actualState: LlmDeploymentActualState; /** Container id from the agent's last report. */ diff --git a/src/lib/database/sqlite/base.ts b/src/lib/database/sqlite/base.ts index e24e0bab..663c4f7e 100644 --- a/src/lib/database/sqlite/base.ts +++ b/src/lib/database/sqlite/base.ts @@ -437,6 +437,16 @@ export class SQLiteProviderBase { 'projectId', 'projectId TEXT', ); + // Explicit `/dev/shm` override for a deployment's container, falling + // back to the agent's per-runtime default when null (added 2026-07-26, + // alongside the Ray Serve runtime — vLLM/Ray need more than Docker's + // 64 MiB default to avoid a silent engine-startup hang). + this.ensureTableColumn( + db, + TABLES.llmDeployments, + 'shmSizeBytes', + 'shmSizeBytes INTEGER', + ); // Sandbox instance per-instance env (added later). Safe to ensure on boot. this.ensureTableColumn(db, 'sandbox_instances', 'env', 'env TEXT'); this.ensureTableColumn( diff --git a/src/lib/database/sqlite/schema.ts b/src/lib/database/sqlite/schema.ts index 31febe1f..e1d3a317 100644 --- a/src/lib/database/sqlite/schema.ts +++ b/src/lib/database/sqlite/schema.ts @@ -1915,6 +1915,7 @@ export const TENANT_SCHEMA_SQL = ` healthPath TEXT NOT NULL DEFAULT '/health', volumes TEXT NOT NULL DEFAULT '[]', restart TEXT NOT NULL DEFAULT 'unless-stopped', + shmSizeBytes INTEGER, desiredState TEXT NOT NULL DEFAULT 'running', actualState TEXT NOT NULL DEFAULT 'pending', containerId TEXT, From 6926952ff890a578d5a4cd1418b63a4fc223f989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Sun, 26 Jul 2026 16:50:33 +0300 Subject: [PATCH 4/7] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Add=20GPU=20fleet=20st?= =?UTF-8?q?andard=20resolvers=20for=20hosts=20and=20deployments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/layout/breadcrumbResolvers.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/components/layout/breadcrumbResolvers.ts b/src/components/layout/breadcrumbResolvers.ts index 9625dee2..aab3248a 100644 --- a/src/components/layout/breadcrumbResolvers.ts +++ b/src/components/layout/breadcrumbResolvers.ts @@ -228,6 +228,16 @@ const STANDARD_RESOLVERS: BreadcrumbResolver[] = [ buildUrl: (name) => `/api/tracing/agents/${encodeURIComponent(name)}/overview`, pickLabel: (b) => wrappedName(b, 'agent', 'label', 'name', 'key'), }), + createStandardResolver({ + path: ['gpu-fleet', 'hosts'], + buildUrl: (id) => `/api/gpu-fleet/hosts/${encodeURIComponent(id)}`, + pickLabel: (b) => wrappedName(b, 'host'), + }), + createStandardResolver({ + path: ['gpu-fleet', 'deployments'], + buildUrl: (id) => `/api/gpu-fleet/deployments/${encodeURIComponent(id)}`, + pickLabel: (b) => wrappedName(b, 'deployment'), + }), ]; /* ----- Custom resolvers --------------------------------------------------- */ From 3c5da120991193de4dff3eed7170dc536b32b7d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Mon, 27 Jul 2026 01:39:10 +0300 Subject: [PATCH 5/7] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Update=20Ray=20Serve?= =?UTF-8?q?=20LLM=20configuration=20to=20specify=20dtype=20as=20'half'=20f?= =?UTF-8?q?or=20compatibility=20with=20older=20GPUs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/gpu-model-library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/gpu-model-library.json b/src/config/gpu-model-library.json index 2d24d544..5e2f3832 100644 --- a/src/config/gpu-model-library.json +++ b/src/config/gpu-model-library.json @@ -103,10 +103,10 @@ }, "ray": { "image": "rayproject/ray-llm:2.47.1-py311-cu124", - "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM.", + "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM. `dtype: half` is explicit (not 'auto') because the vLLM 0.8.5 bundled in this Ray image does not auto-downgrade bfloat16 to float16 on <8.0 compute-capability GPUs the way the newer standalone vllm/vllm-openai image does — Qwen3's HF config defaults to bfloat16, which hard-crashes on T4/V100 without this override.", "args": [ "python3", "-c", - "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}})]}); serve.run(app, blocking=True)" + "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" ], "port": 8000, "healthPath": "/-/healthz", From b44a3366a93230d80fd394ee0ae3f9f45cbb5332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Mon, 27 Jul 2026 02:20:07 +0300 Subject: [PATCH 6/7] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Update=20Ray=20Serve?= =?UTF-8?q?=20LLM=20configuration=20to=20include=20mandatory=20HTTP=20prox?= =?UTF-8?q?y=20binding=20for=20Docker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/gpu-model-library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/gpu-model-library.json b/src/config/gpu-model-library.json index 5e2f3832..c5d563d7 100644 --- a/src/config/gpu-model-library.json +++ b/src/config/gpu-model-library.json @@ -103,10 +103,10 @@ }, "ray": { "image": "rayproject/ray-llm:2.47.1-py311-cu124", - "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM. `dtype: half` is explicit (not 'auto') because the vLLM 0.8.5 bundled in this Ray image does not auto-downgrade bfloat16 to float16 on <8.0 compute-capability GPUs the way the newer standalone vllm/vllm-openai image does — Qwen3's HF config defaults to bfloat16, which hard-crashes on T4/V100 without this override.", + "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM. `dtype: half` is explicit (not 'auto') because the vLLM 0.8.5 bundled in this Ray image does not auto-downgrade bfloat16 to float16 on <8.0 compute-capability GPUs the way the newer standalone vllm/vllm-openai image does — Qwen3's HF config defaults to bfloat16, which hard-crashes on T4/V100 without this override. `serve.start(http_options={'host': '0.0.0.0'})` before serve.run() is mandatory: Ray Serve's HTTP proxy binds to localhost by default, which is unreachable through Docker's published port — without this the container looks 'starting' forever with every request getting Connection reset by peer.", "args": [ "python3", "-c", - "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" + "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; serve.start(http_options={'host': '0.0.0.0', 'port': 8000}); app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" ], "port": 8000, "healthPath": "/-/healthz", From 01d0becfb9beb4c86f73e5f2f6a3370c7ffa9614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sercan=20=C3=9Cste?= Date: Mon, 27 Jul 2026 02:45:15 +0300 Subject: [PATCH 7/7] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Fix=20model=20ID=20ref?= =?UTF-8?q?erence=20in=20Ray=20Serve=20LLM=20configuration=20for=20Qwen=20?= =?UTF-8?q?3=208B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/gpu-model-library.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/gpu-model-library.json b/src/config/gpu-model-library.json index c5d563d7..0e904127 100644 --- a/src/config/gpu-model-library.json +++ b/src/config/gpu-model-library.json @@ -106,7 +106,7 @@ "_meta": "Ray Serve LLM wraps vLLM under Ray Serve for autoscaling/multi-replica serving. rayproject/ray-llm ships no ENTRYPOINT, so `args` is the full process command: boot a tiny inline script that builds an OpenAI-compatible Serve app and calls serve.run(). Requires an NVIDIA GPU host, same as vLLM. `dtype: half` is explicit (not 'auto') because the vLLM 0.8.5 bundled in this Ray image does not auto-downgrade bfloat16 to float16 on <8.0 compute-capability GPUs the way the newer standalone vllm/vllm-openai image does — Qwen3's HF config defaults to bfloat16, which hard-crashes on T4/V100 without this override. `serve.start(http_options={'host': '0.0.0.0'})` before serve.run() is mandatory: Ray Serve's HTTP proxy binds to localhost by default, which is unreachable through Docker's published port — without this the container looks 'starting' forever with every request getting Connection reset by peer.", "args": [ "python3", "-c", - "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; serve.start(http_options={'host': '0.0.0.0', 'port': 8000}); app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'qwen3-8b', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" + "from ray import serve; from ray.serve.llm import LLMConfig, build_openai_app; serve.start(http_options={'host': '0.0.0.0', 'port': 8000}); app = build_openai_app({'llm_configs': [LLMConfig(model_loading_config={'model_id': 'Qwen/Qwen3-8B', 'model_source': 'Qwen/Qwen3-8B'}, deployment_config={'autoscaling_config': {'min_replicas': 1, 'max_replicas': 1}}, engine_kwargs={'tensor_parallel_size': {{gpuCount}}, 'dtype': 'half'})]}); serve.run(app, blocking=True)" ], "port": 8000, "healthPath": "/-/healthz",