From dd3185025478279b33f208d64f7f33c5f0f404e5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 11:39:27 -0400 Subject: [PATCH 1/3] fix: Distinguish workspace admission capacity from execution expiry --- .github/workflows/ci.yml | 7 + packages/code/README.md | 13 ++ service/src/bridge/fleet.test.ts | 132 ++++++++++++++++++ service/src/bridge/store.ts | 14 ++ service/src/bridge/worker-admission.test.ts | 2 +- .../src/sandbox-backend/remote-bridge.test.ts | 1 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/workspace-tools/router.test.ts | 2 + service/src/workspace-tools/router.ts | 2 + 9 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 service/src/bridge/fleet.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f32178..7259ec87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,13 @@ jobs: - name: Install Redis for ledger integration tests run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Verify fleet admission with isolated Redis + run: | + redis_socket="$RUNNER_TEMP/byom-admission.sock" + redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes + trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + - name: Build service run: bun run build diff --git a/packages/code/README.md b/packages/code/README.md index 720a0b35..08dc54a7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -692,6 +692,19 @@ librechat-code run \ --allow-workspace-commands ``` +Slots are per machine, not a fleet-wide execution limit. A busy machine does not +consume another machine's slots. Requests for the same root remain serialized, +including commands started through background tools. Independent checkouts can +use different slots; selecting subdirectories beneath one registered parent root +does not create separate scheduling boundaries. Linked Git worktrees share Git +metadata and are not supported by selected-project registration. + +Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP +503, `Retry-After: 1`) means the operation was not assigned or started; wait for +capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` +or a transport timeout after dispatch, where execution may have occurred and +mutations must not be blindly retried. No automatic retry is added by this policy. + Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat `--workspace id=path` to add named roots, up to the protocol's 32-root limit. diff --git a/service/src/bridge/fleet.test.ts b/service/src/bridge/fleet.test.ts new file mode 100644 index 00000000..3eac5088 --- /dev/null +++ b/service/src/bridge/fleet.test.ts @@ -0,0 +1,132 @@ +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { randomUUID } from 'node:crypto'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; + +/** Opt-in integration check against a disposable Redis, never a deployment database. */ +test.skipIf(!process.env.BRIDGE_TEST_REDIS_URL)( + 'admission saturation is isolated across machines and independent roots', + async () => { + const redis = new Redis(process.env.BRIDGE_TEST_REDIS_URL!); + const store = new RedisBridgeStore(redis, 60, 1000, 2); + const prefix = `fleet-${randomUUID()}`; + const machines = [`${prefix}-a`, `${prefix}-b`]; + const incarnationId = 'fleet-incarnation'; + const pending: Promise[] = []; + const controller = new AbortController(); + const dispatch = ( + workerId: string, + workspaceId: string, + budgetMs = 3000, + ) => { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId, + path: 'probe', + }, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }; + const settle = async (assignment: CodeBridgeAssignment) => { + await store.acknowledgeLease( + assignment.workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(assignment.workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + }; + try { + for (const workerId of machines) { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'a' }, { id: 'b' }], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + } + const busy = dispatch(machines[0], 'a'); + const held = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(held).toBeDefined(); + const blocked = dispatch(machines[0], 'a', 300); + const independent = dispatch(machines[0], 'b'); + const otherMachine = dispatch(machines[1], 'a'); + const root = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 1, + ); + const remote = await store.lease( + machines[1], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(root?.request).toMatchObject({ workspaceId: 'b' }); + expect(remote?.workerId).toBe(machines[1]); + await settle(root!); + await settle(remote!); + await Promise.all([independent, otherMachine]); + await expect(blocked).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + await settle(held!); + await busy; + expect( + await store.lease( + machines[0], + incarnationId, + 20, + undefined, + undefined, + 0, + ), + ).toBeUndefined(); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } + }, +); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index d6b91469..81fbe672 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -47,6 +47,7 @@ export class BridgeStoreError extends Error { | 'WORKER_UNAUTHORIZED' | 'WORKER_BUSY' | 'WORKER_QUEUE_FULL' + | 'WORKSPACE_QUEUE_TIMEOUT' | 'ASSIGNMENT_EXPIRED' | 'ASSIGNMENT_FENCED' | 'ASSIGNMENT_NOT_FOUND' @@ -1130,6 +1131,19 @@ export class RedisBridgeStore { } throw error; } + } catch (error) { + // Before an assignment exists, no enqueue or worker execution is possible. + // Once enqueue starts, a timeout is ambiguous and must retain its old code. + if ( + admission != null && assignment == null && !args.signal.aborted && + error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' + ) { + throw new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + throw error; } finally { if (admission != null) { // Expiry remains the fallback if Redis is unavailable during cancellation. diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index db2e5956..e92adcbd 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -114,7 +114,7 @@ test('an expired queued call never reaches the worker and does not strand later const assignment = await store.lease(workerId, incarnationId, 1000); await expect( dispatch('expired', new AbortController(), 25, 1000), - ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + ).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); const third = dispatch('third'); await settle(assignment); await first; diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index 697271ee..4f50898e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -108,6 +108,7 @@ describe('RemoteBridgeSandboxBackend', () => { WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], WORKER_QUEUE_FULL: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + WORKSPACE_QUEUE_TIMEOUT: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 6e06eda8..2cdbd767 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -19,6 +19,7 @@ const bridgeErrorCodes = { WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', WORKER_BUSY: 'BRIDGE_WORKER_BUSY', WORKER_QUEUE_FULL: 'BRIDGE_WORKER_BUSY', + WORKSPACE_QUEUE_TIMEOUT: 'BRIDGE_WORKER_BUSY', ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index ae291a4b..04738b8a 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -320,6 +320,7 @@ test.each([ ['ASSIGNMENT_EXPIRED', 504], ['WORKER_OFFLINE', 503], ['WORKER_BUSY', 503], + ['WORKSPACE_QUEUE_TIMEOUT', 503], ['WORKER_MISMATCH', 409], ] as const)('logs store rejection %s with actual HTTP %i', async (errorCode, expectedStatus) => { const app = express(); @@ -360,6 +361,7 @@ test.each([ }), }); expect(response.status).toBe(expectedStatus); + expect(response.headers.get('retry-after')).toBe(errorCode === 'WORKSPACE_QUEUE_TIMEOUT' ? '1' : null); await response.text(); expect(logSpy).toHaveBeenCalledTimes(1); expect(logSpy).toHaveBeenCalledWith( diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 17085963..eb89370e 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -36,6 +36,7 @@ function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promi } export function bridgeStoreStatus(error: BridgeStoreError): number { + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') return 503; if (error.code === 'WORKER_QUEUE_FULL') return 429; if (error.code === 'WORKER_UNAUTHORIZED') return 403; if (error.code === 'ASSIGNMENT_INVALID') return 400; @@ -172,6 +173,7 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } catch (error) { if (error instanceof BridgeStoreError) { outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code, From e50c37667c3070409912e716f478181b4a8e9332 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 11:42:40 -0400 Subject: [PATCH 2/3] test: Preserve execution uncertainty while classifying blocked follow-ups --- service/src/bridge/concurrent-worker.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index ea5ac601..c107576a 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -281,7 +281,7 @@ for (const failure of [ ).rejects.toMatchObject({ code: failure === 'delivery-outage' - ? 'ASSIGNMENT_EXPIRED' + ? 'WORKSPACE_QUEUE_TIMEOUT' : 'WORKSPACE_QUARANTINED', }); await expect( From 8c61bd1c616af6c3cb2ecf4d7689854545029d74 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 11:44:40 -0400 Subject: [PATCH 3/3] fix: Classify admission expiry at the enqueue boundary --- service/src/bridge/store.ts | 14 +++++++------ service/src/bridge/worker-admission.test.ts | 22 ++++++++++++++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 81fbe672..4eaabd7f 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -839,6 +839,7 @@ export class RedisBridgeStore { ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; + let enqueueAttempted = false; let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; @@ -1025,12 +1026,14 @@ export class RedisBridgeStore { this.assertDispatchActive(args.signal, args.deadlineAtMs); assignment.incarnationId = registration.incarnationId; queued = await this.dispatchCommand( - () => - this.enqueueForActiveIncarnation( + () => { + enqueueAttempted = true; + return this.enqueueForActiveIncarnation( assignment!, ttlSeconds, readyToken, - ), + ); + }, args, 'Bridge assignment enqueue', ); @@ -1132,10 +1135,9 @@ export class RedisBridgeStore { throw error; } } catch (error) { - // Before an assignment exists, no enqueue or worker execution is possible. - // Once enqueue starts, a timeout is ambiguous and must retain its old code. + // Once enqueue starts, even a lost Redis response may hide execution. if ( - admission != null && assignment == null && !args.signal.aborted && + admission != null && !enqueueAttempted && !args.signal.aborted && error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' ) { throw new BridgeStoreError( diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index e92adcbd..655748b1 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from 'bun:test'; +import { afterEach, expect, spyOn, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; @@ -170,3 +170,23 @@ test('execution expires independently of an unused queue allowance', async () => expect(Date.parse(assignment!.expiresAt) - Date.now()).toBeLessThanOrEqual(150); await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); }); + +test('expiry after generation allocation but before enqueue is definitely not started', async () => { + await register(); + const now = Date.now; + const incr = redis.incr.bind(redis); + let expired = false; + const clock = spyOn(Date, 'now').mockImplementation(() => now() + (expired ? 10_000 : 0)); + const generation = spyOn(redis, 'incr').mockImplementation(async (key) => { + const value = await incr(key); + expired = true; + return value; + }); + try { + await expect(dispatch('not-enqueued')).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + } finally { + clock.mockRestore(); + generation.mockRestore(); + } + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +});